boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

- added parsing of Dublin Core - compliant metadata (see RFC 5013 and ISO 15836) to html parser

Browse Source 
- refactoring of plasmaParserDocument to use Dublin Core - compatible property names
- redesign of url handling in parser and condenser (less String-to-yacyURL conversion)
- more generics

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4352 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 17 years ago
parent cfd4fecd12
commit efd0b8371a
33 changed files with 484 additions and 453 deletions
Show Stats Download Patch File Download Diff File

4
htroot/Bookmarks.java
Unescape View File

@ -179,9 +179,9 @@ public class Bookmarks {
prop.put("mode_edit", "0"); // create mode prop.put("mode_edit", "0"); // create mode
prop.put("mode_url", comp.url().toNormalform(false, true)); prop.put("mode_url", comp.url().toNormalform(false, true));
prop.putHTML("mode_title", comp.title()); prop.putHTML("mode_title", comp.title());
prop.putHTML("mode_description", (document == null) ? comp.title(): document.getTitle()); prop.putHTML("mode_description", (document == null) ? comp.title(): document.dc_title());
prop.putHTML("mode_author", comp.author()); prop.putHTML("mode_author", comp.author());
prop.putHTML("mode_tags", (document == null) ? comp.tags() : document.getKeywords(',')); prop.putHTML("mode_tags", (document == null) ? comp.tags() : document.dc_subject(','));
prop.put("mode_public", "0"); prop.put("mode_public", "0");
prop.put("mode_feed", "0"); //TODO: check if it IS a feed prop.put("mode_feed", "0"); //TODO: check if it IS a feed
} }

22
htroot/ViewFile.java
Unescape View File

@ -297,7 +297,7 @@ public class ViewFile {
} }
} }
resMime = document.getMimeType(); resMime = document.dc_format();
String[] wordArray = wordArray(post.get("words", null)); String[] wordArray = wordArray(post.get("words", null));
if (viewMode.equals("parsed")) { if (viewMode.equals("parsed")) {
@ -310,7 +310,7 @@ public class ViewFile {
} else if (viewMode.equals("sentences")) { } else if (viewMode.equals("sentences")) {
prop.put("viewMode", VIEW_MODE_AS_PARSED_SENTENCES); prop.put("viewMode", VIEW_MODE_AS_PARSED_SENTENCES);
final Iterator sentences = document.getSentences(pre); final Iterator<StringBuffer> sentences = document.getSentences(pre);
boolean dark = true; boolean dark = true;
int i = 0; int i = 0;
@ -319,7 +319,7 @@ public class ViewFile {
// Search word highlighting // Search word highlighting
while (sentences.hasNext()) { while (sentences.hasNext()) {
sentence = ((StringBuffer) sentences.next()).toString(); sentence = sentences.next().toString();
if (sentence.trim().length() > 0) { if (sentence.trim().length() > 0) {
prop.put("viewMode_sentences_" + i + "_nr", i + 1); prop.put("viewMode_sentences_" + i + "_nr", i + 1);
prop.put("viewMode_sentences_" + i + "_text", markup(wordArray, sentence)); prop.put("viewMode_sentences_" + i + "_text", markup(wordArray, sentence));
@ -339,11 +339,11 @@ public class ViewFile {
i += putMediaInfo(prop, wordArray, i, document.getAudiolinks(), "audio", (i % 2 == 0)); i += putMediaInfo(prop, wordArray, i, document.getAudiolinks(), "audio", (i % 2 == 0));
dark = (i % 2 == 0); dark = (i % 2 == 0);
TreeSet ts = document.getImages(); TreeSet<htmlFilterImageEntry> ts = document.getImages();
Iterator tsi = ts.iterator(); Iterator<htmlFilterImageEntry> tsi = ts.iterator();
htmlFilterImageEntry entry; htmlFilterImageEntry entry;
while (tsi.hasNext()) { while (tsi.hasNext()) {
entry = (htmlFilterImageEntry) tsi.next(); entry = tsi.next();
prop.put("viewMode_links_" + i + "_nr", i); prop.put("viewMode_links_" + i + "_nr", i);
prop.put("viewMode_links_" + i + "_dark", dark ? "1" : "0"); prop.put("viewMode_links_" + i + "_dark", dark ? "1" : "0");
prop.put("viewMode_links_" + i + "_type", "image"); prop.put("viewMode_links_" + i + "_type", "image");
@ -399,9 +399,9 @@ public class ViewFile {
return message; return message;
} }
private static int putMediaInfo(serverObjects prop, String[] wordArray, int c, Map<String, String> media, String name, boolean dark) { private static int putMediaInfo(serverObjects prop, String[] wordArray, int c, Map<yacyURL, String> media, String name, boolean dark) {
Iterator<Map.Entry<String, String>> mi = media.entrySet().iterator(); Iterator<Map.Entry<yacyURL, String>> mi = media.entrySet().iterator();
Map.Entry<String, String> entry; Map.Entry<yacyURL, String> entry;
int i = 0; int i = 0;
while (mi.hasNext()) { while (mi.hasNext()) {
entry = mi.next(); entry = mi.next();
@ -409,8 +409,8 @@ public class ViewFile {
prop.put("viewMode_links_" + c + "_dark", ((dark) ? 1 : 0)); prop.put("viewMode_links_" + c + "_dark", ((dark) ? 1 : 0));
prop.putHTML("viewMode_links_" + c + "_type", name); prop.putHTML("viewMode_links_" + c + "_type", name);
prop.put("viewMode_links_" + c + "_text", markup(wordArray, (String) entry.getValue())); prop.put("viewMode_links_" + c + "_text", markup(wordArray, (String) entry.getValue()));
prop.put("viewMode_links_" + c + "_link", markup(wordArray, (String) entry.getKey())); prop.put("viewMode_links_" + c + "_link", markup(wordArray, entry.getKey().toNormalform(true, false)));
prop.put("viewMode_links_" + c + "_url", entry.getKey()); prop.put("viewMode_links_" + c + "_url", entry.getKey().toNormalform(true, false));
prop.putHTML("viewMode_links_" + c + "_attr", ""); prop.putHTML("viewMode_links_" + c + "_attr", "");
dark = !dark; dark = !dark;
c++; c++;

6
htroot/yacysearch.java
Unescape View File

@ -241,9 +241,9 @@ public class yacysearch {
HashMap<String, String> map = new HashMap<String, String>(); HashMap<String, String> map = new HashMap<String, String>();
map.put("url", comp.url().toNormalform(false, true).replace(',', '|')); map.put("url", comp.url().toNormalform(false, true).replace(',', '|'));
map.put("title", comp.title().replace(',', ' ')); map.put("title", comp.title().replace(',', ' '));
map.put("description", ((document == null) ? comp.title() : document.getTitle()).replace(',', ' ')); map.put("description", ((document == null) ? comp.title() : document.dc_title()).replace(',', ' '));
map.put("author", ((document == null) ? "" : document.getAuthor())); map.put("author", ((document == null) ? "" : document.dc_creator()));
map.put("tags", ((document == null) ? "" : document.getKeywords(' '))); map.put("tags", ((document == null) ? "" : document.dc_subject(' ')));
yacyCore.newsPool.publishMyNews(yacyNewsRecord.newRecord(yacyNewsPool.CATEGORY_SURFTIPP_ADD, map)); yacyCore.newsPool.publishMyNews(yacyNewsRecord.newRecord(yacyNewsPool.CATEGORY_SURFTIPP_ADD, map));
document.close(); document.close();
} }

10
htroot/yacysearchitem.java
Unescape View File

@ -230,13 +230,11 @@ public class yacysearchitem {
ArrayList<plasmaSnippetCache.MediaSnippet> images = result.mediaSnippets(); ArrayList<plasmaSnippetCache.MediaSnippet> images = result.mediaSnippets();
if (images != null) { if (images != null) {
plasmaSnippetCache.MediaSnippet ms; plasmaSnippetCache.MediaSnippet ms;
yacyURL url;
int c = 0; int c = 0;
for (int i = 0; i < images.size(); i++) { for (int i = 0; i < images.size(); i++) {
ms = (plasmaSnippetCache.MediaSnippet) images.get(i); ms = (plasmaSnippetCache.MediaSnippet) images.get(i);
try {url = new yacyURL(ms.href, null);} catch (MalformedURLException e) {continue;} prop.putHTML("content_items_" + i + "_href", ms.href.toNormalform(true, false));
prop.putHTML("content_items_" + i + "_href", ms.href); prop.put("content_items_" + i + "_code", sb.licensedURLs.aquireLicense(ms.href));
prop.put("content_items_" + i + "_code", sb.licensedURLs.aquireLicense(url));
prop.putHTML("content_items_" + i + "_name", shorten(ms.name, namelength)); prop.putHTML("content_items_" + i + "_name", shorten(ms.name, namelength));
prop.put("content_items_" + i + "_attr", ms.attr); // attributes, here: original size of image prop.put("content_items_" + i + "_attr", ms.attr); // attributes, here: original size of image
c++; c++;
@ -260,8 +258,8 @@ public class yacysearchitem {
int c = 0; int c = 0;
for (int i = 0; i < media.size(); i++) { for (int i = 0; i < media.size(); i++) {
ms = (plasmaSnippetCache.MediaSnippet) media.get(i); ms = (plasmaSnippetCache.MediaSnippet) media.get(i);
prop.putHTML("content_items_" + i + "_href", ms.href); prop.putHTML("content_items_" + i + "_href", ms.href.toNormalform(true, false));
prop.putHTML("content_items_" + i + "_hrefshort", nxTools.shortenURLString(ms.href, urllength)); prop.putHTML("content_items_" + i + "_hrefshort", nxTools.shortenURLString(ms.href.toNormalform(true, false), urllength));
prop.putHTML("content_items_" + i + "_name", shorten(ms.name, namelength)); prop.putHTML("content_items_" + i + "_name", shorten(ms.name, namelength));
prop.put("content_items_" + i + "_col", (col) ? "0" : "1"); prop.put("content_items_" + i + "_col", (col) ? "0" : "1");
c++; c++;

52
source/de/anomic/htmlFilter/htmlFilterContentScraper.java
Unescape View File

@ -100,12 +100,12 @@ public class htmlFilterContentScraper extends htmlFilterAbstractScraper implemen
} }
// class variables: collectors for links // class variables: collectors for links
private HashMap<String, String> anchors; private HashMap<yacyURL, String> anchors;
private TreeSet<htmlFilterImageEntry> images; // String(absolute url)/ImageEntry relation private TreeSet<htmlFilterImageEntry> images; // String(absolute url)/ImageEntry relation
private HashMap<String, String> metas; private HashMap<String, String> metas;
private String title; private String title;
//private String headline; //private String headline;
private List[] headlines; private List<String>[] headlines;
private serverCharBuffer content; private serverCharBuffer content;
private EventListenerList htmlFilterEventListeners = new EventListenerList(); private EventListenerList htmlFilterEventListeners = new EventListenerList();
@ -119,12 +119,13 @@ public class htmlFilterContentScraper extends htmlFilterAbstractScraper implemen
*/ */
private yacyURL root; private yacyURL root;
@SuppressWarnings("unchecked")
public htmlFilterContentScraper(yacyURL root) { public htmlFilterContentScraper(yacyURL root) {
// the root value here will not be used to load the resource. // the root value here will not be used to load the resource.
// it is only the reference for relative links // it is only the reference for relative links
super(linkTags0, linkTags1); super(linkTags0, linkTags1);
this.root = root; this.root = root;
this.anchors = new HashMap<String, String>(); this.anchors = new HashMap<yacyURL, String>();
this.images = new TreeSet<htmlFilterImageEntry>(); this.images = new TreeSet<htmlFilterImageEntry>();
this.metas = new HashMap<String, String>(); this.metas = new HashMap<String, String>();
this.title = ""; this.title = "";
@ -159,11 +160,11 @@ public class htmlFilterContentScraper extends htmlFilterAbstractScraper implemen
return normalizedURL.toLowerCase().split(splitrex); // word components of the url return normalizedURL.toLowerCase().split(splitrex); // word components of the url
} }
private String absolutePath(String relativePath) { private yacyURL absolutePath(String relativePath) {
try { try {
return yacyURL.newURL(root, relativePath).toNormalform(false, true); return yacyURL.newURL(root, relativePath);
} catch (Exception e) { } catch (Exception e) {
return ""; return null;
} }
} }
@ -174,11 +175,9 @@ public class htmlFilterContentScraper extends htmlFilterAbstractScraper implemen
width = Integer.parseInt(tagopts.getProperty("width", "-1")); width = Integer.parseInt(tagopts.getProperty("width", "-1"));
height = Integer.parseInt(tagopts.getProperty("height", "-1")); height = Integer.parseInt(tagopts.getProperty("height", "-1"));
} catch (NumberFormatException e) {} } catch (NumberFormatException e) {}
try { yacyURL url = absolutePath(tagopts.getProperty("src", ""));
yacyURL url = new yacyURL(absolutePath(tagopts.getProperty("src", "")), null);
htmlFilterImageEntry ie = new htmlFilterImageEntry(url, tagopts.getProperty("alt",""), width, height); htmlFilterImageEntry ie = new htmlFilterImageEntry(url, tagopts.getProperty("alt",""), width, height);
images.add(ie); images.add(ie);
} catch (MalformedURLException e) {}
} }
if (tagname.equalsIgnoreCase("base")) try { if (tagname.equalsIgnoreCase("base")) try {
root = new yacyURL(tagopts.getProperty("href", ""), null); root = new yacyURL(tagopts.getProperty("href", ""), null);
@ -204,10 +203,7 @@ public class htmlFilterContentScraper extends htmlFilterAbstractScraper implemen
if (href.length() > 0) anchors.put(absolutePath(href), areatitle); if (href.length() > 0) anchors.put(absolutePath(href), areatitle);
} }
if (tagname.equalsIgnoreCase("link")) { if (tagname.equalsIgnoreCase("link")) {
yacyURL newLink = null; yacyURL newLink = absolutePath(tagopts.getProperty("href", ""));
try {
newLink = new yacyURL(absolutePath(tagopts.getProperty("href", "")), null);
} catch (MalformedURLException e) {}
if (newLink != null) { if (newLink != null) {
String type = tagopts.getProperty("rel", ""); String type = tagopts.getProperty("rel", "");
@ -218,7 +214,7 @@ public class htmlFilterContentScraper extends htmlFilterAbstractScraper implemen
images.add(ie); images.add(ie);
this.favicon = newLink; this.favicon = newLink;
} else if (!type.equalsIgnoreCase("stylesheet") && !type.equalsIgnoreCase("alternate stylesheet")) { } else if (!type.equalsIgnoreCase("stylesheet") && !type.equalsIgnoreCase("alternate stylesheet")) {
anchors.put(newLink.toString(), linktitle); anchors.put(newLink, linktitle);
} }
} }
} }
@ -346,7 +342,7 @@ public class htmlFilterContentScraper extends htmlFilterAbstractScraper implemen
} }
} }
public Map<String, String> getAnchors() { public Map<yacyURL, String> getAnchors() {
// returns a url (String) / name (String) relation // returns a url (String) / name (String) relation
return anchors; return anchors;
} }
@ -367,31 +363,44 @@ public class htmlFilterContentScraper extends htmlFilterAbstractScraper implemen
return this.favicon; return this.favicon;
} }
/*
DC in html example:
<meta name="DC.title" lang="en" content="Expressing Dublin Core in HTML/XHTML meta and link elements" />
<meta name="DC.creator" content="Andy Powell, UKOLN, University of Bath" />
<meta name="DC.identifier" scheme="DCTERMS.URI" content="http://dublincore.org/documents/dcq-html/" />
<meta name="DC.format" scheme="DCTERMS.IMT" content="text/html" />
<meta name="DC.type" scheme="DCTERMS.DCMIType" content="Text" />
*/
public String getDescription() { public String getDescription() {
String s = (String) metas.get("description"); String s = metas.get("description");
if (s == null) s = metas.get("DC.description");
if (s == null) return ""; else return s; if (s == null) return ""; else return s;
} }
public String getContentType() { public String getContentType() {
String s = (String) metas.get("content-type"); String s = metas.get("content-type");
if (s == null) return ""; else return s; if (s == null) return ""; else return s;
} }
public String getAuthor() { public String getAuthor() {
String s = (String) metas.get("author"); String s = metas.get("author");
if (s == null) s = (String) metas.get("copyright"); if (s == null) s = metas.get("copyright");
if (s == null) s = metas.get("DC.creator");
if (s == null) return ""; if (s == null) return "";
return s; return s;
} }
public String[] getContentLanguages() { public String[] getContentLanguages() {
String s = (String) metas.get("content-language"); String s = metas.get("content-language");
if (s == null) s = metas.get("DC.language");
if (s == null) s = ""; if (s == null) s = "";
return s.split(" |,"); return s.split(" |,");
} }
public String[] getKeywords() { public String[] getKeywords() {
String s = (String) metas.get("keywords"); String s = metas.get("keywords");
if (s == null) s = metas.get("DC.description");
if (s == null) s = ""; if (s == null) s = "";
if (s.length() == 0) { if (s.length() == 0) {
return getTitle().toLowerCase().split(splitrex); return getTitle().toLowerCase().split(splitrex);
@ -500,3 +509,4 @@ public class htmlFilterContentScraper extends htmlFilterAbstractScraper implemen
return scraper; return scraper;
} }
} }

2
source/de/anomic/plasma/parser/Parser.java
Unescape View File

@ -110,7 +110,7 @@ public interface Parser {
* @return a {@link Hashtable} containing a list of MimeTypes that are supported by * @return a {@link Hashtable} containing a list of MimeTypes that are supported by
* the parser * the parser
*/ */
public Hashtable getSupportedMimeTypes(); public Hashtable<String, String> getSupportedMimeTypes();
/** /**
* This function should be called before reusing the parser object. * This function should be called before reusing the parser object.

4
source/de/anomic/plasma/parser/odt/odtParser.java
Unescape View File

@ -91,7 +91,7 @@ public class odtParser extends AbstractParser implements Parser {
this.parserName = "OASIS OpenDocument V2 Text Document Parser"; this.parserName = "OASIS OpenDocument V2 Text Document Parser";
} }
public Hashtable getSupportedMimeTypes() { public Hashtable<String, String> getSupportedMimeTypes() {
return SUPPORTED_MIME_TYPES; return SUPPORTED_MIME_TYPES;
} }
@ -108,7 +108,7 @@ public class odtParser extends AbstractParser implements Parser {
// opening the file as zip file // opening the file as zip file
ZipFile zipFile= new ZipFile(dest); ZipFile zipFile= new ZipFile(dest);
Enumeration zipEnum = zipFile.entries(); Enumeration<? extends ZipEntry> zipEnum = zipFile.entries();
// looping through all containing files // looping through all containing files
while (zipEnum.hasMoreElements()) { while (zipEnum.hasMoreElements()) {

2
source/de/anomic/plasma/parser/pdf/pdfParser.java
Unescape View File

@ -86,7 +86,7 @@ public class pdfParser extends AbstractParser implements Parser {
this.parserName = "Acrobat Portable Document Parser"; this.parserName = "Acrobat Portable Document Parser";
} }
public Hashtable getSupportedMimeTypes() { public Hashtable<String, String> getSupportedMimeTypes() {
return SUPPORTED_MIME_TYPES; return SUPPORTED_MIME_TYPES;
} }

2
source/de/anomic/plasma/parser/ppt/pptParser.java
Unescape View File

@ -132,7 +132,7 @@ public class pptParser extends AbstractParser implements Parser {
} }
} }
public Hashtable getSupportedMimeTypes() { public Hashtable<String, String> getSupportedMimeTypes() {
return SUPPORTED_MIME_TYPES; return SUPPORTED_MIME_TYPES;
} }

2
source/de/anomic/plasma/parser/ps/psParser.java
Unescape View File

@ -91,7 +91,7 @@ public class psParser extends AbstractParser implements Parser {
} }
} }
public Hashtable getSupportedMimeTypes() { public Hashtable<String, String> getSupportedMimeTypes() {
return SUPPORTED_MIME_TYPES; return SUPPORTED_MIME_TYPES;
} }

6
source/de/anomic/plasma/parser/rpm/rpmParser.java
Unescape View File

@ -88,7 +88,7 @@ public class rpmParser extends AbstractParser implements Parser {
this.parserName = "rpm Parser"; this.parserName = "rpm Parser";
} }
public Hashtable getSupportedMimeTypes() { public Hashtable<String, String> getSupportedMimeTypes() {
return SUPPORTED_MIME_TYPES; return SUPPORTED_MIME_TYPES;
} }
@ -110,7 +110,7 @@ public class rpmParser extends AbstractParser implements Parser {
RPMFile rpmFile = null; RPMFile rpmFile = null;
try { try {
String summary = null, description = null, packager = null, name = sourceFile.getName(); String summary = null, description = null, packager = null, name = sourceFile.getName();
HashMap<String, String> anchors = new HashMap<String, String>(); HashMap<yacyURL, String> anchors = new HashMap<yacyURL, String>();
StringBuffer content = new StringBuffer(); StringBuffer content = new StringBuffer();
// opening the rpm file // opening the rpm file
@ -138,7 +138,7 @@ public class rpmParser extends AbstractParser implements Parser {
else if (headerNames[i].equalsIgnoreCase("SUMMARY")) summary = tag.toString(); else if (headerNames[i].equalsIgnoreCase("SUMMARY")) summary = tag.toString();
else if (headerNames[i].equalsIgnoreCase("DESCRIPTION")) description = tag.toString(); else if (headerNames[i].equalsIgnoreCase("DESCRIPTION")) description = tag.toString();
else if (headerNames[i].equalsIgnoreCase("PACKAGER")) packager = tag.toString(); else if (headerNames[i].equalsIgnoreCase("PACKAGER")) packager = tag.toString();
else if (headerNames[i].equalsIgnoreCase("URL")) anchors.put(tag.toString(), tag.toString()); else if (headerNames[i].equalsIgnoreCase("URL")) anchors.put(new yacyURL(tag.toString(), null), tag.toString());
} }
// closing the rpm file // closing the rpm file

8
source/de/anomic/plasma/parser/rss/rssParser.java
Unescape View File

@ -96,7 +96,7 @@ public class rssParser extends AbstractParser implements Parser {
try { try {
LinkedList<String> feedSections = new LinkedList<String>(); LinkedList<String> feedSections = new LinkedList<String>();
HashMap<String, String> anchors = new HashMap<String, String>(); HashMap<yacyURL, String> anchors = new HashMap<yacyURL, String>();
TreeSet<htmlFilterImageEntry> images = new TreeSet<htmlFilterImageEntry>(); TreeSet<htmlFilterImageEntry> images = new TreeSet<htmlFilterImageEntry>();
serverByteBuffer text = new serverByteBuffer(); serverByteBuffer text = new serverByteBuffer();
serverCharBuffer authors = new serverCharBuffer(); serverCharBuffer authors = new serverCharBuffer();
@ -132,7 +132,7 @@ public class rssParser extends AbstractParser implements Parser {
if (itemCreator != null && itemCreator.length() > 0) authors.append(",").append(itemCreator); if (itemCreator != null && itemCreator.length() > 0) authors.append(",").append(itemCreator);
feedSections.add(itemTitle); feedSections.add(itemTitle);
anchors.put(itemURL.toString(),itemTitle); anchors.put(itemURL, itemTitle);
if ((text.length() != 0) && (text.byteAt(text.length() - 1) != 32)) text.append((byte) 32); if ((text.length() != 0) && (text.byteAt(text.length() - 1) != 32)) text.append((byte) 32);
text.append(new serverCharBuffer(htmlFilterAbstractScraper.stripAll(new serverCharBuffer(itemDescr.toCharArray()))).trim().toString()).append(' '); text.append(new serverCharBuffer(htmlFilterAbstractScraper.stripAll(new serverCharBuffer(itemDescr.toCharArray()))).trim().toString()).append(' ');
@ -149,7 +149,7 @@ public class rssParser extends AbstractParser implements Parser {
feedSections.add(itemHeadline); feedSections.add(itemHeadline);
} }
Map<String, String> itemLinks = scraper.getAnchors(); Map<yacyURL, String> itemLinks = scraper.getAnchors();
if ((itemLinks != null) && (itemLinks.size() > 0)) { if ((itemLinks != null) && (itemLinks.size() > 0)) {
anchors.putAll(itemLinks); anchors.putAll(itemLinks);
} }
@ -191,7 +191,7 @@ public class rssParser extends AbstractParser implements Parser {
} }
} }
public Hashtable getSupportedMimeTypes() { public Hashtable<String, String> getSupportedMimeTypes() {
return SUPPORTED_MIME_TYPES; return SUPPORTED_MIME_TYPES;
} }

2
source/de/anomic/plasma/parser/rtf/rtfParser.java
Unescape View File

@ -116,7 +116,7 @@ public class rtfParser extends AbstractParser implements Parser {
} }
} }
public java.util.Hashtable getSupportedMimeTypes() { public Hashtable<String, String> getSupportedMimeTypes() {
return rtfParser.SUPPORTED_MIME_TYPES; return rtfParser.SUPPORTED_MIME_TYPES;
} }

29
source/de/anomic/plasma/parser/sevenzip/SZParserExtractCallback.java
Unescape View File

@ -48,6 +48,9 @@ import java.util.HashMap;
import java.util.Iterator; import java.util.Iterator;
import java.util.Map; import java.util.Map;
import SevenZip.ArchiveExtractCallback;
import SevenZip.Archive.IInArchive;
import SevenZip.Archive.SevenZipEntry;
import de.anomic.plasma.plasmaParser; import de.anomic.plasma.plasmaParser;
import de.anomic.plasma.plasmaParserDocument; import de.anomic.plasma.plasmaParserDocument;
import de.anomic.plasma.parser.AbstractParser; import de.anomic.plasma.parser.AbstractParser;
@ -56,10 +59,6 @@ import de.anomic.server.serverCachedFileOutputStream;
import de.anomic.server.logging.serverLog; import de.anomic.server.logging.serverLog;
import de.anomic.yacy.yacyURL; import de.anomic.yacy.yacyURL;
import SevenZip.ArchiveExtractCallback;
import SevenZip.Archive.IInArchive;
import SevenZip.Archive.SevenZipEntry;
// wrapper class to redirect output of standard ArchiveExtractCallback to serverLog // wrapper class to redirect output of standard ArchiveExtractCallback to serverLog
// and parse the extracted content // and parse the extracted content
public class SZParserExtractCallback extends ArchiveExtractCallback { public class SZParserExtractCallback extends ArchiveExtractCallback {
@ -117,7 +116,7 @@ public class SZParserExtractCallback extends ArchiveExtractCallback {
plasmaParserDocument theDoc; plasmaParserDocument theDoc;
// workaround for relative links in file, normally '#' shall be used behind the location, see // workaround for relative links in file, normally '#' shall be used behind the location, see
// below for reversion of the effects // below for reversion of the effects
yacyURL url = yacyURL.newURL(doc.getLocation(), this.prefix + "/" + super.filePath); yacyURL url = yacyURL.newURL(doc.dc_source(), this.prefix + "/" + super.filePath);
String mime = plasmaParser.getMimeTypeByFileExt(super.filePath.substring(super.filePath.lastIndexOf('.') + 1)); String mime = plasmaParser.getMimeTypeByFileExt(super.filePath.substring(super.filePath.lastIndexOf('.') + 1));
if (this.cfos.isFallback()) { if (this.cfos.isFallback()) {
theDoc = this.parser.parseSource(url, mime, null, this.cfos.getContentFile()); theDoc = this.parser.parseSource(url, mime, null, this.cfos.getContentFile());
@ -126,18 +125,20 @@ public class SZParserExtractCallback extends ArchiveExtractCallback {
} }
// revert the above workaround // revert the above workaround
Map<String, String> nanchors = new HashMap<String, String>(theDoc.getAnchors().size(), 1f); Map<yacyURL, String> nanchors = new HashMap<yacyURL, String>(theDoc.getAnchors().size(), 1f);
Iterator it = theDoc.getAnchors().entrySet().iterator(); Iterator<Map.Entry<yacyURL, String>> it = theDoc.getAnchors().entrySet().iterator();
Map.Entry entry; Map.Entry<yacyURL, String> entry;
String base = doc.getLocation().toNormalform(false, true); String base = doc.dc_source().toNormalform(false, true);
String u;
while (it.hasNext()) { while (it.hasNext()) {
entry = (Map.Entry)it.next(); entry = it.next();
if (((String)entry.getKey()).startsWith(base + "/")) { u = entry.getKey().toNormalform(true, true);
String ref = "#" + ((String)entry.getKey()).substring(base.length() + 1); if (u.startsWith(base + "/")) {
String ref = "#" + u.substring(base.length() + 1);
this.log.logFinest("changing " + entry.getKey() + " to use reference " + ref); this.log.logFinest("changing " + entry.getKey() + " to use reference " + ref);
nanchors.put(base + ref, (String)entry.getValue()); nanchors.put(new yacyURL(base + ref, null), entry.getValue());
} else { } else {
nanchors.put((String)entry.getKey(), (String)entry.getValue()); nanchors.put(entry.getKey(), entry.getValue());
} }
} }
theDoc.getAnchors().clear(); theDoc.getAnchors().clear();

3
source/de/anomic/plasma/parser/sevenzip/sevenzipParser.java
Unescape View File

@ -50,7 +50,6 @@ import java.util.Hashtable;
import SevenZip.IInStream; import SevenZip.IInStream;
import SevenZip.MyRandomAccessFile; import SevenZip.MyRandomAccessFile;
import SevenZip.Archive.SevenZip.Handler; import SevenZip.Archive.SevenZip.Handler;
import de.anomic.plasma.plasmaParserDocument; import de.anomic.plasma.plasmaParserDocument;
import de.anomic.plasma.parser.AbstractParser; import de.anomic.plasma.parser.AbstractParser;
import de.anomic.plasma.parser.Parser; import de.anomic.plasma.parser.Parser;
@ -140,7 +139,7 @@ public class sevenzipParser extends AbstractParser implements Parser {
} }
} }
public Hashtable getSupportedMimeTypes() { public Hashtable<String, String> getSupportedMimeTypes() {
return SUPPORTED_MIME_TYPES; return SUPPORTED_MIME_TYPES;
} }
} }

11
source/de/anomic/plasma/parser/swf/swfParser.java
Unescape View File

@ -44,11 +44,10 @@
package de.anomic.plasma.parser.swf; package de.anomic.plasma.parser.swf;
import java.io.InputStream; import java.io.InputStream;
import java.util.Hashtable;
import java.util.HashMap; import java.util.HashMap;
import java.util.Hashtable;
import pt.tumba.parser.swf.*; import pt.tumba.parser.swf.SWF2HTML;
import de.anomic.plasma.plasmaParserDocument; import de.anomic.plasma.plasmaParserDocument;
import de.anomic.plasma.parser.AbstractParser; import de.anomic.plasma.parser.AbstractParser;
import de.anomic.plasma.parser.Parser; import de.anomic.plasma.parser.Parser;
@ -82,7 +81,7 @@ public class swfParser extends AbstractParser implements Parser {
/** /**
* returns a hashtable containing the mimetypes that are supported by this class * returns a hashtable containing the mimetypes that are supported by this class
*/ */
public Hashtable getSupportedMimeTypes() { public Hashtable<String, String> getSupportedMimeTypes() {
return SUPPORTED_MIME_TYPES; return SUPPORTED_MIME_TYPES;
} }
@ -101,7 +100,7 @@ public class swfParser extends AbstractParser implements Parser {
String[] sections = null; String[] sections = null;
String abstrct = null; String abstrct = null;
//TreeSet images = null; //TreeSet images = null;
HashMap<String, String> anchors = new HashMap<String, String>(); HashMap<yacyURL, String> anchors = new HashMap<yacyURL, String>();
int urls = 0; int urls = 0;
int urlStart = -1; int urlStart = -1;
int urlEnd = 0; int urlEnd = 0;
@ -118,7 +117,7 @@ public class swfParser extends AbstractParser implements Parser {
urlEnd = contents.indexOf(linebreak,urlStart); urlEnd = contents.indexOf(linebreak,urlStart);
url = contents.substring(urlStart,urlEnd); url = contents.substring(urlStart,urlEnd);
urlnr = (new Integer(++urls)).toString(); urlnr = (new Integer(++urls)).toString();
anchors.put(url,urlnr); anchors.put(new yacyURL(url, null), urlnr);
contents = contents.substring(0,urlStart)+contents.substring(urlEnd); contents = contents.substring(0,urlStart)+contents.substring(urlEnd);
} }

10
source/de/anomic/plasma/parser/tar/tarParser.java
Unescape View File

@ -94,7 +94,7 @@ public class tarParser extends AbstractParser implements Parser {
this.parserName = "Tape Archive File Parser"; this.parserName = "Tape Archive File Parser";
} }
public Hashtable getSupportedMimeTypes() { public Hashtable<String, String> getSupportedMimeTypes() {
return SUPPORTED_MIME_TYPES; return SUPPORTED_MIME_TYPES;
} }
@ -131,7 +131,7 @@ public class tarParser extends AbstractParser implements Parser {
LinkedList<String> docSections = new LinkedList<String>(); LinkedList<String> docSections = new LinkedList<String>();
StringBuffer docAbstrct = new StringBuffer(); StringBuffer docAbstrct = new StringBuffer();
Map<String, String> docAnchors = new HashMap<String, String>(); Map<yacyURL, String> docAnchors = new HashMap<yacyURL, String>();
TreeSet<htmlFilterImageEntry> docImages = new TreeSet<htmlFilterImageEntry>(); TreeSet<htmlFilterImageEntry> docImages = new TreeSet<htmlFilterImageEntry>();
// looping through the contained files // looping through the contained files
@ -177,15 +177,15 @@ public class tarParser extends AbstractParser implements Parser {
// merging all documents together // merging all documents together
if (docKeywords.length() > 0) docKeywords.append(","); if (docKeywords.length() > 0) docKeywords.append(",");
docKeywords.append(subDoc.getKeywords(',')); docKeywords.append(subDoc.dc_subject(','));
if (docLongTitle.length() > 0) docLongTitle.append("\n"); if (docLongTitle.length() > 0) docLongTitle.append("\n");
docLongTitle.append(subDoc.getTitle()); docLongTitle.append(subDoc.dc_title());
docSections.addAll(Arrays.asList(subDoc.getSectionTitles())); docSections.addAll(Arrays.asList(subDoc.getSectionTitles()));
if (docAbstrct.length() > 0) docAbstrct.append("\n"); if (docAbstrct.length() > 0) docAbstrct.append("\n");
docAbstrct.append(subDoc.getAbstract()); docAbstrct.append(subDoc.dc_description());
if (subDoc.getTextLength() > 0) { if (subDoc.getTextLength() > 0) {
if (docTextLength > 0) docText.write('\n'); if (docTextLength > 0) docText.write('\n');

4
source/de/anomic/plasma/parser/vcf/vcfParser.java
Unescape View File

@ -101,7 +101,7 @@ public class vcfParser extends AbstractParser implements Parser {
StringBuffer parsedTitle = new StringBuffer(); StringBuffer parsedTitle = new StringBuffer();
StringBuffer parsedDataText = new StringBuffer(); StringBuffer parsedDataText = new StringBuffer();
HashMap<String, String> parsedData = new HashMap<String, String>(); HashMap<String, String> parsedData = new HashMap<String, String>();
HashMap<String, String> anchors = new HashMap<String, String>(); HashMap<yacyURL, String> anchors = new HashMap<yacyURL, String>();
LinkedList<String> parsedNames = new LinkedList<String>(); LinkedList<String> parsedNames = new LinkedList<String>();
boolean useLastLine = false; boolean useLastLine = false;
@ -211,7 +211,7 @@ public class vcfParser extends AbstractParser implements Parser {
} else if (key.toUpperCase().startsWith("URL")) { } else if (key.toUpperCase().startsWith("URL")) {
try { try {
yacyURL newURL = new yacyURL(value, null); yacyURL newURL = new yacyURL(value, null);
anchors.put(newURL.toString(),newURL.toString()); anchors.put(newURL, newURL.toString());
//parsedData.put(key,value); //parsedData.put(key,value);
} catch (MalformedURLException ex) {/* ignore this */} } catch (MalformedURLException ex) {/* ignore this */}
} else if ( } else if (

2
source/de/anomic/plasma/parser/xls/xlsParser.java
Unescape View File

@ -162,7 +162,7 @@ public class xlsParser extends AbstractParser implements Parser, HSSFListener {
} }
} }
public Hashtable getSupportedMimeTypes() { public Hashtable<String, String> getSupportedMimeTypes() {
return SUPPORTED_MIME_TYPES; return SUPPORTED_MIME_TYPES;
} }

10
source/de/anomic/plasma/parser/zip/zipParser.java
Unescape View File

@ -43,7 +43,6 @@
package de.anomic.plasma.parser.zip; package de.anomic.plasma.parser.zip;
import de.anomic.htmlFilter.htmlFilterImageEntry;
import java.io.BufferedOutputStream; import java.io.BufferedOutputStream;
import java.io.File; import java.io.File;
import java.io.FileOutputStream; import java.io.FileOutputStream;
@ -58,6 +57,7 @@ import java.util.TreeSet;
import java.util.zip.ZipEntry; import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream; import java.util.zip.ZipInputStream;
import de.anomic.htmlFilter.htmlFilterImageEntry;
import de.anomic.plasma.plasmaParser; import de.anomic.plasma.plasmaParser;
import de.anomic.plasma.plasmaParserDocument; import de.anomic.plasma.plasmaParserDocument;
import de.anomic.plasma.parser.AbstractParser; import de.anomic.plasma.parser.AbstractParser;
@ -114,7 +114,7 @@ public class zipParser extends AbstractParser implements Parser {
StringBuffer docLongTitle = new StringBuffer(); StringBuffer docLongTitle = new StringBuffer();
LinkedList<String> docSections = new LinkedList<String>(); LinkedList<String> docSections = new LinkedList<String>();
StringBuffer docAbstrct = new StringBuffer(); StringBuffer docAbstrct = new StringBuffer();
Map<String, String> docAnchors = new HashMap<String, String>(); Map<yacyURL, String> docAnchors = new HashMap<yacyURL, String>();
TreeSet<htmlFilterImageEntry> docImages = new TreeSet<htmlFilterImageEntry>(); TreeSet<htmlFilterImageEntry> docImages = new TreeSet<htmlFilterImageEntry>();
// creating a new parser class to parse the unzipped content // creating a new parser class to parse the unzipped content
@ -160,15 +160,15 @@ public class zipParser extends AbstractParser implements Parser {
// merging all documents together // merging all documents together
if (docKeywords.length() > 0) docKeywords.append(","); if (docKeywords.length() > 0) docKeywords.append(",");
docKeywords.append(subDoc.getKeywords(',')); docKeywords.append(subDoc.dc_subject(','));
if (docLongTitle.length() > 0) docLongTitle.append("\n"); if (docLongTitle.length() > 0) docLongTitle.append("\n");
docLongTitle.append(subDoc.getTitle()); docLongTitle.append(subDoc.dc_title());
docSections.addAll(Arrays.asList(subDoc.getSectionTitles())); docSections.addAll(Arrays.asList(subDoc.getSectionTitles()));
if (docAbstrct.length() > 0) docAbstrct.append("\n"); if (docAbstrct.length() > 0) docAbstrct.append("\n");
docAbstrct.append(subDoc.getAbstract()); docAbstrct.append(subDoc.dc_description());
if (subDoc.getTextLength() > 0) { if (subDoc.getTextLength() > 0) {
if (docTextLength > 0) docText.write('\n'); if (docTextLength > 0) docText.write('\n');

117
source/de/anomic/plasma/plasmaCondenser.java
Unescape View File

@ -73,6 +73,7 @@ import de.anomic.kelondro.kelondroBitfield;
import de.anomic.kelondro.kelondroMSetTools; import de.anomic.kelondro.kelondroMSetTools;
import de.anomic.server.serverCodings; import de.anomic.server.serverCodings;
import de.anomic.yacy.yacySeedDB; import de.anomic.yacy.yacySeedDB;
import de.anomic.yacy.yacyURL;
public final class plasmaCondenser { public final class plasmaCondenser {
@ -130,9 +131,9 @@ public final class plasmaCondenser {
//System.out.println("DEBUG: condensing " + document.getMainLongTitle() + ", indexText=" + Boolean.toString(indexText) + ", indexMedia=" + Boolean.toString(indexMedia)); //System.out.println("DEBUG: condensing " + document.getMainLongTitle() + ", indexText=" + Boolean.toString(indexText) + ", indexMedia=" + Boolean.toString(indexMedia));
insertTextToWords(document.getLocation().toNormalform(false, true), 0, indexRWIEntry.flag_app_url, RESULT_FLAGS); insertTextToWords(document.dc_source().toNormalform(false, true), 0, indexRWIEntry.flag_app_url, RESULT_FLAGS);
Map.Entry entry; Map.Entry<yacyURL, String> entry;
if (indexText) { if (indexText) {
createCondensement(document.getText(), document.getCharset()); createCondensement(document.getText(), document.getCharset());
// the phrase counter: // the phrase counter:
@ -147,9 +148,9 @@ public final class plasmaCondenser {
// phrase 99 is taken from the media Link url and anchor description // phrase 99 is taken from the media Link url and anchor description
// phrase 100 and above are lines from the text // phrase 100 and above are lines from the text
insertTextToWords(document.getTitle(), 1, indexRWIEntry.flag_app_descr, RESULT_FLAGS); insertTextToWords(document.dc_title(), 1, indexRWIEntry.flag_app_descr, RESULT_FLAGS);
insertTextToWords(document.getAbstract(), 3, indexRWIEntry.flag_app_descr, RESULT_FLAGS); insertTextToWords(document.dc_description(), 3, indexRWIEntry.flag_app_descr, RESULT_FLAGS);
insertTextToWords(document.getAuthor(), 4, indexRWIEntry.flag_app_descr, RESULT_FLAGS); insertTextToWords(document.dc_creator(), 4, indexRWIEntry.flag_app_descr, RESULT_FLAGS);
// missing: tags! // missing: tags!
String[] titles = document.getSectionTitles(); String[] titles = document.getSectionTitles();
for (int i = 0; i < titles.length; i++) { for (int i = 0; i < titles.length; i++) {
@ -157,10 +158,10 @@ public final class plasmaCondenser {
} }
// anchors // anchors
Iterator i = document.getAnchors().entrySet().iterator(); Iterator<Map.Entry<yacyURL, String>> i = document.getAnchors().entrySet().iterator();
while (i.hasNext()) { while (i.hasNext()) {
entry = (Map.Entry) i.next(); entry = i.next();
insertTextToWords((String) entry.getKey(), 98, indexRWIEntry.flag_app_reference, RESULT_FLAGS); insertTextToWords(entry.getKey().toNormalform(false, false), 98, indexRWIEntry.flag_app_reference, RESULT_FLAGS);
insertTextToWords((String) entry.getValue(), 98, indexRWIEntry.flag_app_reference, RESULT_FLAGS); insertTextToWords((String) entry.getValue(), 98, indexRWIEntry.flag_app_reference, RESULT_FLAGS);
} }
} else { } else {
@ -172,45 +173,45 @@ public final class plasmaCondenser {
if (indexMedia) { if (indexMedia) {
// audio // audio
Iterator i = document.getAudiolinks().entrySet().iterator(); Iterator<Map.Entry<yacyURL, String>> i = document.getAudiolinks().entrySet().iterator();
while (i.hasNext()) { while (i.hasNext()) {
entry = (Map.Entry) i.next(); entry = i.next();
insertTextToWords((String) entry.getKey(), 99, flag_cat_hasaudio, RESULT_FLAGS); insertTextToWords(entry.getKey().toNormalform(false, false), 99, flag_cat_hasaudio, RESULT_FLAGS);
insertTextToWords((String) entry.getValue(), 99, flag_cat_hasaudio, RESULT_FLAGS); insertTextToWords((String) entry.getValue(), 99, flag_cat_hasaudio, RESULT_FLAGS);
} }
// video // video
i = document.getVideolinks().entrySet().iterator(); i = document.getVideolinks().entrySet().iterator();
while (i.hasNext()) { while (i.hasNext()) {
entry = (Map.Entry) i.next(); entry = i.next();
insertTextToWords((String) entry.getKey(), 99, flag_cat_hasvideo, RESULT_FLAGS); insertTextToWords(entry.getKey().toNormalform(false, false), 99, flag_cat_hasvideo, RESULT_FLAGS);
insertTextToWords((String) entry.getValue(), 99, flag_cat_hasvideo, RESULT_FLAGS); insertTextToWords((String) entry.getValue(), 99, flag_cat_hasvideo, RESULT_FLAGS);
} }
// applications // applications
i = document.getApplinks().entrySet().iterator(); i = document.getApplinks().entrySet().iterator();
while (i.hasNext()) { while (i.hasNext()) {
entry = (Map.Entry) i.next(); entry = i.next();
insertTextToWords((String) entry.getKey(), 99, flag_cat_hasapp, RESULT_FLAGS); insertTextToWords(entry.getKey().toNormalform(false, false), 99, flag_cat_hasapp, RESULT_FLAGS);
insertTextToWords((String) entry.getValue(), 99, flag_cat_hasapp, RESULT_FLAGS); insertTextToWords((String) entry.getValue(), 99, flag_cat_hasapp, RESULT_FLAGS);
} }
// images // images
i = document.getImages().iterator(); Iterator<htmlFilterImageEntry> j = document.getImages().iterator();
htmlFilterImageEntry ientry; htmlFilterImageEntry ientry;
while (i.hasNext()) { while (j.hasNext()) {
ientry = (htmlFilterImageEntry) i.next(); ientry = j.next();
insertTextToWords(ientry.url().toNormalform(false, true), 99, flag_cat_hasimage, RESULT_FLAGS); insertTextToWords(ientry.url().toNormalform(false, false), 99, flag_cat_hasimage, RESULT_FLAGS);
insertTextToWords(ientry.alt(), 99, flag_cat_hasimage, RESULT_FLAGS); insertTextToWords(ientry.alt(), 99, flag_cat_hasimage, RESULT_FLAGS);
} }
// finally check all words for missing flag entry // finally check all words for missing flag entry
Iterator<Map.Entry<String, wordStatProp>> j = words.entrySet().iterator(); Iterator<Map.Entry<String, wordStatProp>> k = words.entrySet().iterator();
wordStatProp wprop; wordStatProp wprop;
Map.Entry<String, wordStatProp> we; Map.Entry<String, wordStatProp> we;
while (j.hasNext()) { while (k.hasNext()) {
we = j.next(); we = k.next();
wprop = (wordStatProp) we.getValue(); wprop = we.getValue();
if (wprop.flags == null) { if (wprop.flags == null) {
wprop.flags = (kelondroBitfield) RESULT_FLAGS.clone(); wprop.flags = (kelondroBitfield) RESULT_FLAGS.clone();
words.put(we.getKey(), wprop); words.put(we.getKey(), wprop);
@ -309,7 +310,7 @@ public final class plasmaCondenser {
public int posInText; // unique handle, is initialized with word position (excluding double occurring words) public int posInText; // unique handle, is initialized with word position (excluding double occurring words)
public int posInPhrase; // position of word in phrase public int posInPhrase; // position of word in phrase
public int numOfPhrase; // number of phrase. 'normal' phrases begin with number 100 public int numOfPhrase; // number of phrase. 'normal' phrases begin with number 100
public HashSet hash; // a set of handles to all sentences where this word appears private HashSet<Integer> hash; // a set of handles to all sentences where this word appears
public kelondroBitfield flags; // the flag bits for each word public kelondroBitfield flags; // the flag bits for each word
public wordStatProp(int handle, int pip, int nop) { public wordStatProp(int handle, int pip, int nop) {
@ -317,7 +318,7 @@ public final class plasmaCondenser {
this.posInText = handle; this.posInText = handle;
this.posInPhrase = pip; this.posInPhrase = pip;
this.numOfPhrase = nop; this.numOfPhrase = nop;
this.hash = new HashSet(); this.hash = new HashSet<Integer>();
this.flags = null; this.flags = null;
} }
@ -326,7 +327,7 @@ public final class plasmaCondenser {
} }
public void check(int i) { public void check(int i) {
hash.add(Integer.toString(i)); hash.add(new Integer(i));
} }
} }
@ -336,12 +337,12 @@ public final class plasmaCondenser {
public int count; // number of occurrences public int count; // number of occurrences
public int handle; // unique handle, is initialized with sentence counter public int handle; // unique handle, is initialized with sentence counter
public HashSet hash; // private HashSet<Integer> hash; //
public phraseStatProp(int handle) { public phraseStatProp(int handle) {
this.count = 1; this.count = 1;
this.handle = handle; this.handle = handle;
this.hash = new HashSet(); this.hash = new HashSet<Integer>();
} }
public void inc() { public void inc() {
@ -349,7 +350,7 @@ public final class plasmaCondenser {
} }
public void check(int i) { public void check(int i) {
hash.add(Integer.toString(i)); hash.add(new Integer(i));
} }
} }
@ -362,7 +363,7 @@ public final class plasmaCondenser {
} }
private void createCondensement(InputStream is, String charset) throws UnsupportedEncodingException { private void createCondensement(InputStream is, String charset) throws UnsupportedEncodingException {
HashSet currsentwords = new HashSet(); HashSet<String> currsentwords = new HashSet<String>();
StringBuffer sentence = new StringBuffer(100); StringBuffer sentence = new StringBuffer(100);
String word = ""; String word = "";
String k; String k;
@ -376,7 +377,6 @@ public final class plasmaCondenser {
int allsentencecounter = 0; int allsentencecounter = 0;
int idx; int idx;
int wordInSentenceCounter = 1; int wordInSentenceCounter = 1;
Iterator it, it1;
boolean comb_indexof = false, last_last = false, last_index = false; boolean comb_indexof = false, last_last = false, last_index = false;
RandomAccessFile fa; RandomAccessFile fa;
final boolean dumpWords = false; final boolean dumpWords = false;
@ -405,6 +405,7 @@ public final class plasmaCondenser {
// distinguish punctuation and words // distinguish punctuation and words
wordlen = word.length(); wordlen = word.length();
Iterator<String> it;
if ((wordlen == 1) && (htmlFilterContentScraper.punctuation(word.charAt(0)))) { if ((wordlen == 1) && (htmlFilterContentScraper.punctuation(word.charAt(0)))) {
// store sentence // store sentence
if (sentence.length() > 0) { if (sentence.length() > 0) {
@ -493,9 +494,9 @@ public final class plasmaCondenser {
String[] s; String[] s;
int wc; int wc;
Object o; Object o;
it = sentences.keySet().iterator(); Iterator<StringBuffer> sit = sentences.keySet().iterator();
while (it.hasNext()) { while (sit.hasNext()) {
o = it.next(); o = sit.next();
if (o != null) { if (o != null) {
sentence = (StringBuffer) o; sentence = (StringBuffer) o;
wc = (sentence.length() - 1) / numlength; wc = (sentence.length() - 1) / numlength;
@ -511,15 +512,15 @@ public final class plasmaCondenser {
} }
} }
Map.Entry entry; Map.Entry<String, wordStatProp> entry;
// we search for similar words and reorganize the corresponding sentences // we search for similar words and reorganize the corresponding sentences
// a word is similar, if a shortened version is equal // a word is similar, if a shortened version is equal
it = words.entrySet().iterator(); // enumerates the keys in descending order Iterator<Map.Entry<String, wordStatProp>> wi = words.entrySet().iterator(); // enumerates the keys in descending order
wordsearch: while (it.hasNext()) { wordsearch: while (wi.hasNext()) {
entry = (Map.Entry) it.next(); entry = wi.next();
word = (String) entry.getKey(); word = entry.getKey();
wordlen = word.length(); wordlen = word.length();
wsp = (wordStatProp) entry.getValue(); wsp = entry.getValue();
for (int i = wordcut; i > 0; i--) { for (int i = wordcut; i > 0; i--) {
if (wordlen > i) { if (wordlen > i) {
k = word.substring(0, wordlen - i); k = word.substring(0, wordlen - i);
@ -528,9 +529,9 @@ public final class plasmaCondenser {
// corresponding links // corresponding links
// in sentences that use this word // in sentences that use this word
wsp1 = (wordStatProp) words.get(k); wsp1 = (wordStatProp) words.get(k);
it1 = wsp.hash.iterator(); // we iterate over all sentences that refer to this word Iterator<Integer> it1 = wsp.hash.iterator(); // we iterate over all sentences that refer to this word
while (it1.hasNext()) { while (it1.hasNext()) {
idx = Integer.parseInt((String) it1.next()); // number of a sentence idx = it1.next().intValue(); // number of a sentence
s = (String[]) orderedSentences[idx]; s = (String[]) orderedSentences[idx];
for (int j = 2; j < s.length; j++) { for (int j = 2; j < s.length; j++) {
if (s[j].equals(intString(wsp.posInText, numlength))) if (s[j].equals(intString(wsp.posInText, numlength)))
@ -542,7 +543,7 @@ public final class plasmaCondenser {
wsp1.count = wsp1.count + wsp.count; wsp1.count = wsp1.count + wsp.count;
words.put(k, wsp1); words.put(k, wsp1);
// remove current word // remove current word
it.remove(); wi.remove();
continue wordsearch; continue wordsearch;
} }
} }
@ -550,8 +551,8 @@ public final class plasmaCondenser {
} }
// depending on the orderedSentences structure, we rebuild the sentence // depending on the orderedSentences structure, we rebuild the sentence
// HashMap to eliminate double occuring sentences // HashMap to eliminate double occurring sentences
sentences = new HashMap(); sentences = new HashMap<StringBuffer, phraseStatProp>();
int le; int le;
for (int i = 0; i < orderedSentences.length; i++) { for (int i = 0; i < orderedSentences.length; i++) {
le = ((String[]) orderedSentences[i]).length; le = ((String[]) orderedSentences[i]).length;
@ -560,7 +561,7 @@ public final class plasmaCondenser {
sentence.append(((String[]) orderedSentences[i])[j]); sentence.append(((String[]) orderedSentences[i])[j]);
if (sentences.containsKey(sentence)) { if (sentences.containsKey(sentence)) {
// add sentence counter to counter of found sentence // add sentence counter to counter of found sentence
psp = (phraseStatProp) sentences.get(sentence); psp = sentences.get(sentence);
psp.count = psp.count + Integer.parseInt(((String[]) orderedSentences[i])[0]); psp.count = psp.count + Integer.parseInt(((String[]) orderedSentences[i])[0]);
sentences.put(sentence, psp); sentences.put(sentence, psp);
// System.out.println("Found double occurring sentence " + i + " // System.out.println("Found double occurring sentence " + i + "
@ -596,14 +597,14 @@ public final class plasmaCondenser {
// this structure is only needed to reconstruct the text // this structure is only needed to reconstruct the text
String word; String word;
wordStatProp wsp; wordStatProp wsp;
Map.Entry entry; Map.Entry<String, wordStatProp> entry;
Iterator it; Iterator<Map.Entry<String, wordStatProp>> it;
String[] orderedWords = new String[words.size() + 99]; // uuiiii, the '99' is only a quick hack... String[] orderedWords = new String[words.size() + 99]; // uuiiii, the '99' is only a quick hack...
it = words.entrySet().iterator(); // enumerates the keys in ascending order it = words.entrySet().iterator(); // enumerates the keys in ascending order
while (it.hasNext()) { while (it.hasNext()) {
entry = (Map.Entry) it.next(); entry = it.next();
word = (String) entry.getKey(); word = entry.getKey();
wsp = (wordStatProp) entry.getValue(); wsp = entry.getValue();
orderedWords[wsp.posInText] = word; orderedWords[wsp.posInText] = word;
} }
@ -632,14 +633,14 @@ public final class plasmaCondenser {
// we reconstruct the sentence hashtable again and create by-handle ordered entries // we reconstruct the sentence hashtable again and create by-handle ordered entries
// this structure is needed to present the strings in the right order in a printout // this structure is needed to present the strings in the right order in a printout
int wc; int wc;
Iterator it;
phraseStatProp psp; phraseStatProp psp;
String[] s; String[] s;
StringBuffer sentence; StringBuffer sentence;
Object[] orderedSentences = new Object[sentences.size()]; Object[] orderedSentences = new Object[sentences.size()];
for (int i = 0; i < sentences.size(); i++) for (int i = 0; i < sentences.size(); i++) {
orderedSentences[i] = null; // this array must be initialized orderedSentences[i] = null; // this array must be initialized
it = sentences.keySet().iterator(); }
Iterator<StringBuffer> it = sentences.keySet().iterator();
while (it.hasNext()) { while (it.hasNext()) {
sentence = (StringBuffer) it.next(); sentence = (StringBuffer) it.next();
wc = (sentence.length() - 1) / numlength; wc = (sentence.length() - 1) / numlength;
@ -717,9 +718,9 @@ public final class plasmaCondenser {
} }
private static class unsievedWordsEnum implements Enumeration { private static class unsievedWordsEnum implements Enumeration<StringBuffer> {
// returns an enumeration of StringBuffer Objects // returns an enumeration of StringBuffer Objects
Object buffer = null; StringBuffer buffer = null;
sentencesFromInputStreamEnum e; sentencesFromInputStreamEnum e;
StringBuffer s; StringBuffer s;
@ -770,8 +771,8 @@ public final class plasmaCondenser {
return buffer != null; return buffer != null;
} }
public Object nextElement() { public StringBuffer nextElement() {
Object r = buffer; StringBuffer r = buffer;
buffer = nextElement0(); buffer = nextElement0();
return r; return r;
} }

24
source/de/anomic/plasma/plasmaCrawlProfile.java
Unescape View File

@ -91,18 +91,18 @@ public class plasmaCrawlProfile {
return profileTable.size(); return profileTable.size();
} }
public Iterator profiles(boolean up) { public Iterator<entry> profiles(boolean up) {
// enumerates profile entries // enumerates profile entries
try { try {
return new profileIterator(up); return new profileIterator(up);
} catch (IOException e) { } catch (IOException e) {
return new HashSet().iterator(); return new HashSet<entry>().iterator();
} }
} }
public class profileIterator implements Iterator { public class profileIterator implements Iterator<entry> {
// the iterator iterates all keys, which are byte[] objects // the iterator iterates all keys, which are byte[] objects
kelondroCloneableIterator handleIterator; kelondroCloneableIterator<String> handleIterator;
String lastkey; String lastkey;
public profileIterator(boolean up) throws IOException { public profileIterator(boolean up) throws IOException {
handleIterator = profileTable.keys(up, false); handleIterator = profileTable.keys(up, false);
@ -116,7 +116,7 @@ public class plasmaCrawlProfile {
return false; return false;
} }
} }
public Object next() { public entry next() {
try { try {
lastkey = (String) handleIterator.next(); lastkey = (String) handleIterator.next();
return getEntry(lastkey); return getEntry(lastkey);
@ -140,7 +140,7 @@ public class plasmaCrawlProfile {
} catch (IOException e) {} } catch (IOException e) {}
} }
public entry newEntry(Map mem) { public entry newEntry(Map<String, String> mem) {
entry ne = new entry(mem); entry ne = new entry(mem);
try { try {
profileTable.set(ne.handle(), ne.map()); profileTable.set(ne.handle(), ne.map());
@ -466,19 +466,19 @@ public class plasmaCrawlProfile {
} }
public String domName(boolean attr, int index){ public String domName(boolean attr, int index){
Iterator domnamesi = doms.entrySet().iterator(); Iterator<Map.Entry<String, DomProfile>> domnamesi = doms.entrySet().iterator();
String domname=""; String domname="";
Map.Entry ey; Map.Entry<String, DomProfile> ey;
DomProfile dp; DomProfile dp;
int i = 0; int i = 0;
while ((domnamesi.hasNext()) && (i < index)) { while ((domnamesi.hasNext()) && (i < index)) {
ey = (Map.Entry) domnamesi.next(); ey = domnamesi.next();
i++; i++;
} }
if(domnamesi.hasNext()){ if(domnamesi.hasNext()){
ey = (Map.Entry) domnamesi.next(); ey = domnamesi.next();
dp = (DomProfile) ey.getValue(); dp = ey.getValue();
domname = ((String) ey.getKey()) + ((attr) ? ("/r=" + dp.referrer + ", d=" + dp.depth + ", c=" + dp.count) : " "); domname = ey.getKey() + ((attr) ? ("/r=" + dp.referrer + ", d=" + dp.depth + ", c=" + dp.count) : " ");
} }
return domname; return domname;
} }

119
source/de/anomic/plasma/plasmaParser.java
Unescape View File

@ -432,12 +432,13 @@ public final class plasmaParser {
String fullClassName = plasmaParserPkgName + "." + currentDir.getName() + "." + className; String fullClassName = plasmaParserPkgName + "." + currentDir.getName() + "." + className;
try { try {
// trying to load the parser class by its name // trying to load the parser class by its name
Class parserClass = Class.forName(fullClassName); Class<?> parserClass = Class.forName(fullClassName);
Object theParser = parserClass.newInstance(); Object theParser0 = (Parser) parserClass.newInstance();
if (!(theParser instanceof Parser)) continue; if (!(theParser0 instanceof Parser)) continue;
Parser theParser = (Parser) theParser0;
// testing if all needed libx libraries are available // testing if all needed libx libraries are available
String[] neededLibx = ((Parser)theParser).getLibxDependences(); String[] neededLibx = theParser.getLibxDependences();
StringBuffer neededLibxBuf = new StringBuffer(); StringBuffer neededLibxBuf = new StringBuffer();
if (neededLibx != null) { if (neededLibx != null) {
for (int libxId=0; libxId < neededLibx.length; libxId++) { for (int libxId=0; libxId < neededLibx.length; libxId++) {
@ -451,7 +452,7 @@ public final class plasmaParser {
} }
// loading the list of mime-types that are supported by this parser class // loading the list of mime-types that are supported by this parser class
Hashtable<String, String> supportedMimeTypes = ((Parser) theParser).getSupportedMimeTypes(); Hashtable<String, String> supportedMimeTypes = theParser.getSupportedMimeTypes();
// creating a parser info object // creating a parser info object
ParserInfo parserInfo = new ParserInfo(); ParserInfo parserInfo = new ParserInfo();
@ -462,7 +463,7 @@ public final class plasmaParser {
parserInfo.parserVersionNr = ((Parser)theParser).getVersion(); parserInfo.parserVersionNr = ((Parser)theParser).getVersion();
parserInfo.parserName = ((Parser) theParser).getName(); parserInfo.parserName = ((Parser) theParser).getName();
Iterator mimeTypeIterator = supportedMimeTypes.keySet().iterator(); Iterator<String> mimeTypeIterator = supportedMimeTypes.keySet().iterator();
while (mimeTypeIterator.hasNext()) { while (mimeTypeIterator.hasNext()) {
String mimeType = (String) mimeTypeIterator.next(); String mimeType = (String) mimeTypeIterator.next();
availableParserList.put(mimeType, parserInfo); availableParserList.put(mimeType, parserInfo);
@ -490,9 +491,9 @@ public final class plasmaParser {
public void close() { public void close() {
// clearing the parser list // clearing the parser list
Iterator configs = parserConfigList.values().iterator(); Iterator<plasmaParserConfig> configs = parserConfigList.values().iterator();
while (configs.hasNext()) { while (configs.hasNext()) {
plasmaParserConfig currentConfig = (plasmaParserConfig) configs.next(); plasmaParserConfig currentConfig = configs.next();
synchronized (currentConfig.enabledParserList) { synchronized (currentConfig.enabledParserList) {
currentConfig.enabledParserList.clear(); currentConfig.enabledParserList.clear();
} }
@ -684,12 +685,11 @@ public final class plasmaParser {
} }
public plasmaParserDocument transformScraper(yacyURL location, String mimeType, String charSet, htmlFilterContentScraper scraper) { public plasmaParserDocument transformScraper(yacyURL location, String mimeType, String charSet, htmlFilterContentScraper scraper) {
try {
String[] sections = new String[scraper.getHeadlines(1).length + scraper.getHeadlines(2).length + scraper.getHeadlines(3).length + scraper.getHeadlines(4).length]; String[] sections = new String[scraper.getHeadlines(1).length + scraper.getHeadlines(2).length + scraper.getHeadlines(3).length + scraper.getHeadlines(4).length];
int p = 0; int p = 0;
for (int i = 1; i <= 4; i++) for (int j = 0; j < scraper.getHeadlines(i).length; j++) sections[p++] = scraper.getHeadlines(i)[j]; for (int i = 1; i <= 4; i++) for (int j = 0; j < scraper.getHeadlines(i).length; j++) sections[p++] = scraper.getHeadlines(i)[j];
plasmaParserDocument ppd = new plasmaParserDocument( plasmaParserDocument ppd = new plasmaParserDocument(
new yacyURL(location.toNormalform(true, true), null), location,
mimeType, mimeType,
charSet, charSet,
scraper.getKeywords(), scraper.getKeywords(),
@ -703,10 +703,6 @@ public final class plasmaParser {
//scraper.close(); //scraper.close();
ppd.setFavicon(scraper.getFavicon()); ppd.setFavicon(scraper.getFavicon());
return ppd; return ppd;
} catch (MalformedURLException e) {
//e.printStackTrace();
return null;
}
} }
/** /**
@ -737,7 +733,7 @@ public final class plasmaParser {
Parser theParser = makeParser(parserClassName); Parser theParser = makeParser(parserClassName);
// checking if the created parser really supports the given mimetype // checking if the created parser really supports the given mimetype
Hashtable supportedMimeTypes = theParser.getSupportedMimeTypes(); Hashtable<String, String> supportedMimeTypes = theParser.getSupportedMimeTypes();
if ((supportedMimeTypes != null) && (supportedMimeTypes.containsKey(mimeType))) { if ((supportedMimeTypes != null) && (supportedMimeTypes.containsKey(mimeType))) {
parserInfo.incUsageCounter(); parserInfo.incUsageCounter();
return theParser; return theParser;
@ -751,64 +747,73 @@ public final class plasmaParser {
} }
static Map<String, String> allReflinks(Set links) { static Map<yacyURL, String> allReflinks(Set<?> links) {
// links is either a Set of Strings (with urls) or htmlFilterImageEntries // links is either a Set of Strings (with urls) or htmlFilterImageEntries
// we find all links that are part of a reference inside a url // we find all links that are part of a reference inside a url
HashMap<String, String> v = new HashMap<String, String>(); HashMap<yacyURL, String> v = new HashMap<yacyURL, String>();
Iterator i = links.iterator(); Iterator<?> i = links.iterator();
Object o; Object o;
String url; yacyURL url;
String u;
int pos; int pos;
loop: while (i.hasNext()) { loop: while (i.hasNext()) try {
o = i.next(); o = i.next();
if (o instanceof String) url = (String) o; if (o instanceof yacyURL) url = (yacyURL) o;
else if (o instanceof htmlFilterImageEntry) url = ((htmlFilterImageEntry) o).url().toNormalform(true, true); else if (o instanceof String) url = new yacyURL((String) o, null);
else if (o instanceof htmlFilterImageEntry) url = ((htmlFilterImageEntry) o).url();
else { else {
assert false; assert false;
continue; continue;
} }
if ((pos = url.toLowerCase().indexOf("http://",7)) > 0) { u = url.toNormalform(true, true);
if ((pos = u.toLowerCase().indexOf("http://",7)) > 0) {
i.remove(); i.remove();
url = url.substring(pos); u = u.substring(pos);
while ((pos = url.toLowerCase().indexOf("http://",7)) > 0) url = url.substring(pos); while ((pos = u.toLowerCase().indexOf("http://",7)) > 0) u = u.substring(pos);
url = new yacyURL(u, null);
if (!(v.containsKey(url))) v.put(url, "ref"); if (!(v.containsKey(url))) v.put(url, "ref");
continue loop; continue loop;
} }
if ((pos = url.toLowerCase().indexOf("/www.",7)) > 0) { if ((pos = u.toLowerCase().indexOf("/www.",7)) > 0) {
i.remove(); i.remove();
url = "http:/" + url.substring(pos); u = "http:/" + u.substring(pos);
while ((pos = url.toLowerCase().indexOf("/www.",7)) > 0) url = "http:/" + url.substring(pos); while ((pos = u.toLowerCase().indexOf("/www.",7)) > 0) u = "http:/" + u.substring(pos);
url = new yacyURL(u, null);
if (!(v.containsKey(url))) v.put(url, "ref"); if (!(v.containsKey(url))) v.put(url, "ref");
continue loop; continue loop;
} }
} } catch (MalformedURLException e) {}
return v; return v;
} }
static Map<String, String> allSubpaths(Set links) { static Map<yacyURL, String> allSubpaths(Set<?> links) {
// links is either a Set of Strings (urls) or a Set of htmlFilterImageEntries // links is either a Set of Strings (urls) or a Set of htmlFilterImageEntries
HashMap<String, String> v = new HashMap<String, String>(); HashMap<yacyURL, String> v = new HashMap<yacyURL, String>();
Iterator i = links.iterator(); Iterator<?> i = links.iterator();
Object o; Object o;
String url; yacyURL url;
String u;
int pos; int pos;
while (i.hasNext()) { while (i.hasNext()) try {
o = i.next(); o = i.next();
if (o instanceof String) url = (String) o; if (o instanceof yacyURL) url = (yacyURL) o;
else if (o instanceof htmlFilterImageEntry) url = ((htmlFilterImageEntry) o).url().toNormalform(true, true); else if (o instanceof String) url = new yacyURL((String) o, null);
else if (o instanceof htmlFilterImageEntry) url = ((htmlFilterImageEntry) o).url();
else { else {
assert false; assert false;
continue; continue;
} }
if (url.endsWith("/")) url = url.substring(0, url.length() - 1); u = url.toNormalform(true, true);
pos = url.lastIndexOf("/"); if (u.endsWith("/")) u = u.substring(0, u.length() - 1);
pos = u.lastIndexOf("/");
while (pos > 8) { while (pos > 8) {
url = url.substring(0, pos + 1); u = u.substring(0, pos + 1);
url = new yacyURL(u, null);
if (!(v.containsKey(url))) v.put(url, "sub"); if (!(v.containsKey(url))) v.put(url, "sub");
url = url.substring(0, pos); u = u.substring(0, pos);
pos = url.lastIndexOf("/"); pos = u.lastIndexOf("/");
}
} }
} catch (MalformedURLException e) {}
return v; return v;
} }
@ -883,24 +888,24 @@ public final class plasmaParser {
// printing out all parsed sentences // printing out all parsed sentences
if (document != null) { if (document != null) {
System.out.print("Document titel: "); System.out.print("Document titel: ");
System.out.println(document.getTitle()); System.out.println(document.dc_title());
// found text // found text
final Iterator sentences = document.getSentences(false); final Iterator<StringBuffer> sentences = document.getSentences(false);
int i = 0; int i = 0;
if (sentences != null) while (sentences.hasNext()) { if (sentences != null) while (sentences.hasNext()) {
System.out.print("line " + i + ": "); System.out.print("line " + i + ": ");
System.out.println(((StringBuffer) sentences.next()).toString()); System.out.println(sentences.next().toString());
i++; i++;
} }
// found links // found links
int anchorNr = 0; int anchorNr = 0;
Map anchors = document.getAnchors(); Map<yacyURL, String> anchors = document.getAnchors();
Iterator anchorIter = anchors.keySet().iterator(); Iterator<yacyURL> anchorIter = anchors.keySet().iterator();
while (anchorIter.hasNext()) { while (anchorIter.hasNext()) {
String key = (String) anchorIter.next(); yacyURL key = anchorIter.next();
System.out.println("URL " + anchorNr + ":\t" + key + " | " + anchors.get(key)); System.out.println("URL " + anchorNr + ":\t" + key.toString() + " | " + anchors.get(key));
anchorNr++; anchorNr++;
} }
document.close(); document.close();
@ -913,9 +918,9 @@ public final class plasmaParser {
public static boolean supportedContent(yacyURL url, String mimeType) { public static boolean supportedContent(yacyURL url, String mimeType) {
if (url == null) throw new NullPointerException(); if (url == null) throw new NullPointerException();
Iterator configs = parserConfigList.values().iterator(); Iterator<plasmaParserConfig> configs = parserConfigList.values().iterator();
while (configs.hasNext()) { while (configs.hasNext()) {
plasmaParserConfig currentConfig = (plasmaParserConfig) configs.next(); plasmaParserConfig currentConfig = configs.next();
synchronized (currentConfig.enabledParserList) { synchronized (currentConfig.enabledParserList) {
if (currentConfig.supportedContent(url, mimeType)) return true; if (currentConfig.supportedContent(url, mimeType)) return true;
} }
@ -944,7 +949,7 @@ public final class plasmaParser {
config.initParseableMimeTypes(configStr); config.initParseableMimeTypes(configStr);
} }
public static String[] setEnabledParserList(String parserMode, Set mimeTypeSet) { public static String[] setEnabledParserList(String parserMode, Set<String> mimeTypeSet) {
if (!PARSER_MODE.contains(parserMode)) throw new IllegalArgumentException(); if (!PARSER_MODE.contains(parserMode)) throw new IllegalArgumentException();
plasmaParserConfig config = (plasmaParserConfig) parserConfigList.get(parserMode); plasmaParserConfig config = (plasmaParserConfig) parserConfigList.get(parserMode);
@ -956,9 +961,9 @@ public final class plasmaParser {
} }
public static boolean supportedFileExtContains(String fileExt) { public static boolean supportedFileExtContains(String fileExt) {
Iterator configs = parserConfigList.values().iterator(); Iterator<plasmaParserConfig> configs = parserConfigList.values().iterator();
while (configs.hasNext()) { while (configs.hasNext()) {
plasmaParserConfig currentConfig = (plasmaParserConfig) configs.next(); plasmaParserConfig currentConfig = configs.next();
synchronized (currentConfig.enabledParserList) { synchronized (currentConfig.enabledParserList) {
if (currentConfig.supportedFileExtContains(fileExt)) return true; if (currentConfig.supportedFileExtContains(fileExt)) return true;
} }
@ -968,9 +973,9 @@ public final class plasmaParser {
} }
public static boolean supportedMimeTypesContains(String mimeType) { public static boolean supportedMimeTypesContains(String mimeType) {
Iterator configs = parserConfigList.values().iterator(); Iterator<plasmaParserConfig> configs = parserConfigList.values().iterator();
while (configs.hasNext()) { while (configs.hasNext()) {
plasmaParserConfig currentConfig = (plasmaParserConfig) configs.next(); plasmaParserConfig currentConfig = configs.next();
synchronized (currentConfig.enabledParserList) { synchronized (currentConfig.enabledParserList) {
if (currentConfig.supportedMimeTypesContains(mimeType)) return true; if (currentConfig.supportedMimeTypesContains(mimeType)) return true;
} }
@ -985,7 +990,7 @@ public final class plasmaParser {
throw new IllegalArgumentException("The object key must be of type string."); throw new IllegalArgumentException("The object key must be of type string.");
// loading class by name // loading class by name
Class moduleClass = Class.forName((String)name); Class<?> moduleClass = Class.forName((String)name);
// instantiating class // instantiating class
Parser theParser = (Parser) moduleClass.newInstance(); Parser theParser = (Parser) moduleClass.newInstance();

27
source/de/anomic/plasma/plasmaParserConfig.java
Unescape View File

@ -65,12 +65,12 @@ public class plasmaParserConfig {
* @see #loadEnabledParserList() * @see #loadEnabledParserList()
* @see #setEnabledParserList(Enumeration) * @see #setEnabledParserList(Enumeration)
*/ */
final HashSet enabledParserList = new HashSet(); final HashSet<String> enabledParserList = new HashSet<String>();
/** /**
* A list of file extensions that are supported by all enabled parsers * A list of file extensions that are supported by all enabled parsers
*/ */
final HashSet supportedFileExt = new HashSet(); final HashSet<String> supportedFileExt = new HashSet<String>();
/** /**
* Parsermode this configuration belongs to * Parsermode this configuration belongs to
@ -134,29 +134,29 @@ public class plasmaParserConfig {
} }
public void initParseableMimeTypes(String enabledMimeTypes) { public void initParseableMimeTypes(String enabledMimeTypes) {
HashSet mimeTypes = null; HashSet<String> mimeTypes = null;
if ((enabledMimeTypes == null) || (enabledMimeTypes.length() == 0)) { if ((enabledMimeTypes == null) || (enabledMimeTypes.length() == 0)) {
mimeTypes = new HashSet(); mimeTypes = new HashSet<String>();
} else { } else {
String[] enabledMimeTypeList = enabledMimeTypes.split(","); String[] enabledMimeTypeList = enabledMimeTypes.split(",");
mimeTypes = new HashSet(enabledMimeTypeList.length); mimeTypes = new HashSet<String>(enabledMimeTypeList.length);
for (int i = 0; i < enabledMimeTypeList.length; i++) mimeTypes.add(enabledMimeTypeList[i].toLowerCase().trim()); for (int i = 0; i < enabledMimeTypeList.length; i++) mimeTypes.add(enabledMimeTypeList[i].toLowerCase().trim());
} }
setEnabledParserList(mimeTypes); setEnabledParserList(mimeTypes);
} }
public void enableAllParsers() { public void enableAllParsers() {
Set availableMimeTypes = plasmaParser.availableParserList.keySet(); Set<String> availableMimeTypes = plasmaParser.availableParserList.keySet();
setEnabledParserList(availableMimeTypes); setEnabledParserList(availableMimeTypes);
} }
public String[] setEnabledParserList(Set mimeTypeSet) { public String[] setEnabledParserList(Set<String> mimeTypeSet) {
HashSet newEnabledParsers = new HashSet(); HashSet<String> newEnabledParsers = new HashSet<String>();
HashSet newSupportedFileExt = new HashSet(); HashSet<String> newSupportedFileExt = new HashSet<String>();
if (mimeTypeSet != null) { if (mimeTypeSet != null) {
Iterator mimeTypes = mimeTypeSet.iterator(); Iterator<String> mimeTypes = mimeTypeSet.iterator();
while (mimeTypes.hasNext()) { while (mimeTypes.hasNext()) {
String mimeType = (String) mimeTypes.next(); String mimeType = (String) mimeTypes.next();
if (plasmaParser.availableParserList.containsKey(mimeType)) { if (plasmaParser.availableParserList.containsKey(mimeType)) {
@ -166,7 +166,7 @@ public class plasmaParserConfig {
theParser = plasmaParser.makeParser(((ParserInfo)plasmaParser.availableParserList.get(mimeType)).parserClassName); theParser = plasmaParser.makeParser(((ParserInfo)plasmaParser.availableParserList.get(mimeType)).parserClassName);
// getting a list of mimeTypes that the parser supports // getting a list of mimeTypes that the parser supports
Hashtable parserSupportsMimeTypes = theParser.getSupportedMimeTypes(); Hashtable<String, String> parserSupportsMimeTypes = theParser.getSupportedMimeTypes();
if (parserSupportsMimeTypes != null) { if (parserSupportsMimeTypes != null) {
Object supportedExtensions = parserSupportsMimeTypes.get(mimeType); Object supportedExtensions = parserSupportsMimeTypes.get(mimeType);
if ((supportedExtensions != null) && if ((supportedExtensions != null) &&
@ -202,9 +202,10 @@ public class plasmaParserConfig {
return (String[])newEnabledParsers.toArray(new String[newEnabledParsers.size()]); return (String[])newEnabledParsers.toArray(new String[newEnabledParsers.size()]);
} }
public HashSet getEnabledParserList() { @SuppressWarnings("unchecked")
public HashSet<String> getEnabledParserList() {
synchronized (this.enabledParserList) { synchronized (this.enabledParserList) {
return (HashSet) this.enabledParserList.clone(); return (HashSet<String>) this.enabledParserList.clone();
} }
} }
} }

167
source/de/anomic/plasma/plasmaParserDocument.java
Unescape View File

@ -48,7 +48,6 @@ import java.io.File;
import java.io.FileInputStream; import java.io.FileInputStream;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.net.MalformedURLException;
import de.anomic.server.serverCachedFileOutputStream; import de.anomic.server.serverCachedFileOutputStream;
import de.anomic.server.serverFileUtils; import de.anomic.server.serverFileUtils;
@ -67,21 +66,21 @@ import de.anomic.plasma.parser.Parser;
public class plasmaParserDocument { public class plasmaParserDocument {
private yacyURL location; // the source url private yacyURL source; // the source url
private String mimeType; // mimeType as taken from http header private String mimeType; // mimeType as taken from http header
private String charset; // the charset of the document private String charset; // the charset of the document
private List<String> keywords; // most resources provide a keyword field private List<String> keywords; // most resources provide a keyword field
private StringBuffer title; // a document title, taken from title or h1 tag; shall appear as headline of search result private StringBuffer title; // a document title, taken from title or h1 tag; shall appear as headline of search result
private StringBuffer author; // author or copyright private StringBuffer creator; // author or copyright
private List<String> sections; // if present: more titles/headlines appearing in the document private List<String> sections; // if present: more titles/headlines appearing in the document
private StringBuffer abstrct; // an abstract, if present: short content description private StringBuffer description; // an abstract, if present: short content description
private Object text; // the clear text, all that is visible private Object text; // the clear text, all that is visible
private Map<String, String> anchors; // all links embedded as clickeable entities (anchor tags) private Map<yacyURL, String> anchors; // all links embedded as clickeable entities (anchor tags)
private TreeSet<htmlFilterImageEntry> images; // all visible pictures in document private TreeSet<htmlFilterImageEntry> images; // all visible pictures in document
// the anchors and images - Maps are URL-to-EntityDescription mappings. // the anchors and images - Maps are URL-to-EntityDescription mappings.
// The EntityDescription appear either as visible text in anchors or as alternative // The EntityDescription appear either as visible text in anchors or as alternative
// text in image tags. // text in image tags.
private Map<String, String> hyperlinks, audiolinks, videolinks, applinks; private Map<yacyURL, String> hyperlinks, audiolinks, videolinks, applinks;
private Map<String, String> emaillinks; private Map<String, String> emaillinks;
private yacyURL favicon; private yacyURL favicon;
private boolean resorted; private boolean resorted;
@ -90,16 +89,16 @@ public class plasmaParserDocument {
protected plasmaParserDocument(yacyURL location, String mimeType, String charset, protected plasmaParserDocument(yacyURL location, String mimeType, String charset,
String[] keywords, String title, String author, String[] keywords, String title, String author,
String[] sections, String abstrct, String[] sections, String abstrct,
Object text, Map<String, String> anchors, TreeSet<htmlFilterImageEntry> images) { Object text, Map<yacyURL, String> anchors, TreeSet<htmlFilterImageEntry> images) {
this.location = location; this.source = location;
this.mimeType = (mimeType == null) ? "application/octet-stream" : mimeType; this.mimeType = (mimeType == null) ? "application/octet-stream" : mimeType;
this.charset = charset; this.charset = charset;
this.keywords = (keywords == null) ? new LinkedList<String>() : Arrays.asList(keywords); this.keywords = (keywords == null) ? new LinkedList<String>() : Arrays.asList(keywords);
this.title = (title == null) ? new StringBuffer() : new StringBuffer(title); this.title = (title == null) ? new StringBuffer() : new StringBuffer(title);
this.author = (author == null) ? new StringBuffer() : new StringBuffer(author); this.creator = (author == null) ? new StringBuffer() : new StringBuffer(author);
this.sections = (sections == null) ? new LinkedList<String>() : Arrays.asList(sections); this.sections = (sections == null) ? new LinkedList<String>() : Arrays.asList(sections);
this.abstrct = (abstrct == null) ? new StringBuffer() : new StringBuffer(abstrct); this.description = (abstrct == null) ? new StringBuffer() : new StringBuffer(abstrct);
this.anchors = (anchors == null) ? new HashMap<String, String>(0) : anchors; this.anchors = (anchors == null) ? new HashMap<yacyURL, String>(0) : anchors;
this.images = (images == null) ? new TreeSet<htmlFilterImageEntry>() : images; this.images = (images == null) ? new TreeSet<htmlFilterImageEntry>() : images;
this.hyperlinks = null; this.hyperlinks = null;
this.audiolinks = null; this.audiolinks = null;
@ -125,32 +124,90 @@ public class plasmaParserDocument {
public plasmaParserDocument(yacyURL location, String mimeType, String charset, public plasmaParserDocument(yacyURL location, String mimeType, String charset,
String[] keywords, String title, String author, String[] keywords, String title, String author,
String[] sections, String abstrct, String[] sections, String abstrct,
byte[] text, Map<String, String> anchors, TreeSet<htmlFilterImageEntry> images) { byte[] text, Map<yacyURL, String> anchors, TreeSet<htmlFilterImageEntry> images) {
this(location, mimeType, charset, keywords, title, author, sections, abstrct, (Object)text, anchors, images); this(location, mimeType, charset, keywords, title, author, sections, abstrct, (Object)text, anchors, images);
} }
public plasmaParserDocument(yacyURL location, String mimeType, String charset, public plasmaParserDocument(yacyURL location, String mimeType, String charset,
String[] keywords, String title, String author, String[] keywords, String title, String author,
String[] sections, String abstrct, String[] sections, String abstrct,
File text, Map<String, String> anchors, TreeSet<htmlFilterImageEntry> images) { File text, Map<yacyURL, String> anchors, TreeSet<htmlFilterImageEntry> images) {
this(location, mimeType, charset, keywords, title, author, sections, abstrct, (Object)text, anchors, images); this(location, mimeType, charset, keywords, title, author, sections, abstrct, (Object)text, anchors, images);
} }
public plasmaParserDocument(yacyURL location, String mimeType, String charset, public plasmaParserDocument(yacyURL location, String mimeType, String charset,
String[] keywords, String title, String author, String[] keywords, String title, String author,
String[] sections, String abstrct, String[] sections, String abstrct,
serverCachedFileOutputStream text, Map<String, String> anchors, TreeSet<htmlFilterImageEntry> images) { serverCachedFileOutputStream text, Map<yacyURL, String> anchors, TreeSet<htmlFilterImageEntry> images) {
this(location, mimeType, charset, keywords, title, author, sections, abstrct, (Object)text, anchors, images); this(location, mimeType, charset, keywords, title, author, sections, abstrct, (Object)text, anchors, images);
} }
public yacyURL getLocation() { /*
return this.location; DC according to rfc 5013
* dc_title
* dc_creator
* dc_subject
* dc_description
* dc_publisher
dc_contributor
dc_date
dc_type
* dc_format
* dc_identifier
* dc_source
dc_language
dc_relation
dc_coverage
dc_rights
*/
public String dc_title() {
return title.toString();
} }
public String getMimeType() { public String dc_creator() {
if (creator != null) return creator.toString(); else return new String();
}
public String dc_subject(char separator) {
// sort out doubles and empty words
TreeSet<String> hs = new TreeSet<String>();
String s;
for (int i = 0; i < this.keywords.size(); i++) {
if (this.keywords.get(i) == null) continue;
s = ((String)this.keywords.get(i)).trim();
if (s.length() > 0) hs.add(s.toLowerCase());
}
if (hs.size() == 0) return "";
// generate a new list
StringBuffer sb = new StringBuffer(this.keywords.size() * 6);
Iterator<String> i = hs.iterator();
while (i.hasNext()) sb.append(i.next()).append(separator);
return sb.substring(0, sb.length() - 1);
}
public String dc_description() {
if (description != null) return description.toString(); else return dc_title();
}
public String dc_publisher() {
// if we don't have a publisher, simply return the host/domain name
return this.source.getHost();
}
public String dc_format() {
return this.mimeType; return this.mimeType;
} }
public String dc_identifier() {
return "yacy.net:" + this.source.hash();
}
public yacyURL dc_source() {
return this.source;
}
/** /**
* @return the supposed charset of this document or <code>null</code> if unknown * @return the supposed charset of this document or <code>null</code> if unknown
*/ */
@ -158,24 +215,12 @@ public class plasmaParserDocument {
return this.charset; return this.charset;
} }
public String getTitle() {
return title.toString();
}
public String[] getSectionTitles() { public String[] getSectionTitles() {
if (sections != null) { if (sections != null) {
return (String[])sections.toArray(new String[this.sections.size()]); return (String[])sections.toArray(new String[this.sections.size()]);
} else { } else {
return new String[] { getTitle() }; return new String[] { dc_title() };
}
}
public String getAbstract() {
if (abstrct != null) return abstrct.toString(); else return getTitle();
} }
public String getAuthor() {
if (author != null) return author.toString(); else return new String();
} }
public InputStream getText() { public InputStream getText() {
@ -236,28 +281,11 @@ public class plasmaParserDocument {
return e; return e;
} }
public String getKeywords(char separator) {
// sort out doubles and empty words
TreeSet<String> hs = new TreeSet<String>();
String s;
for (int i = 0; i < this.keywords.size(); i++) {
if (this.keywords.get(i) == null) continue;
s = ((String)this.keywords.get(i)).trim();
if (s.length() > 0) hs.add(s.toLowerCase());
}
if (hs.size() == 0) return "";
// generate a new list
StringBuffer sb = new StringBuffer(this.keywords.size() * 6);
Iterator<String> i = hs.iterator();
while (i.hasNext()) sb.append(i.next()).append(separator);
return sb.substring(0, sb.length() - 1);
}
public List<String> getKeywords() { public List<String> getKeywords() {
return this.keywords; return this.keywords;
} }
public Map<String, String> getAnchors() { public Map<yacyURL, String> getAnchors() {
// returns all links embedded as anchors (clickeable entities) // returns all links embedded as anchors (clickeable entities)
// this is a url(String)/text(String) map // this is a url(String)/text(String) map
return anchors; return anchors;
@ -266,18 +294,18 @@ public class plasmaParserDocument {
// the next three methods provide a calculated view on the getAnchors/getImages: // the next three methods provide a calculated view on the getAnchors/getImages:
public Map<String, String> getHyperlinks() { public Map<yacyURL, String> getHyperlinks() {
// this is a subset of the getAnchor-set: only links to other hyperrefs // this is a subset of the getAnchor-set: only links to other hyperrefs
if (!resorted) resortLinks(); if (!resorted) resortLinks();
return hyperlinks; return hyperlinks;
} }
public Map<String, String> getAudiolinks() { public Map<yacyURL, String> getAudiolinks() {
if (!resorted) resortLinks(); if (!resorted) resortLinks();
return this.audiolinks; return this.audiolinks;
} }
public Map<String, String> getVideolinks() { public Map<yacyURL, String> getVideolinks() {
if (!resorted) resortLinks(); if (!resorted) resortLinks();
return this.videolinks; return this.videolinks;
} }
@ -289,7 +317,7 @@ public class plasmaParserDocument {
return images; return images;
} }
public Map<String, String> getApplinks() { public Map<yacyURL, String> getApplinks() {
if (!resorted) resortLinks(); if (!resorted) resortLinks();
return this.applinks; return this.applinks;
} }
@ -307,17 +335,19 @@ public class plasmaParserDocument {
String u; String u;
int extpos, qpos; int extpos, qpos;
String ext = null; String ext = null;
Iterator<Map.Entry<String, String>> i = anchors.entrySet().iterator(); Iterator<Map.Entry<yacyURL, String>> i = anchors.entrySet().iterator();
hyperlinks = new HashMap<String, String>(); hyperlinks = new HashMap<yacyURL, String>();
videolinks = new HashMap<String, String>(); videolinks = new HashMap<yacyURL, String>();
audiolinks = new HashMap<String, String>(); audiolinks = new HashMap<yacyURL, String>();
applinks = new HashMap<String, String>(); applinks = new HashMap<yacyURL, String>();
emaillinks = new HashMap<String, String>(); emaillinks = new HashMap<String, String>();
TreeSet<htmlFilterImageEntry> collectedImages = new TreeSet<htmlFilterImageEntry>(); // this is a set that is collected now and joined later to the imagelinks TreeSet<htmlFilterImageEntry> collectedImages = new TreeSet<htmlFilterImageEntry>(); // this is a set that is collected now and joined later to the imagelinks
Map.Entry<String, String> entry; Map.Entry<yacyURL, String> entry;
while (i.hasNext()) { while (i.hasNext()) {
entry = i.next(); entry = i.next();
u = entry.getKey(); url = entry.getKey();
if (url == null) continue;
u = url.toNormalform(true, false);
if ((u != null) && (u.startsWith("mailto:"))) { if ((u != null) && (u.startsWith("mailto:"))) {
emaillinks.put(u.substring(7), entry.getValue()); emaillinks.put(u.substring(7), entry.getValue());
} else { } else {
@ -328,21 +358,16 @@ public class plasmaParserDocument {
} else { } else {
ext = u.substring(extpos + 1).toLowerCase(); ext = u.substring(extpos + 1).toLowerCase();
} }
try {
url = new yacyURL(u, null);
u = url.toNormalform(true, true);
if (plasmaParser.mediaExtContains(ext)) { if (plasmaParser.mediaExtContains(ext)) {
// this is not a normal anchor, its a media link // this is not a normal anchor, its a media link
if (plasmaParser.imageExtContains(ext)) { if (plasmaParser.imageExtContains(ext)) {
collectedImages.add(new htmlFilterImageEntry(url, (String) entry.getValue(), -1, -1)); collectedImages.add(new htmlFilterImageEntry(url, (String) entry.getValue(), -1, -1));
} }
else if (plasmaParser.audioExtContains(ext)) audiolinks.put(u, (String)entry.getValue()); else if (plasmaParser.audioExtContains(ext)) audiolinks.put(url, (String)entry.getValue());
else if (plasmaParser.videoExtContains(ext)) videolinks.put(u, (String)entry.getValue()); else if (plasmaParser.videoExtContains(ext)) videolinks.put(url, (String)entry.getValue());
else if (plasmaParser.appsExtContains(ext)) applinks.put(u, (String)entry.getValue()); else if (plasmaParser.appsExtContains(ext)) applinks.put(url, (String)entry.getValue());
} else { } else {
hyperlinks.put(u, (String)entry.getValue()); hyperlinks.put(url, (String) entry.getValue());
}
} catch (MalformedURLException e1) {
} }
} }
} }
@ -378,12 +403,12 @@ public class plasmaParserDocument {
this.sections.addAll(Arrays.asList(doc.getSectionTitles())); this.sections.addAll(Arrays.asList(doc.getSectionTitles()));
if (this.title.length() > 0) this.title.append('\n'); if (this.title.length() > 0) this.title.append('\n');
this.title.append(doc.getTitle()); this.title.append(doc.dc_title());
this.keywords.addAll(doc.getKeywords()); this.keywords.addAll(doc.getKeywords());
if (this.abstrct.length() > 0) this.abstrct.append('\n'); if (this.description.length() > 0) this.description.append('\n');
this.abstrct.append(doc.getAbstract()); this.description.append(doc.dc_description());
if (!(this.text instanceof serverCachedFileOutputStream)) { if (!(this.text instanceof serverCachedFileOutputStream)) {
this.text = new serverCachedFileOutputStream(Parser.MAX_KEEP_IN_MEMORY_SIZE); this.text = new serverCachedFileOutputStream(Parser.MAX_KEEP_IN_MEMORY_SIZE);

4
source/de/anomic/plasma/plasmaSearchImages.java
Unescape View File

@ -79,11 +79,11 @@ public final class plasmaSearchImages {
// add also links from pages one step deeper, if depth > 0 // add also links from pages one step deeper, if depth > 0
if (depth > 0) { if (depth > 0) {
Iterator<String> i = document.getHyperlinks().keySet().iterator(); Iterator<yacyURL> i = document.getHyperlinks().keySet().iterator();
String nexturlstring; String nexturlstring;
while (i.hasNext()) { while (i.hasNext()) {
try { try {
nexturlstring = new yacyURL(i.next(), null).toNormalform(true, true); nexturlstring = i.next().toNormalform(true, true);
addAll(new plasmaSearchImages(serverDate.remainingTime(start, maxTime, 10), new yacyURL(nexturlstring, null), depth - 1)); addAll(new plasmaSearchImages(serverDate.remainingTime(start, maxTime, 10), new yacyURL(nexturlstring, null), depth - 1));
} catch (MalformedURLException e1) { } catch (MalformedURLException e1) {
e1.printStackTrace(); e1.printStackTrace();

34
source/de/anomic/plasma/plasmaSnippetCache.java
Unescape View File

@ -228,8 +228,9 @@ public class plasmaSnippetCache {
public static class MediaSnippet { public static class MediaSnippet {
public int type; public int type;
public String href, name, attr; public yacyURL href;
public MediaSnippet(int type, String href, String name, String attr) { public String name, attr;
public MediaSnippet(int type, yacyURL href, String name, String attr) {
this.type = type; this.type = type;
this.href = href; this.href = href;
this.name = name; this.name = name;
@ -469,17 +470,18 @@ public class plasmaSnippetCache {
return snippetsCache.get(key); return snippetsCache.get(key);
} }
private static String computeMediaSnippet(Map<String, String> media, Set<String> queryhashes) { private static String computeMediaSnippet(Map<yacyURL, String> media, Set<String> queryhashes) {
Iterator<Map.Entry<String, String>> i = media.entrySet().iterator(); Iterator<Map.Entry<yacyURL, String>> i = media.entrySet().iterator();
Map.Entry<String, String> entry; Map.Entry<yacyURL, String> entry;
String url, desc; yacyURL url;
String desc;
Set<String> s; Set<String> s;
String result = ""; String result = "";
while (i.hasNext()) { while (i.hasNext()) {
entry = i.next(); entry = i.next();
url = entry.getKey(); url = entry.getKey();
desc = entry.getValue(); desc = entry.getValue();
s = removeAppearanceHashes(url, queryhashes); s = removeAppearanceHashes(url.toNormalform(false, false), queryhashes);
if (s.size() == 0) { if (s.size() == 0) {
result += "<br /><a href=\"" + url + "\">" + ((desc.length() == 0) ? url : desc) + "</a>"; result += "<br /><a href=\"" + url + "\">" + ((desc.length() == 0) ? url : desc) + "</a>";
continue; continue;
@ -643,22 +645,23 @@ public class plasmaSnippetCache {
public static ArrayList<MediaSnippet> computeMediaSnippets(plasmaParserDocument document, Set<String> queryhashes, int mediatype) { public static ArrayList<MediaSnippet> computeMediaSnippets(plasmaParserDocument document, Set<String> queryhashes, int mediatype) {
if (document == null) return new ArrayList<MediaSnippet>(); if (document == null) return new ArrayList<MediaSnippet>();
Map<String, String> media = null; Map<yacyURL, String> media = null;
if (mediatype == plasmaSearchQuery.CONTENTDOM_AUDIO) media = document.getAudiolinks(); if (mediatype == plasmaSearchQuery.CONTENTDOM_AUDIO) media = document.getAudiolinks();
else if (mediatype == plasmaSearchQuery.CONTENTDOM_VIDEO) media = document.getVideolinks(); else if (mediatype == plasmaSearchQuery.CONTENTDOM_VIDEO) media = document.getVideolinks();
else if (mediatype == plasmaSearchQuery.CONTENTDOM_APP) media = document.getApplinks(); else if (mediatype == plasmaSearchQuery.CONTENTDOM_APP) media = document.getApplinks();
if (media == null) return null; if (media == null) return null;
Iterator<Map.Entry<String, String>> i = media.entrySet().iterator(); Iterator<Map.Entry<yacyURL, String>> i = media.entrySet().iterator();
Map.Entry<String, String> entry; Map.Entry<yacyURL, String> entry;
String url, desc; yacyURL url;
String desc;
Set<String> s; Set<String> s;
ArrayList<MediaSnippet> result = new ArrayList<MediaSnippet>(); ArrayList<MediaSnippet> result = new ArrayList<MediaSnippet>();
while (i.hasNext()) { while (i.hasNext()) {
entry = i.next(); entry = i.next();
url = entry.getKey(); url = entry.getKey();
desc = entry.getValue(); desc = entry.getValue();
s = removeAppearanceHashes(url, queryhashes); s = removeAppearanceHashes(url.toNormalform(false, false), queryhashes);
if (s.size() == 0) { if (s.size() == 0) {
result.add(new MediaSnippet(mediatype, url, desc, null)); result.add(new MediaSnippet(mediatype, url, desc, null));
continue; continue;
@ -678,14 +681,15 @@ public class plasmaSnippetCache {
Iterator<htmlFilterImageEntry> i = images.iterator(); Iterator<htmlFilterImageEntry> i = images.iterator();
htmlFilterImageEntry ientry; htmlFilterImageEntry ientry;
String url, desc; yacyURL url;
String desc;
Set<String> s; Set<String> s;
ArrayList<MediaSnippet> result = new ArrayList<MediaSnippet>(); ArrayList<MediaSnippet> result = new ArrayList<MediaSnippet>();
while (i.hasNext()) { while (i.hasNext()) {
ientry = i.next(); ientry = i.next();
url = ientry.url().toNormalform(true, true); url = ientry.url();
desc = ientry.alt(); desc = ientry.alt();
s = removeAppearanceHashes(url, queryhashes); s = removeAppearanceHashes(url.toNormalform(false, false), queryhashes);
if (s.size() == 0) { if (s.size() == 0) {
result.add(new MediaSnippet(plasmaSearchQuery.CONTENTDOM_IMAGE, url, desc, ientry.width() + " x " + ientry.height())); result.add(new MediaSnippet(plasmaSearchQuery.CONTENTDOM_IMAGE, url, desc, ientry.width() + " x " + ientry.height()));
continue; continue;

91
source/de/anomic/plasma/plasmaSwitchboard.java
Unescape View File

@ -138,7 +138,7 @@ import de.anomic.plasma.crawler.plasmaCrawlQueues;
import de.anomic.plasma.crawler.plasmaProtocolLoader; import de.anomic.plasma.crawler.plasmaProtocolLoader;
import de.anomic.plasma.dbImport.dbImportManager; import de.anomic.plasma.dbImport.dbImportManager;
import de.anomic.plasma.parser.ParserException; import de.anomic.plasma.parser.ParserException;
import de.anomic.plasma.plasmaSwitchboardQueue.Entry; import de.anomic.plasma.plasmaCondenser.wordStatProp;
import de.anomic.plasma.urlPattern.defaultURLPattern; import de.anomic.plasma.urlPattern.defaultURLPattern;
import de.anomic.plasma.urlPattern.plasmaURLPattern; import de.anomic.plasma.urlPattern.plasmaURLPattern;
import de.anomic.server.serverAbstractSwitch; import de.anomic.server.serverAbstractSwitch;
@ -998,8 +998,8 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
this.log.logConfig("Starting blacklist engine ..."); this.log.logConfig("Starting blacklist engine ...");
try { try {
Class blacklistClass = Class.forName(blacklistClassName); Class<?> blacklistClass = Class.forName(blacklistClassName);
Constructor blacklistClassConstr = blacklistClass.getConstructor( new Class[] { File.class } ); Constructor<?> blacklistClassConstr = blacklistClass.getConstructor( new Class[] { File.class } );
urlBlacklist = (plasmaURLPattern) blacklistClassConstr.newInstance(new Object[] { blacklistsPath }); urlBlacklist = (plasmaURLPattern) blacklistClassConstr.newInstance(new Object[] { blacklistsPath });
this.log.logFine("Used blacklist engine class: " + blacklistClassName); this.log.logFine("Used blacklist engine class: " + blacklistClassName);
this.log.logConfig("Using blacklist engine: " + urlBlacklist.getEngineInfo()); this.log.logConfig("Using blacklist engine: " + urlBlacklist.getEngineInfo());
@ -1276,8 +1276,8 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
String wikiParserClassName = getConfig(WIKIPARSER_CLASS, WIKIPARSER_CLASS_DEFAULT); String wikiParserClassName = getConfig(WIKIPARSER_CLASS, WIKIPARSER_CLASS_DEFAULT);
this.log.logConfig("Loading wiki parser " + wikiParserClassName + " ..."); this.log.logConfig("Loading wiki parser " + wikiParserClassName + " ...");
try { try {
Class wikiParserClass = Class.forName(wikiParserClassName); Class<?> wikiParserClass = Class.forName(wikiParserClassName);
Constructor wikiParserClassConstr = wikiParserClass.getConstructor(new Class[] { plasmaSwitchboard.class }); Constructor<?> wikiParserClassConstr = wikiParserClass.getConstructor(new Class[] { plasmaSwitchboard.class });
wikiParser = (wikiParser)wikiParserClassConstr.newInstance(new Object[] { this }); wikiParser = (wikiParser)wikiParserClassConstr.newInstance(new Object[] { this });
} catch (Exception e) { } catch (Exception e) {
this.log.logSevere("Unable to load wiki parser, the wiki won't work", e); this.log.logSevere("Unable to load wiki parser, the wiki won't work", e);
@ -1532,11 +1532,11 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
this.defaultRemoteProfile = null; this.defaultRemoteProfile = null;
this.defaultTextSnippetProfile = null; this.defaultTextSnippetProfile = null;
this.defaultMediaSnippetProfile = null; this.defaultMediaSnippetProfile = null;
Iterator i = this.profilesActiveCrawls.profiles(true); Iterator<plasmaCrawlProfile.entry> i = this.profilesActiveCrawls.profiles(true);
plasmaCrawlProfile.entry profile; plasmaCrawlProfile.entry profile;
String name; String name;
while (i.hasNext()) { while (i.hasNext()) {
profile = (plasmaCrawlProfile.entry) i.next(); profile = i.next();
name = profile.name(); name = profile.name();
if (name.equals(CRAWL_PROFILE_PROXY)) this.defaultProxyProfile = profile; if (name.equals(CRAWL_PROFILE_PROXY)) this.defaultProxyProfile = profile;
if (name.equals(CRAWL_PROFILE_REMOTE)) this.defaultRemoteProfile = profile; if (name.equals(CRAWL_PROFILE_REMOTE)) this.defaultRemoteProfile = profile;
@ -1606,7 +1606,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
(crawlStacker != null && crawlStacker.size() > 0) || (crawlStacker != null && crawlStacker.size() > 0) ||
(crawlQueues.noticeURL.notEmpty())) (crawlQueues.noticeURL.notEmpty()))
return false; return false;
final Iterator iter = profilesActiveCrawls.profiles(true); final Iterator<plasmaCrawlProfile.entry> iter = profilesActiveCrawls.profiles(true);
plasmaCrawlProfile.entry entry; plasmaCrawlProfile.entry entry;
boolean hasDoneSomething = false; boolean hasDoneSomething = false;
try { try {
@ -1615,7 +1615,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
if (Thread.currentThread().isInterrupted()) throw new InterruptedException("Shutdown in progress"); if (Thread.currentThread().isInterrupted()) throw new InterruptedException("Shutdown in progress");
// getting next profile // getting next profile
entry = (plasmaCrawlProfile.entry) iter.next(); entry = iter.next();
if (!((entry.name().equals(CRAWL_PROFILE_PROXY)) || if (!((entry.name().equals(CRAWL_PROFILE_PROXY)) ||
(entry.name().equals(CRAWL_PROFILE_REMOTE)) || (entry.name().equals(CRAWL_PROFILE_REMOTE)) ||
(entry.name().equals(CRAWL_PROFILE_SNIPPET_TEXT)) || (entry.name().equals(CRAWL_PROFILE_SNIPPET_TEXT)) ||
@ -1961,13 +1961,13 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
// clean up seed-dbs // clean up seed-dbs
if(getConfigBool("routing.deleteOldSeeds.permission",true)) { if(getConfigBool("routing.deleteOldSeeds.permission",true)) {
final long deleteOldSeedsTime = getConfigLong("routing.deleteOldSeeds.time",7)*24*3600000; final long deleteOldSeedsTime = getConfigLong("routing.deleteOldSeeds.time",7)*24*3600000;
Iterator e = yacyCore.seedDB.seedsSortedDisconnected(true,yacySeed.LASTSEEN); Iterator<yacySeed> e = yacyCore.seedDB.seedsSortedDisconnected(true,yacySeed.LASTSEEN);
yacySeed seed = null; yacySeed seed = null;
ArrayList deleteQueue = new ArrayList(); ArrayList<String> deleteQueue = new ArrayList<String>();
checkInterruption(); checkInterruption();
//clean passive seeds //clean passive seeds
while(e.hasNext()) { while(e.hasNext()) {
seed = (yacySeed)e.next(); seed = e.next();
if(seed != null) { if(seed != null) {
//list is sorted -> break when peers are too young to delete //list is sorted -> break when peers are too young to delete
if(seed.getLastSeenUTC() > (System.currentTimeMillis()-deleteOldSeedsTime)) if(seed.getLastSeenUTC() > (System.currentTimeMillis()-deleteOldSeedsTime))
@ -2183,24 +2183,19 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
((processCase == PROCESSCASE_4_PROXY_LOAD) || (processCase == PROCESSCASE_5_LOCAL_CRAWLING)) && ((processCase == PROCESSCASE_4_PROXY_LOAD) || (processCase == PROCESSCASE_5_LOCAL_CRAWLING)) &&
((entry.profile() == null) || (entry.depth() < entry.profile().generalDepth())) ((entry.profile() == null) || (entry.depth() < entry.profile().generalDepth()))
) { ) {
Map hl = document.getHyperlinks(); Map<yacyURL, String> hl = document.getHyperlinks();
Iterator i = hl.entrySet().iterator(); Iterator<Map.Entry<yacyURL, String>> i = hl.entrySet().iterator();
String nextUrlString;
yacyURL nextUrl; yacyURL nextUrl;
Map.Entry nextEntry; Map.Entry<yacyURL, String> nextEntry;
while (i.hasNext()) { while (i.hasNext()) {
// check for interruption // check for interruption
checkInterruption(); checkInterruption();
// fetching the next hyperlink // fetching the next hyperlink
nextEntry = (Map.Entry) i.next(); nextEntry = i.next();
nextUrlString = (String) nextEntry.getKey(); nextUrl = nextEntry.getKey();
try {
nextUrl = new yacyURL(nextUrlString, null);
// enqueue the hyperlink into the pre-notice-url db // enqueue the hyperlink into the pre-notice-url db
crawlStacker.enqueueEntry(nextUrl, entry.urlHash(), initiatorPeerHash, (String) nextEntry.getValue(), docDate, entry.depth() + 1, entry.profile()); crawlStacker.enqueueEntry(nextUrl, entry.urlHash(), initiatorPeerHash, nextEntry.getValue(), docDate, entry.depth() + 1, entry.profile());
} catch (MalformedURLException e1) {}
} }
log.logInfo("CRAWL: ADDED " + hl.size() + " LINKS FROM " + entry.url().toNormalform(false, true) + log.logInfo("CRAWL: ADDED " + hl.size() + " LINKS FROM " + entry.url().toNormalform(false, true) +
", NEW CRAWL STACK SIZE IS " + crawlQueues.noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_CORE)); ", NEW CRAWL STACK SIZE IS " + crawlQueues.noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_CORE));
@ -2210,7 +2205,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
/* ========================================================================= /* =========================================================================
* CREATE INDEX * CREATE INDEX
* ========================================================================= */ * ========================================================================= */
String docDescription = document.getTitle(); String docDescription = document.dc_title();
yacyURL referrerURL = entry.referrerURL(); yacyURL referrerURL = entry.referrerURL();
String noIndexReason = plasmaCrawlEURL.DENIED_UNSPECIFIED_INDEXING_ERROR; String noIndexReason = plasmaCrawlEURL.DENIED_UNSPECIFIED_INDEXING_ERROR;
@ -2242,8 +2237,8 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
indexURLEntry newEntry = new indexURLEntry( indexURLEntry newEntry = new indexURLEntry(
entry.url(), // URL entry.url(), // URL
docDescription, // document description docDescription, // document description
document.getAuthor(), // author document.dc_creator(), // author
document.getKeywords(' '), // tags document.dc_subject(' '), // tags
"", // ETag "", // ETag
docDate, // modification date docDate, // modification date
new Date(), // loaded date new Date(), // loaded date
@ -2252,7 +2247,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
new byte[0], // md5 new byte[0], // md5
(int) entry.size(), // size (int) entry.size(), // size
condenser.RESULT_NUMB_WORDS, // word count condenser.RESULT_NUMB_WORDS, // word count
plasmaHTCache.docType(document.getMimeType()), // doctype plasmaHTCache.docType(document.dc_format()), // doctype
condenser.RESULT_FLAGS, // flags condenser.RESULT_FLAGS, // flags
yacyURL.language(entry.url()), // language yacyURL.language(entry.url()), // language
ioLinks[0].intValue(), // llocal ioLinks[0].intValue(), // llocal
@ -2314,7 +2309,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
document, // document content document, // document content
condenser, // document condenser condenser, // document condenser
yacyURL.language(entry.url()), // document language yacyURL.language(entry.url()), // document language
plasmaHTCache.docType(document.getMimeType()),// document type plasmaHTCache.docType(document.dc_format()),// document type
ioLinks[0].intValue(), // outlinkSame ioLinks[0].intValue(), // outlinkSame
ioLinks[1].intValue() // outlinkOthers ioLinks[1].intValue() // outlinkOthers
); );
@ -2322,31 +2317,31 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
/* ======================================================================== /* ========================================================================
* SEND PAGE INDEX TO STORAGE PEER * SEND PAGE INDEX TO STORAGE PEER
* ======================================================================== */ * ======================================================================== */
HashMap urlCache = new HashMap(1); HashMap<String, indexURLEntry> urlCache = new HashMap<String, indexURLEntry>(1);
urlCache.put(newEntry.hash(), newEntry); urlCache.put(newEntry.hash(), newEntry);
ArrayList tmpContainers = new ArrayList(condenser.words().size()); ArrayList<indexContainer> tmpContainers = new ArrayList<indexContainer>(condenser.words().size());
String language = yacyURL.language(entry.url()); String language = yacyURL.language(entry.url());
char doctype = plasmaHTCache.docType(document.getMimeType()); char doctype = plasmaHTCache.docType(document.dc_format());
indexURLEntry.Components comp = newEntry.comp(); indexURLEntry.Components comp = newEntry.comp();
int urlLength = comp.url().toNormalform(true, true).length(); int urlLength = comp.url().toNormalform(true, true).length();
int urlComps = htmlFilterContentScraper.urlComps(comp.url().toNormalform(true, true)).length; int urlComps = htmlFilterContentScraper.urlComps(comp.url().toNormalform(true, true)).length;
// iterate over all words // iterate over all words
Iterator i = condenser.words().entrySet().iterator(); Iterator<Map.Entry<String, wordStatProp>> i = condenser.words().entrySet().iterator();
Map.Entry wentry; Map.Entry<String, wordStatProp> wentry;
plasmaCondenser.wordStatProp wordStat; plasmaCondenser.wordStatProp wordStat;
while (i.hasNext()) { while (i.hasNext()) {
wentry = (Map.Entry) i.next(); wentry = i.next();
String word = (String) wentry.getKey(); String word = wentry.getKey();
wordStat = (plasmaCondenser.wordStatProp) wentry.getValue(); wordStat = wentry.getValue();
String wordHash = plasmaCondenser.word2hash(word); String wordHash = plasmaCondenser.word2hash(word);
indexRWIEntry wordIdxEntry = new indexRWIRowEntry( indexRWIEntry wordIdxEntry = new indexRWIRowEntry(
urlHash, urlHash,
urlLength, urlComps, urlLength, urlComps,
wordStat.count, wordStat.count,
document.getTitle().length(), document.dc_title().length(),
condenser.words().size(), condenser.words().size(),
condenser.sentences().size(), condenser.sentences().size(),
wordStat.posInText, wordStat.posInText,
@ -2371,7 +2366,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
// transfering the index to the storage peer // transfering the index to the storage peer
indexContainer[] indexData = (indexContainer[]) tmpContainers.toArray(new indexContainer[tmpContainers.size()]); indexContainer[] indexData = (indexContainer[]) tmpContainers.toArray(new indexContainer[tmpContainers.size()]);
HashMap resultObj = yacyClient.transferIndex( HashMap<String, Object> resultObj = yacyClient.transferIndex(
seed, // target seed seed, // target seed
indexData, // word index data indexData, // word index data
urlCache, // urls urlCache, // urls
@ -2392,7 +2387,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
document, document,
condenser, condenser,
yacyURL.language(entry.url()), yacyURL.language(entry.url()),
plasmaHTCache.docType(document.getMimeType()), plasmaHTCache.docType(document.dc_format()),
ioLinks[0].intValue(), ioLinks[0].intValue(),
ioLinks[1].intValue() ioLinks[1].intValue()
); );
@ -2412,7 +2407,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
log.logInfo("*Indexed " + words + " words in URL " + entry.url() + log.logInfo("*Indexed " + words + " words in URL " + entry.url() +
" [" + entry.urlHash() + "]" + " [" + entry.urlHash() + "]" +
"\n\tDescription: " + docDescription + "\n\tDescription: " + docDescription +
"\n\tMimeType: " + document.getMimeType() + " | Charset: " + document.getCharset() + " | " + "\n\tMimeType: " + document.dc_format() + " | Charset: " + document.getCharset() + " | " +
"Size: " + document.getTextLength() + " bytes | " + "Size: " + document.getTextLength() + " bytes | " +
"Anchors: " + ((document.getAnchors() == null) ? 0 : document.getAnchors().size()) + "Anchors: " + ((document.getAnchors() == null) ? 0 : document.getAnchors().size()) +
"\n\tStackingTime: " + (stackEndTime-stackStartTime) + " ms | " + "\n\tStackingTime: " + (stackEndTime-stackStartTime) + " ms | " +
@ -2548,7 +2543,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
plasmaParserDocument document = plasmaSnippetCache.parseDocument(comp.url(), resourceContentLength.longValue(), resourceContent); plasmaParserDocument document = plasmaSnippetCache.parseDocument(comp.url(), resourceContentLength.longValue(), resourceContent);
// get the word set // get the word set
Set words = null; Set<String> words = null;
try { try {
words = new plasmaCondenser(document, true, true).words().keySet(); words = new plasmaCondenser(document, true, true).words().keySet();
} catch (UnsupportedEncodingException e) { } catch (UnsupportedEncodingException e) {
@ -2642,10 +2637,10 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
} }
public static int accessFrequency(HashMap tracker, String host) { public static int accessFrequency(HashMap<String, TreeSet<Long>> tracker, String host) {
// returns the access frequency in queries per hour for a given host and a specific tracker // returns the access frequency in queries per hour for a given host and a specific tracker
long timeInterval = 1000 * 60 * 60; long timeInterval = 1000 * 60 * 60;
TreeSet accessSet = (TreeSet) tracker.get(host); TreeSet<Long> accessSet = tracker.get(host);
if (accessSet == null) return 0; if (accessSet == null) return 0;
return accessSet.tailSet(new Long(System.currentTimeMillis() - timeInterval)).size(); return accessSet.tailSet(new Long(System.currentTimeMillis() - timeInterval)).size();
} }
@ -2769,7 +2764,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
try { try {
// find a list of DHT-peers // find a list of DHT-peers
double maxDist = 0.2; double maxDist = 0.2;
ArrayList seeds = yacyCore.dhtAgent.getDHTTargets(log, peerCount, Math.min(8, (int) (yacyCore.seedDB.sizeConnected() * maxDist)), dhtChunk.firstContainer().getWordHash(), dhtChunk.lastContainer().getWordHash(), maxDist); ArrayList<yacySeed> seeds = yacyCore.dhtAgent.getDHTTargets(log, peerCount, Math.min(8, (int) (yacyCore.seedDB.sizeConnected() * maxDist)), dhtChunk.firstContainer().getWordHash(), dhtChunk.lastContainer().getWordHash(), maxDist);
if (seeds.size() < peerCount) { if (seeds.size() < peerCount) {
log.logWarning("found not enough (" + seeds.size() + ") peers for distribution for dhtchunk [" + dhtChunk.firstContainer().getWordHash() + " .. " + dhtChunk.lastContainer().getWordHash() + "]"); log.logWarning("found not enough (" + seeds.size() + ") peers for distribution for dhtchunk [" + dhtChunk.firstContainer().getWordHash() + " .. " + dhtChunk.lastContainer().getWordHash() + "]");
return false; return false;
@ -2784,8 +2779,8 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
int retries = 0; int retries = 0;
// starting up multiple DHT transfer threads // starting up multiple DHT transfer threads
Iterator seedIter = seeds.iterator(); Iterator<yacySeed> seedIter = seeds.iterator();
ArrayList transfer = new ArrayList(peerCount); ArrayList<plasmaDHTTransfer> transfer = new ArrayList<plasmaDHTTransfer>(peerCount);
while (hc1 < peerCount && (transfer.size() > 0 || seedIter.hasNext())) { while (hc1 < peerCount && (transfer.size() > 0 || seedIter.hasNext())) {
// starting up some transfer threads // starting up some transfer threads
@ -2804,12 +2799,12 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
} }
// waiting for the transfer threads to finish // waiting for the transfer threads to finish
Iterator transferIter = transfer.iterator(); Iterator<plasmaDHTTransfer> transferIter = transfer.iterator();
while (transferIter.hasNext()) { while (transferIter.hasNext()) {
// check for interruption // check for interruption
checkInterruption(); checkInterruption();
plasmaDHTTransfer t = (plasmaDHTTransfer)transferIter.next(); plasmaDHTTransfer t = transferIter.next();
if (!t.isAlive()) { if (!t.isAlive()) {
// remove finished thread from the list // remove finished thread from the list
transferIter.remove(); transferIter.remove();

13
source/de/anomic/plasma/plasmaWebStructure.java
Unescape View File

@ -29,14 +29,13 @@ package de.anomic.plasma;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.net.MalformedURLException;
import java.util.ConcurrentModificationException; import java.util.ConcurrentModificationException;
import java.util.Date; import java.util.Date;
import java.util.HashMap;
import java.util.Iterator; import java.util.Iterator;
import java.util.Map; import java.util.Map;
import java.util.HashMap;
import java.util.TreeMap;
import java.util.SortedMap; import java.util.SortedMap;
import java.util.TreeMap;
import java.util.TreeSet; import java.util.TreeSet;
import de.anomic.kelondro.kelondroBase64Order; import de.anomic.kelondro.kelondroBase64Order;
@ -97,8 +96,8 @@ public class plasmaWebStructure {
assert url.hash().equals(baseurlhash); assert url.hash().equals(baseurlhash);
// generate citation reference // generate citation reference
Map<String, String> hl = document.getHyperlinks(); Map<yacyURL, String> hl = document.getHyperlinks();
Iterator<String> it = hl.keySet().iterator(); Iterator<yacyURL> it = hl.keySet().iterator();
String nexturlhash; String nexturlhash;
StringBuffer cpg = new StringBuffer(12 * (hl.size() + 1) + 1); StringBuffer cpg = new StringBuffer(12 * (hl.size() + 1) + 1);
StringBuffer cpl = new StringBuffer(12 * (hl.size() + 1) + 1); StringBuffer cpl = new StringBuffer(12 * (hl.size() + 1) + 1);
@ -106,8 +105,7 @@ public class plasmaWebStructure {
int GCount = 0; int GCount = 0;
int LCount = 0; int LCount = 0;
while (it.hasNext()) { while (it.hasNext()) {
try { nexturlhash = it.next().hash();
nexturlhash = (new yacyURL(it.next(), null)).hash();
if (nexturlhash != null) { if (nexturlhash != null) {
if (nexturlhash.substring(6).equals(lhp)) { if (nexturlhash.substring(6).equals(lhp)) {
// this is a inbound link // this is a inbound link
@ -119,7 +117,6 @@ public class plasmaWebStructure {
GCount++; GCount++;
} }
} }
} catch (MalformedURLException e) {}
} }
// append this reference to buffer // append this reference to buffer

2
source/de/anomic/plasma/plasmaWordIndex.java
Unescape View File

@ -301,7 +301,7 @@ public final class plasmaWordIndex implements indexRI {
wprop = wentry.getValue(); wprop = wentry.getValue();
assert (wprop.flags != null); assert (wprop.flags != null);
ientry = new indexRWIRowEntry(url.hash(), ientry = new indexRWIRowEntry(url.hash(),
urlLength, urlComps, (document == null) ? urlLength : document.getTitle().length(), urlLength, urlComps, (document == null) ? urlLength : document.dc_title().length(),
wprop.count, wprop.count,
condenser.words().size(), condenser.words().size(),
condenser.sentences().size(), condenser.sentences().size(),

43
source/de/anomic/yacy/yacyClient.java
Unescape View File

@ -55,6 +55,7 @@ import de.anomic.http.httpRemoteProxyConfig;
import de.anomic.http.httpc; import de.anomic.http.httpc;
import de.anomic.index.indexContainer; import de.anomic.index.indexContainer;
import de.anomic.index.indexRWIEntry; import de.anomic.index.indexRWIEntry;
import de.anomic.index.indexRWIRowEntry;
import de.anomic.index.indexURLEntry; import de.anomic.index.indexURLEntry;
import de.anomic.kelondro.kelondroBase64Order; import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroBitfield; import de.anomic.kelondro.kelondroBitfield;
@ -575,19 +576,19 @@ public final class yacyClient {
// read index abstract // read index abstract
if (abstractCache != null) { if (abstractCache != null) {
Iterator i = result.entrySet().iterator(); Iterator<Map.Entry<String, String>> i = result.entrySet().iterator();
Map.Entry entry; Map.Entry<String, String> entry;
TreeMap singleAbstract; TreeMap<String, String> singleAbstract;
String wordhash; String wordhash;
serverByteBuffer ci; serverByteBuffer ci;
while (i.hasNext()) { while (i.hasNext()) {
entry = (Map.Entry) i.next(); entry = i.next();
if (((String) entry.getKey()).startsWith("indexabstract.")) { if (entry.getKey().startsWith("indexabstract.")) {
wordhash = ((String) entry.getKey()).substring(14); wordhash = entry.getKey().substring(14);
synchronized (abstractCache) { synchronized (abstractCache) {
singleAbstract = (TreeMap<String, String>) abstractCache.get(wordhash); // a mapping from url-hashes to a string of peer-hashes singleAbstract = (TreeMap<String, String>) abstractCache.get(wordhash); // a mapping from url-hashes to a string of peer-hashes
if (singleAbstract == null) singleAbstract = new TreeMap(); if (singleAbstract == null) singleAbstract = new TreeMap<String, String>();
ci = new serverByteBuffer(((String) entry.getValue()).getBytes()); ci = new serverByteBuffer(entry.getValue().getBytes());
//System.out.println("DEBUG-ABSTRACTFETCH: for word hash " + wordhash + " received " + ci.toString()); //System.out.println("DEBUG-ABSTRACTFETCH: for word hash " + wordhash + " received " + ci.toString());
indexContainer.decompressIndex(singleAbstract, ci, target.hash); indexContainer.decompressIndex(singleAbstract, ci, target.hash);
abstractCache.put(wordhash, singleAbstract); abstractCache.put(wordhash, singleAbstract);
@ -621,7 +622,7 @@ public final class yacyClient {
return urls; return urls;
} }
public static HashMap permissionMessage(String targetHash) { public static HashMap<String, String> permissionMessage(String targetHash) {
// ask for allowed message size and attachement size // ask for allowed message size and attachement size
// if this replies null, the peer does not answer // if this replies null, the peer does not answer
if (yacyCore.seedDB == null || yacyCore.seedDB.mySeed() == null) { return null; } if (yacyCore.seedDB == null || yacyCore.seedDB.mySeed() == null) { return null; }
@ -651,7 +652,7 @@ public final class yacyClient {
} }
} }
public static HashMap postMessage(String targetHash, String subject, byte[] message) { public static HashMap<String, String> postMessage(String targetHash, String subject, byte[] message) {
// this post a message to the remote message board // this post a message to the remote message board
// prepare request // prepare request
@ -699,7 +700,7 @@ public final class yacyClient {
return address; return address;
} }
public static HashMap transferPermission(String targetAddress, long filesize, String filename) { public static HashMap<String, String> transferPermission(String targetAddress, long filesize, String filename) {
// prepare request // prepare request
final serverObjects post = yacyNetwork.basicRequestPost(plasmaSwitchboard.getSwitchboard(), null); final serverObjects post = yacyNetwork.basicRequestPost(plasmaSwitchboard.getSwitchboard(), null);
@ -731,7 +732,7 @@ public final class yacyClient {
} }
} }
public static HashMap transferStore(String targetAddress, String access, String filename, byte[] file) { public static HashMap<String, String> transferStore(String targetAddress, String access, String filename, byte[] file) {
// prepare request // prepare request
final serverObjects post = yacyNetwork.basicRequestPost(plasmaSwitchboard.getSwitchboard(), null); final serverObjects post = yacyNetwork.basicRequestPost(plasmaSwitchboard.getSwitchboard(), null);
@ -741,7 +742,7 @@ public final class yacyClient {
post.put("filesize", Long.toString(file.length)); post.put("filesize", Long.toString(file.length));
post.put("md5", serverCodings.encodeMD5Hex(file)); post.put("md5", serverCodings.encodeMD5Hex(file));
post.put("access", access); post.put("access", access);
HashMap files = new HashMap(); HashMap<String, byte[]> files = new HashMap<String, byte[]>();
files.put("filename", file); files.put("filename", file);
// send request // send request
@ -766,7 +767,7 @@ public final class yacyClient {
} }
public static String transfer(String targetAddress, String filename, byte[] file) { public static String transfer(String targetAddress, String filename, byte[] file) {
HashMap phase1 = transferPermission(targetAddress, file.length, filename); HashMap<String, String> phase1 = transferPermission(targetAddress, file.length, filename);
if (phase1 == null) return "no connection to remote address " + targetAddress + "; phase 1"; if (phase1 == null) return "no connection to remote address " + targetAddress + "; phase 1";
String access = (String) phase1.get("access"); String access = (String) phase1.get("access");
String nextaddress = (String) phase1.get("address"); String nextaddress = (String) phase1.get("address");
@ -778,7 +779,7 @@ public final class yacyClient {
if (!(response.equals("ok"))) return "remote peer rejected transfer: " + response; if (!(response.equals("ok"))) return "remote peer rejected transfer: " + response;
String accesscode = serverCodings.encodeMD5Hex(kelondroBase64Order.standardCoder.encodeString(access)); String accesscode = serverCodings.encodeMD5Hex(kelondroBase64Order.standardCoder.encodeString(access));
if (protocol.equals("http")) { if (protocol.equals("http")) {
HashMap phase2 = transferStore(nextaddress, accesscode, filename, file); HashMap<String, String> phase2 = transferStore(nextaddress, accesscode, filename, file);
if (phase2 == null) return "no connection to remote address " + targetAddress + "; phase 2"; if (phase2 == null) return "no connection to remote address " + targetAddress + "; phase 2";
response = (String) phase2.get("response"); response = (String) phase2.get("response");
if (response == null) return "wrong return values from other peer; phase 2"; if (response == null) return "wrong return values from other peer; phase 2";
@ -848,14 +849,14 @@ public final class yacyClient {
} }
} }
public static HashMap transferIndex(yacySeed targetSeed, indexContainer[] indexes, HashMap urlCache, boolean gzipBody, int timeout) { public static HashMap<String, Object> transferIndex(yacySeed targetSeed, indexContainer[] indexes, HashMap<String, indexURLEntry> urlCache, boolean gzipBody, int timeout) {
HashMap resultObj = new HashMap(); HashMap<String, Object> resultObj = new HashMap<String, Object>();
int payloadSize = 0; int payloadSize = 0;
try { try {
// check if we got all necessary urls in the urlCache (only for debugging) // check if we got all necessary urls in the urlCache (only for debugging)
Iterator eenum; Iterator<indexRWIRowEntry> eenum;
indexRWIEntry entry; indexRWIEntry entry;
for (int i = 0; i < indexes.length; i++) { for (int i = 0; i < indexes.length; i++) {
eenum = indexes[i].entries(); eenum = indexes[i].entries();
@ -938,7 +939,7 @@ public final class yacyClient {
} }
} }
private static HashMap transferRWI(yacySeed targetSeed, indexContainer[] indexes, boolean gzipBody, int timeout) { private static HashMap<String, String> transferRWI(yacySeed targetSeed, indexContainer[] indexes, boolean gzipBody, int timeout) {
final String address = targetSeed.getPublicAddress(); final String address = targetSeed.getPublicAddress();
if (address == null) { return null; } if (address == null) { return null; }
@ -953,7 +954,7 @@ public final class yacyClient {
int indexcount = 0; int indexcount = 0;
final StringBuffer entrypost = new StringBuffer(indexes.length*73); final StringBuffer entrypost = new StringBuffer(indexes.length*73);
Iterator eenum; Iterator<indexRWIRowEntry> eenum;
indexRWIEntry entry; indexRWIEntry entry;
for (int i = 0; i < indexes.length; i++) { for (int i = 0; i < indexes.length; i++) {
eenum = indexes[i].entries(); eenum = indexes[i].entries();
@ -968,7 +969,7 @@ public final class yacyClient {
if (indexcount == 0) { if (indexcount == 0) {
// nothing to do but everything ok // nothing to do but everything ok
final HashMap result = new HashMap(2); final HashMap<String, String> result = new HashMap<String, String>(2);
result.put("result", "ok"); result.put("result", "ok");
result.put("unknownURL", ""); result.put("unknownURL", "");
return result; return result;

2
source/de/anomic/yacy/yacyURL.java
Unescape View File

@ -891,7 +891,7 @@ public class yacyURL {
} }
public int hashCode() { public int hashCode() {
return this.toString().hashCode(); return this.hash().hashCode();
} }
public int compareTo(Object h) { public int compareTo(Object h) {

11
source/de/anomic/yacy/yacyVersion.java
Unescape View File

@ -29,7 +29,6 @@ package de.anomic.yacy;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.net.MalformedURLException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Comparator; import java.util.Comparator;
import java.util.HashMap; import java.util.HashMap;
@ -298,17 +297,13 @@ public final class yacyVersion implements Comparator<yacyVersion>, Comparable<ya
} }
// analyse links in scraper resource, and find link to latest release in it // analyse links in scraper resource, and find link to latest release in it
Map<String, String> anchors = scraper.getAnchors(); // a url (String) / name (String) relation Map<yacyURL, String> anchors = scraper.getAnchors(); // a url (String) / name (String) relation
Iterator<String> i = anchors.keySet().iterator(); Iterator<yacyURL> i = anchors.keySet().iterator();
TreeSet<yacyVersion> devreleases = new TreeSet<yacyVersion>(); TreeSet<yacyVersion> devreleases = new TreeSet<yacyVersion>();
TreeSet<yacyVersion> mainreleases = new TreeSet<yacyVersion>(); TreeSet<yacyVersion> mainreleases = new TreeSet<yacyVersion>();
yacyVersion release; yacyVersion release;
while (i.hasNext()) { while (i.hasNext()) {
try { url = i.next();
url = new yacyURL((String) i.next(), null);
} catch (MalformedURLException e1) {
continue; // just ignore invalid urls
}
try { try {
release = new yacyVersion(url); release = new yacyVersion(url);
//System.out.println("r " + release.toAnchor()); //System.out.println("r " + release.toAnchor());

Write Preview
Loading…
Cancel
Save