diff --git a/htroot/api/ymarks/get_metadata.java b/htroot/api/ymarks/get_metadata.java index 053c83668..702804e69 100644 --- a/htroot/api/ymarks/get_metadata.java +++ b/htroot/api/ymarks/get_metadata.java @@ -1,6 +1,8 @@ import java.io.IOException; import java.net.MalformedURLException; import java.util.EnumMap; +import java.util.Iterator; + import net.yacy.cora.protocol.RequestHeader; import net.yacy.document.Document; import net.yacy.document.Parser.Failure; @@ -8,6 +10,7 @@ import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.search.Switchboard; import de.anomic.data.UserDB; import de.anomic.data.ymark.YMarkAutoTagger; +import de.anomic.data.ymark.YMarkCrawlStart; import de.anomic.data.ymark.YMarkEntry; import de.anomic.data.ymark.YMarkMetadata; import de.anomic.data.ymark.YMarkTables; @@ -27,18 +30,43 @@ public class get_metadata { final boolean isAuthUser = user!= null && user.hasRight(UserDB.AccessRight.BOOKMARK_RIGHT); if(isAdmin || isAuthUser) { + final String bmk_user = (isAuthUser ? user.getUserName() : YMarkTables.USER_ADMIN); - try { - final String url = post.get(YMarkEntry.BOOKMARK.URL.key()); + + + String url = post.get(YMarkEntry.BOOKMARK.URL.key(),YMarkEntry.BOOKMARK.URL.deflt()); + boolean hasProtocol = false; + for (YMarkTables.PROTOCOLS p : YMarkTables.PROTOCOLS.values()) { + if(url.toLowerCase().startsWith(p.protocol())) { + hasProtocol = true; + break; + } + } + if (!hasProtocol) { + url=YMarkTables.PROTOCOLS.HTTP.protocol(url); + } + + try { YMarkMetadata meta = new YMarkMetadata(new DigestURI(url), sb.indexSegments); final Document document = meta.loadDocument(sb.loader); final EnumMap metadata = meta.loadMetadata(); prop.putXML("title", metadata.get(YMarkMetadata.METADATA.TITLE)); - prop.putXML("desc", metadata.get(YMarkMetadata.METADATA.DESCRIPTION)); - + prop.putXML("desc", metadata.get(YMarkMetadata.METADATA.DESCRIPTION)); prop.put("keywords", putTags(document.dc_subject(','), "keywords")); prop.put("autotags", putTags(YMarkAutoTagger.autoTag(document, 5, sb.tables.bookmarks.getTags(bmk_user)), "autotags")); + + final YMarkCrawlStart crawlStart = new YMarkCrawlStart(sb.tables, url); + final Iterator iter = crawlStart.keySet().iterator(); + int count = 0; + String key; + while(iter.hasNext()) { + key = iter.next(); + prop.putXML("crawlstart_"+count+"_key",key.toLowerCase()); + prop.putXML("crawlstart_"+count+"_value",crawlStart.get(key)); + count++; + } + prop.put("crawlstart", count); } catch (MalformedURLException e1) { // TODO Auto-generated catch block diff --git a/htroot/api/ymarks/get_metadata.xml b/htroot/api/ymarks/get_metadata.xml index 1cab22a98..adbb9a3b7 100644 --- a/htroot/api/ymarks/get_metadata.xml +++ b/htroot/api/ymarks/get_metadata.xml @@ -10,4 +10,9 @@ #{autotags}# #{/autotags}# + + \ No newline at end of file diff --git a/htroot/api/ymarks/get_treeview.java b/htroot/api/ymarks/get_treeview.java index 1bb453e4d..a44d97331 100644 --- a/htroot/api/ymarks/get_treeview.java +++ b/htroot/api/ymarks/get_treeview.java @@ -1,6 +1,5 @@ import java.io.IOException; import java.net.MalformedURLException; -import java.util.ArrayList; import java.util.Date; import java.util.EnumMap; import java.util.Iterator; @@ -9,18 +8,20 @@ import java.util.TreeMap; import net.yacy.cora.date.ISO8601Formatter; import net.yacy.cora.document.UTF8; import net.yacy.cora.protocol.RequestHeader; +import net.yacy.document.Document; import net.yacy.document.Parser.Failure; import net.yacy.kelondro.blob.Tables; import net.yacy.kelondro.data.meta.DigestURI; -import net.yacy.kelondro.data.word.Word; import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.logging.Log; import net.yacy.search.Switchboard; import de.anomic.data.UserDB; +import de.anomic.data.ymark.YMarkAutoTagger; import de.anomic.data.ymark.YMarkCrawlStart; import de.anomic.data.ymark.YMarkEntry; import de.anomic.data.ymark.YMarkMetadata; import de.anomic.data.ymark.YMarkTables; +import de.anomic.data.ymark.YMarkTag; import de.anomic.data.ymark.YMarkUtil; import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; @@ -50,7 +51,7 @@ public class get_treeview { boolean isMetadata = false; boolean isURLdb = false; boolean isCrawlStart = false; - boolean isWordCount = false; + boolean isAutoTagger = false; boolean displayBmk = false; if (post != null){ @@ -73,7 +74,7 @@ public class get_treeview { isURLdb = true; isFolder = false; } else if (post.get(ROOT).startsWith("w:")) { - isWordCount = true; + isAutoTagger = true; isFolder = false; } else if (post.get(ROOT).startsWith("c:")) { isCrawlStart = true; @@ -192,7 +193,7 @@ public class get_treeview { prop.put("folders_"+count+"_hash", "c:"+url); prop.put("folders_"+count+"_hasChildren", "true"); count++; - prop.put("folders_"+count+"_foldername","WordCounts"); + prop.put("folders_"+count+"_foldername","AutoTagger"); putProp(count, "meta"); prop.put("folders_"+count+"_hash", "w:"+url); prop.put("folders_"+count+"_hasChildren", "true"); @@ -205,23 +206,21 @@ public class get_treeview { } catch (RowSpaceExceededException e) { Log.logException(e); } - } else if (isWordCount || isMetadata || isURLdb || isCrawlStart) { + } else if (isAutoTagger || isMetadata || isURLdb || isCrawlStart) { try { final YMarkMetadata meta = new YMarkMetadata(new DigestURI(post.get(ROOT).substring(2)), sb.indexSegments); - meta.loadDocument(sb.loader); - if(isWordCount) { - final TreeMap words = meta.getWordCounts(); - final ArrayList topwords = new ArrayList(words.descendingKeySet()); - for(int i = 0; i < 20 && i < topwords.size(); i++) { - String word = topwords.get(i); - int occur = words.get(word).occurrences(); - prop.put("folders_"+count+"_foldername",""+word+": [" + occur + "]"); - putProp(count, "meta"); - count++; - } - count--; - prop.put("folders_"+count+"_comma", ""); + final Document document = meta.loadDocument(sb.loader); + final TreeMap tags = sb.tables.bookmarks.getTags(bmk_user); + if(isAutoTagger) { + prop.put("folders_"+count+"_foldername","meta-"+YMarkMetadata.METADATA.KEYWORDS.name().toLowerCase()+": " + meta.loadMetadata().get(YMarkMetadata.METADATA.KEYWORDS) + ""); + putProp(count, "meta"); + count++; + prop.put("folders_"+count+"_foldername","with preference: "+YMarkAutoTagger.autoTag(document, 4, tags)+""); + putProp(count, "meta"); count++; + prop.put("folders_"+count+"_foldername","without preference: "+YMarkAutoTagger.autoTag(document, 4, new TreeMap())+""); + putProp(count, "meta"); + count++; prop.put("folders", count); } else if(isMetadata) { count = putMeta(count, meta.loadMetadata()); diff --git a/source/de/anomic/data/ymark/TablesRowComparator.java b/source/de/anomic/data/ymark/TablesRowComparator.java index fe4a8f829..570193273 100644 --- a/source/de/anomic/data/ymark/TablesRowComparator.java +++ b/source/de/anomic/data/ymark/TablesRowComparator.java @@ -6,9 +6,14 @@ import net.yacy.kelondro.blob.Tables; public class TablesRowComparator implements Comparator { private String sortname; + private boolean desc; - public TablesRowComparator(final String sortname) { + public TablesRowComparator(final String sortname, final String sortorder) { setSortName(sortname); + if(sortorder.equals("desc")) + this.desc = true; + else + this.desc = false; } public void setSortName(final String sortname) { @@ -20,7 +25,10 @@ public class TablesRowComparator implements Comparator { if(row0.containsKey(this.sortname) && row1.containsKey(this.sortname)) { String name1 = UTF8.String(row0.get(this.sortname)).toLowerCase(); String name2 = UTF8.String(row1.get(this.sortname)).toLowerCase(); - return name1.compareTo(name2); + if(desc) + return name2.compareTo(name1); + else + return name1.compareTo(name2); } } return 0; diff --git a/source/de/anomic/data/ymark/YMarkAutoTagger.java b/source/de/anomic/data/ymark/YMarkAutoTagger.java index 8012aee88..0261d7631 100644 --- a/source/de/anomic/data/ymark/YMarkAutoTagger.java +++ b/source/de/anomic/data/ymark/YMarkAutoTagger.java @@ -29,7 +29,8 @@ public class YMarkAutoTagger implements Runnable, Thread.UncaughtExceptionHandle public final static String SPACE = " "; public final static String POISON = ""; - public final static HashSet stopwords = new HashSet(Arrays.asList(".", "!", "?", "nbsp", "uuml", "ouml", "auml", "amp", "quot", "laquo", "raquo")); + public final static HashSet stopwords = new HashSet(Arrays.asList(".", "!", "?", "nbsp", "uuml", "ouml", "auml", "amp", "quot", "laquo", "raquo", + "and", "with", "the", "gt", "lt")); private final ArrayBlockingQueue bmkQueue; @@ -90,35 +91,40 @@ public class YMarkAutoTagger implements Runnable, Thread.UncaughtExceptionHandle // generate potential tags from document title, description and subject final int bufferSize = document.dc_title().length() + document.dc_description().length() + document.dc_subject(' ').length() + 32; final StringBuilder buffer = new StringBuilder(bufferSize); + final StringBuilder pwords = new StringBuilder(1000); buffer.append(document.dc_title().toLowerCase()); buffer.append(document.dc_description().toLowerCase()); buffer.append(document.dc_subject(' ').toLowerCase()); final Enumeration tokens = new WordTokenizer(new ByteArrayInputStream(UTF8.getBytes(buffer.toString())), LibraryProvider.dymLib); - int count = 0; + int score = 0; // get phrases final TreeMap phrases = getPhrases(document, 2); phrases.putAll(getPhrases(document, 3)); - phrases.putAll(getPhrases(document, 4)); final Iterator iter = phrases.keySet().iterator(); while(iter.hasNext()) { - count = 10; + score = 10; final String phrase = iter.next(); if(phrases.get(phrase).size() > 3 && phrases.get(phrase).size() < 10) { - count = phrases.get(phrase).size() * phrase.split(" ").length * 35; + score = phrases.get(phrase).size() * phrase.split(" ").length * 20; } if(isDigitSpace(phrase)) { - count = 10; + score = 10; } if(phrases.get(phrase).size() > 2 && buffer.indexOf(phrase) > 1) { - count = count * 10; - } - topwords.add(new YMarkTag(phrase, count)); + score = score * 10; + } + if (tags.containsKey(phrase)) { + score = score * 20; + } + topwords.add(new YMarkTag(phrase, score)); + pwords.append(phrase); + pwords.append(' '); } - + // loop through potential tag and rank them while(tokens.hasMoreElements()) { - count = 0; + score = 0; token = tokens.nextElement(); // check if the token appears in the text @@ -126,23 +132,27 @@ public class YMarkAutoTagger implements Runnable, Thread.UncaughtExceptionHandle final Word word = words.get(token.toString()); // token appears in text and matches an existing bookmark tag if (tags.containsKey(token.toString())) { - count = word.occurrences() * tags.get(token.toString()).size() * 200; + score = word.occurrences() * tags.get(token.toString()).size() * 200; } // token appears in text and has more than 3 characters else if (token.length()>3) { - count = word.occurrences() * 100; + score = word.occurrences() * 100; + } + // if token is already part of a phrase, reduce score + if(pwords.toString().indexOf(token.toString())>1) { + score = score / 3; } - topwords.add(new YMarkTag(token.toString(), count)); + topwords.add(new YMarkTag(token.toString(), score)); } } - count = 0; + score = 0; buffer.setLength(0); for(final YMarkTag tag : topwords) { - if(count < max) { + if(score < max) { if(tag.size() > 100) { buffer.append(tag.name()); buffer.append(YMarkUtil.TAGS_SEPARATOR); - count++; + score++; } } else { break; @@ -165,7 +175,7 @@ public class YMarkAutoTagger implements Runnable, Thread.UncaughtExceptionHandle while(tokens.hasMoreElements()) { token = tokens.nextElement(); - if(stopwords.contains(token.toString())) + if(stopwords.contains(token.toString()) || isDigitSpace(token.toString())) continue; // if we have a full phrase, delete the first token diff --git a/source/de/anomic/data/ymark/YMarkCrawlStart.java b/source/de/anomic/data/ymark/YMarkCrawlStart.java index 7aeb4aea0..8315798e4 100644 --- a/source/de/anomic/data/ymark/YMarkCrawlStart.java +++ b/source/de/anomic/data/ymark/YMarkCrawlStart.java @@ -52,11 +52,13 @@ public class YMarkCrawlStart extends HashMap{ public void load(final String url) { try { final StringBuilder buffer = new StringBuilder(500); - buffer.append("^.*crawlingURL=\\Q"); + //buffer.append("^.*crawlingURL=\\Q"); + buffer.append("^crawl start for \\Q"); buffer.append(url); buffer.append("\\E?.*"); final Pattern pattern = Pattern.compile(buffer.toString()); - final Iterator APIcalls = this.worktables.iterator(WorkTables.TABLE_API_NAME, WorkTables.TABLE_API_COL_URL, pattern); + //final Iterator APIcalls = this.worktables.iterator(WorkTables.TABLE_API_NAME, WorkTables.TABLE_API_COL_URL, pattern); + final Iterator APIcalls = this.worktables.iterator(WorkTables.TABLE_API_NAME, WorkTables.TABLE_API_COL_COMMENT, pattern); Tables.Row row = null; while(APIcalls.hasNext()) { row = APIcalls.next(); diff --git a/source/de/anomic/data/ymark/YMarkMetadata.java b/source/de/anomic/data/ymark/YMarkMetadata.java index 1d5e46aa4..1d9cb9707 100644 --- a/source/de/anomic/data/ymark/YMarkMetadata.java +++ b/source/de/anomic/data/ymark/YMarkMetadata.java @@ -29,19 +29,14 @@ package de.anomic.data.ymark; import java.io.IOException; import java.net.MalformedURLException; import java.util.EnumMap; -import java.util.Map; -import java.util.TreeMap; import net.yacy.cora.date.ISO8601Formatter; import net.yacy.cora.document.UTF8; import net.yacy.cora.services.federated.yacy.CacheStrategy; -import net.yacy.document.Condenser; import net.yacy.document.Document; -import net.yacy.document.LibraryProvider; import net.yacy.document.Parser.Failure; import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.data.meta.URIMetadataRow; -import net.yacy.kelondro.data.word.Word; import net.yacy.repository.LoaderDispatcher; import net.yacy.search.index.Segments; import de.anomic.crawler.retrieval.Response; @@ -141,18 +136,4 @@ public class YMarkMetadata { } return metadata; } - - public TreeMap getWordCounts() { - if (this.document != null) { - return sortWordCounts(new Condenser(this.document, true, true, LibraryProvider.dymLib).words()); - } - return new TreeMap(); - } - - public static TreeMap sortWordCounts(final Map unsorted_words) { - final TreeMap sorted_words = new TreeMap(new YMarkWordCountComparator(unsorted_words)); - sorted_words.putAll(unsorted_words); - return sorted_words; - } - } diff --git a/source/de/anomic/data/ymark/YMarkTables.java b/source/de/anomic/data/ymark/YMarkTables.java index 6f8282b4e..0b4afd59c 100644 --- a/source/de/anomic/data/ymark/YMarkTables.java +++ b/source/de/anomic/data/ymark/YMarkTables.java @@ -27,9 +27,11 @@ package de.anomic.data.ymark; import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; import java.util.HashSet; import java.util.Iterator; -import java.util.SortedSet; +import java.util.List; import java.util.TreeMap; import java.util.TreeSet; import java.util.regex.Pattern; @@ -214,17 +216,16 @@ public class YMarkTables { return this.worktables.iterator(bmk_table, YMarkEntry.BOOKMARK.TAGS.key(), p); } - public SortedSet orderBookmarksBy(final Iterator rowIterator, final String sortname, final String sortorder) { - final TreeSet sortTree = new TreeSet(new TablesRowComparator(sortname)); + public List orderBookmarksBy(final Iterator rowIterator, final String sortname, final String sortorder) { + final List sortList = new ArrayList(); Row row; while (rowIterator.hasNext()) { row = rowIterator.next(); if(row != null) - sortTree.add(row); + sortList.add(row); } - if(sortorder.equals("desc")) - return sortTree.descendingSet(); - return sortTree; + Collections.sort(sortList, new TablesRowComparator(sortname, sortorder)); + return sortList; } public void addTags(final String bmk_user, final String url, final String tagString, final boolean merge) throws IOException, RowSpaceExceededException { diff --git a/source/de/anomic/data/ymark/YMarkWordCountComparator.java b/source/de/anomic/data/ymark/YMarkWordCountComparator.java deleted file mode 100644 index 8ba178369..000000000 --- a/source/de/anomic/data/ymark/YMarkWordCountComparator.java +++ /dev/null @@ -1,53 +0,0 @@ -// YMarkWordCountComparator.java -// (C) 2011 by Stefan Förster, sof@gmx.de, Norderstedt, Germany -// first published 2010 on http://yacy.net -// -// This is a part of YaCy, a peer-to-peer based web search engine -// -// $LastChangedDate$ -// $LastChangedRevision$ -// $LastChangedBy$ -// -// LICENSE -// -// This program is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - -package de.anomic.data.ymark; - -import java.util.Comparator; -import java.util.Map; - -import net.yacy.kelondro.data.word.Word; - -public class YMarkWordCountComparator implements Comparator { - - private Map words; - - public YMarkWordCountComparator(final Map words) { - this.words = words; - } - - public int compare(final String k1, final String k2) { - final Word w1 = this.words.get(k1); - final Word w2 = this.words.get(k2); - - if(w1.occurrences() > w2.occurrences()) - return 1; - else if(w1.occurrences() < w2.occurrences()) - return -1; - else - return 0; - } -}