From f035f257da8d43b855a3e057131e6b1d88fc1b4b Mon Sep 17 00:00:00 2001 From: apfelmaennchen Date: Mon, 18 Oct 2010 21:09:41 +0000 Subject: [PATCH] added some more bookmark code... git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7261 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- htroot/api/ymarks/add_ymark.java | 70 +++--- htroot/api/ymarks/delete_ymark.java | 22 +- htroot/api/ymarks/get_ymark.java | 30 +-- htroot/api/ymarks/import.html | 37 ++++ htroot/api/ymarks/import_html.java | 58 +++++ htroot/api/ymarks/import_html.xml | 2 + source/de/anomic/data/WorkTables.java | 63 +----- source/de/anomic/data/YMarkTables.java | 209 ++++++++++++++++++ source/de/anomic/data/YMarksHTMLImporter.java | 95 ++++++++ 9 files changed, 469 insertions(+), 117 deletions(-) create mode 100644 htroot/api/ymarks/import.html create mode 100644 htroot/api/ymarks/import_html.java create mode 100644 htroot/api/ymarks/import_html.xml create mode 100644 source/de/anomic/data/YMarkTables.java create mode 100644 source/de/anomic/data/YMarksHTMLImporter.java diff --git a/htroot/api/ymarks/add_ymark.java b/htroot/api/ymarks/add_ymark.java index 375b94fdd..e457ba327 100644 --- a/htroot/api/ymarks/add_ymark.java +++ b/htroot/api/ymarks/add_ymark.java @@ -9,7 +9,7 @@ import net.yacy.kelondro.blob.Tables.Data; import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.util.DateFormatter; -import de.anomic.data.YMarkStatics; +import de.anomic.data.YMarkTables; import de.anomic.data.userDB; import de.anomic.search.Switchboard; import de.anomic.server.serverObjects; @@ -29,18 +29,23 @@ public class add_ymark { final boolean isAuthUser = user!= null && user.hasRight(userDB.Entry.BOOKMARK_RIGHT); if(isAdmin || isAuthUser) { - final String bmk_table = (isAuthUser ? user.getUserName() : YMarkStatics.TABLE_BOOKMARKS_USER_ADMIN)+YMarkStatics.TABLE_BOOKMARKS_BASENAME; - final String tag_table = (isAuthUser ? user.getUserName() : YMarkStatics.TABLE_BOOKMARKS_USER_ADMIN)+YMarkStatics.TABLE_TAGS_BASENAME; + final String bmk_table = (isAuthUser ? user.getUserName() : YMarkTables.TABLE_BOOKMARKS_USER_ADMIN)+YMarkTables.TABLE_BOOKMARKS_BASENAME; + final String tag_table = (isAuthUser ? user.getUserName() : YMarkTables.TABLE_BOOKMARKS_USER_ADMIN)+YMarkTables.TABLE_TAGS_BASENAME; byte[] urlHash = null; - if(post.containsKey(YMarkStatics.TABLE_BOOKMARKS_COL_URL)) { + String url =""; + if(post.containsKey(YMarkTables.TABLE_BOOKMARKS_COL_URL)) { try { - urlHash = YMarkStatics.getBookmarkId(post.get(YMarkStatics.TABLE_BOOKMARKS_COL_URL,YMarkStatics.TABLE_BOOKMARKS_COL_DEFAULT)); + url = post.get(YMarkTables.TABLE_BOOKMARKS_COL_URL,YMarkTables.TABLE_BOOKMARKS_COL_DEFAULT); + if (!url.toLowerCase().startsWith(YMarkTables.TABLE_BOOKMARKS_URL_PROTOCOL_HTTP) && !url.toLowerCase().startsWith(YMarkTables.TABLE_BOOKMARKS_URL_PROTOCOL_HTTPS)) { + url=YMarkTables.TABLE_BOOKMARKS_URL_PROTOCOL_HTTP+url; + } + urlHash = YMarkTables.getBookmarkId(url); } catch (MalformedURLException e) { Log.logException(e); } - } else if (post.containsKey(YMarkStatics.TABLE_BOOKMARKS_COL_ID)) { - urlHash = post.get(YMarkStatics.TABLE_BOOKMARKS_COL_ID).getBytes(); + } else if (post.containsKey(YMarkTables.TABLE_BOOKMARKS_COL_ID)) { + urlHash = post.get(YMarkTables.TABLE_BOOKMARKS_COL_ID).getBytes(); } if(urlHash == null) { prop.put("result", "0"); @@ -62,39 +67,42 @@ public class add_ymark { if (bmk_row == null) { // create and insert new entry Data data = new Data(); - final String tagsString = YMarkStatics.cleanTagsString(post.get(YMarkStatics.TABLE_BOOKMARKS_COL_TAGS,YMarkStatics.TABLE_BOOKMARKS_COL_DEFAULT)); + final String tagsString = YMarkTables.cleanTagsString(post.get(YMarkTables.TABLE_BOOKMARKS_COL_TAGS,YMarkTables.TABLE_BOOKMARKS_COL_DEFAULT)); final byte[] date = DateFormatter.formatShortMilliSecond(new Date()).getBytes(); - data.put(YMarkStatics.TABLE_BOOKMARKS_COL_URL, post.get(YMarkStatics.TABLE_BOOKMARKS_COL_URL,YMarkStatics.TABLE_BOOKMARKS_COL_DEFAULT).getBytes()); - data.put(YMarkStatics.TABLE_BOOKMARKS_COL_TITLE, post.get(YMarkStatics.TABLE_BOOKMARKS_COL_TITLE,YMarkStatics.TABLE_BOOKMARKS_COL_DEFAULT).getBytes()); - data.put(YMarkStatics.TABLE_BOOKMARKS_COL_DESC, post.get(YMarkStatics.TABLE_BOOKMARKS_COL_DESC,YMarkStatics.TABLE_BOOKMARKS_COL_DEFAULT).getBytes()); - data.put(YMarkStatics.TABLE_BOOKMARKS_COL_PUBLIC, post.get(YMarkStatics.TABLE_BOOKMARKS_COL_PUBLIC,"false").getBytes()); - data.put(YMarkStatics.TABLE_BOOKMARKS_COL_TAGS, tagsString.getBytes()); - data.put(YMarkStatics.TABLE_BOOKMARKS_COL_DATE_ADDED, date); - data.put(YMarkStatics.TABLE_BOOKMARKS_COL_DATE_MODIFIED, date); - data.put(YMarkStatics.TABLE_BOOKMARKS_COL_DATE_VISITED, date); + data.put(YMarkTables.TABLE_BOOKMARKS_COL_URL, url.getBytes()); + data.put(YMarkTables.TABLE_BOOKMARKS_COL_TITLE, post.get(YMarkTables.TABLE_BOOKMARKS_COL_TITLE,YMarkTables.TABLE_BOOKMARKS_COL_DEFAULT).getBytes()); + data.put(YMarkTables.TABLE_BOOKMARKS_COL_DESC, post.get(YMarkTables.TABLE_BOOKMARKS_COL_DESC,YMarkTables.TABLE_BOOKMARKS_COL_DEFAULT).getBytes()); + data.put(YMarkTables.TABLE_BOOKMARKS_COL_PUBLIC, post.get(YMarkTables.TABLE_BOOKMARKS_COL_PUBLIC,YMarkTables.TABLE_BOOKMARKS_COL_PUBLIC_FALSE).getBytes()); + data.put(YMarkTables.TABLE_BOOKMARKS_COL_TAGS, tagsString.getBytes()); + data.put(YMarkTables.TABLE_BOOKMARKS_COL_VISITS, YMarkTables.TABLE_BOOKMARKS_COL_VISITS_ZERO.getBytes()); + data.put(YMarkTables.TABLE_BOOKMARKS_COL_FOLDER, post.get(YMarkTables.TABLE_BOOKMARKS_COL_FOLDER,YMarkTables.TABLE_FOLDERS_UNSORTED).getBytes()); + data.put(YMarkTables.TABLE_BOOKMARKS_COL_DATE_ADDED, date); + data.put(YMarkTables.TABLE_BOOKMARKS_COL_DATE_MODIFIED, date); + data.put(YMarkTables.TABLE_BOOKMARKS_COL_DATE_VISITED, date); sb.tables.insert(bmk_table, urlHash, data); - final String[] tagArray = tagsString.split(YMarkStatics.TABLE_TAGS_SEPARATOR); + final String[] tagArray = tagsString.split(YMarkTables.TABLE_TAGS_SEPARATOR); for (final String tag : tagArray) { - sb.tables.updateTAGTable(tag_table, tag, urlHash, YMarkStatics.TABLE_TAGS_ACTION_ADD); + sb.tables.bookmarks.updateTAGTable(tag_table, tag, urlHash, YMarkTables.TABLE_TAGS_ACTION_ADD); } } else { // modify and update existing entry - bmk_row.put(YMarkStatics.TABLE_BOOKMARKS_COL_TITLE, post.get(YMarkStatics.TABLE_BOOKMARKS_COL_TITLE,bmk_row.get(YMarkStatics.TABLE_BOOKMARKS_COL_TITLE,YMarkStatics.TABLE_BOOKMARKS_COL_DEFAULT)).getBytes()); - bmk_row.put(YMarkStatics.TABLE_BOOKMARKS_COL_DESC, post.get(YMarkStatics.TABLE_BOOKMARKS_COL_DESC,bmk_row.get(YMarkStatics.TABLE_BOOKMARKS_COL_DESC,YMarkStatics.TABLE_BOOKMARKS_COL_DEFAULT)).getBytes()); - bmk_row.put(YMarkStatics.TABLE_BOOKMARKS_COL_PUBLIC, post.get(YMarkStatics.TABLE_BOOKMARKS_COL_PUBLIC,bmk_row.get(YMarkStatics.TABLE_BOOKMARKS_COL_PUBLIC,YMarkStatics.TABLE_BOOKMARKS_COL_PUBLIC_FALSE)).getBytes()); - - final String tagsString = YMarkStatics.cleanTagsString(post.get(YMarkStatics.TABLE_BOOKMARKS_COL_TAGS,YMarkStatics.TABLE_BOOKMARKS_COL_DEFAULT)); - HashSetold_tagSet = YMarkStatics.getTagSet(bmk_row.get(YMarkStatics.TABLE_BOOKMARKS_COL_TAGS,YMarkStatics.TABLE_BOOKMARKS_COL_DEFAULT), false); - HashSetnew_tagSet = YMarkStatics.getTagSet(tagsString, false); - bmk_row.put(YMarkStatics.TABLE_BOOKMARKS_COL_TAGS, tagsString.getBytes()); + bmk_row.put(YMarkTables.TABLE_BOOKMARKS_COL_TITLE, post.get(YMarkTables.TABLE_BOOKMARKS_COL_TITLE,bmk_row.get(YMarkTables.TABLE_BOOKMARKS_COL_TITLE,YMarkTables.TABLE_BOOKMARKS_COL_DEFAULT)).getBytes()); + bmk_row.put(YMarkTables.TABLE_BOOKMARKS_COL_DESC, post.get(YMarkTables.TABLE_BOOKMARKS_COL_DESC,bmk_row.get(YMarkTables.TABLE_BOOKMARKS_COL_DESC,YMarkTables.TABLE_BOOKMARKS_COL_DEFAULT)).getBytes()); + bmk_row.put(YMarkTables.TABLE_BOOKMARKS_COL_PUBLIC, post.get(YMarkTables.TABLE_BOOKMARKS_COL_PUBLIC,bmk_row.get(YMarkTables.TABLE_BOOKMARKS_COL_PUBLIC,YMarkTables.TABLE_BOOKMARKS_COL_PUBLIC_FALSE)).getBytes()); + bmk_row.put(YMarkTables.TABLE_BOOKMARKS_COL_FOLDER, post.get(YMarkTables.TABLE_BOOKMARKS_COL_FOLDER,bmk_row.get(YMarkTables.TABLE_BOOKMARKS_COL_FOLDER,YMarkTables.TABLE_FOLDERS_UNSORTED)).getBytes()); + + final String tagsString = YMarkTables.cleanTagsString(post.get(YMarkTables.TABLE_BOOKMARKS_COL_TAGS,YMarkTables.TABLE_BOOKMARKS_COL_DEFAULT)); + HashSetold_tagSet = YMarkTables.getTagSet(bmk_row.get(YMarkTables.TABLE_BOOKMARKS_COL_TAGS,YMarkTables.TABLE_BOOKMARKS_COL_DEFAULT), false); + HashSetnew_tagSet = YMarkTables.getTagSet(tagsString, false); + bmk_row.put(YMarkTables.TABLE_BOOKMARKS_COL_TAGS, tagsString.getBytes()); // modify date attribute - bmk_row.put(YMarkStatics.TABLE_BOOKMARKS_COL_DATE_MODIFIED, DateFormatter.formatShortMilliSecond(new Date()).getBytes()); + bmk_row.put(YMarkTables.TABLE_BOOKMARKS_COL_DATE_MODIFIED, DateFormatter.formatShortMilliSecond(new Date()).getBytes()); // update bmk_table sb.tables.update(bmk_table, bmk_row); @@ -105,14 +113,14 @@ public class add_ymark { new_tagSet.removeAll(old_tagSet); tagIter = new_tagSet.iterator(); while(tagIter.hasNext()) { - sb.tables.updateTAGTable(tag_table, tagIter.next(), urlHash, YMarkStatics.TABLE_TAGS_ACTION_ADD); + sb.tables.bookmarks.updateTAGTable(tag_table, tagIter.next(), urlHash, YMarkTables.TABLE_TAGS_ACTION_ADD); } - new_tagSet = YMarkStatics.getTagSet(tagsString, false); + new_tagSet = YMarkTables.getTagSet(tagsString, false); old_tagSet.removeAll(new_tagSet); tagIter=old_tagSet.iterator(); while(tagIter.hasNext()) { - sb.tables.updateTAGTable(tag_table, tagIter.next(), urlHash, YMarkStatics.TABLE_TAGS_ACTION_REMOVE); + sb.tables.bookmarks.updateTAGTable(tag_table, tagIter.next(), urlHash, YMarkTables.TABLE_TAGS_ACTION_REMOVE); } } } catch (IOException e) { @@ -120,7 +128,7 @@ public class add_ymark { } prop.put("result", "1"); } else { - prop.put(YMarkStatics.TABLE_BOOKMARKS_USER_AUTHENTICATE,YMarkStatics.TABLE_BOOKMARKS_USER_AUTHENTICATE_MSG); + prop.put(YMarkTables.TABLE_BOOKMARKS_USER_AUTHENTICATE,YMarkTables.TABLE_BOOKMARKS_USER_AUTHENTICATE_MSG); } // return rewrite properties return prop; diff --git a/htroot/api/ymarks/delete_ymark.java b/htroot/api/ymarks/delete_ymark.java index c2cfad5c0..9bc36dd77 100644 --- a/htroot/api/ymarks/delete_ymark.java +++ b/htroot/api/ymarks/delete_ymark.java @@ -4,7 +4,7 @@ import net.yacy.cora.protocol.RequestHeader; import net.yacy.kelondro.blob.Tables; import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.logging.Log; -import de.anomic.data.YMarkStatics; +import de.anomic.data.YMarkTables; import de.anomic.data.userDB; import de.anomic.search.Switchboard; import de.anomic.server.serverObjects; @@ -24,16 +24,16 @@ public class delete_ymark { final boolean isAuthUser = user!= null && user.hasRight(userDB.Entry.BOOKMARK_RIGHT); if(isAdmin || isAuthUser) { - final String bmk_table = (isAuthUser ? user.getUserName() : YMarkStatics.TABLE_BOOKMARKS_USER_ADMIN)+YMarkStatics.TABLE_BOOKMARKS_BASENAME; - final String tag_table = (isAuthUser ? user.getUserName() : YMarkStatics.TABLE_BOOKMARKS_USER_ADMIN)+YMarkStatics.TABLE_TAGS_BASENAME; + final String bmk_table = (isAuthUser ? user.getUserName() : YMarkTables.TABLE_BOOKMARKS_USER_ADMIN)+YMarkTables.TABLE_BOOKMARKS_BASENAME; + final String tag_table = (isAuthUser ? user.getUserName() : YMarkTables.TABLE_BOOKMARKS_USER_ADMIN)+YMarkTables.TABLE_TAGS_BASENAME; byte[] urlHash = null; try { - if(post.containsKey(YMarkStatics.TABLE_BOOKMARKS_COL_ID)) { - urlHash = post.get(YMarkStatics.TABLE_BOOKMARKS_COL_ID).getBytes(); - } else if(post.containsKey(YMarkStatics.TABLE_BOOKMARKS_COL_URL)) { - urlHash = YMarkStatics.getBookmarkId(post.get(YMarkStatics.TABLE_BOOKMARKS_COL_URL)); + if(post.containsKey(YMarkTables.TABLE_BOOKMARKS_COL_ID)) { + urlHash = post.get(YMarkTables.TABLE_BOOKMARKS_COL_ID).getBytes(); + } else if(post.containsKey(YMarkTables.TABLE_BOOKMARKS_COL_URL)) { + urlHash = YMarkTables.getBookmarkId(post.get(YMarkTables.TABLE_BOOKMARKS_COL_URL)); } else { prop.put("result", "0"); return prop; @@ -41,10 +41,10 @@ public class delete_ymark { Tables.Row bmk_row = null; bmk_row = sb.tables.select(bmk_table, urlHash); if(bmk_row != null) { - final String tagsString = bmk_row.get(YMarkStatics.TABLE_BOOKMARKS_COL_TAGS,YMarkStatics.TABLE_BOOKMARKS_COL_DEFAULT); - final String[] tagArray = tagsString.split(YMarkStatics.TABLE_TAGS_SEPARATOR); + final String tagsString = bmk_row.get(YMarkTables.TABLE_BOOKMARKS_COL_TAGS,YMarkTables.TABLE_BOOKMARKS_COL_DEFAULT); + final String[] tagArray = tagsString.split(YMarkTables.TABLE_TAGS_SEPARATOR); for (final String tag : tagArray) { - sb.tables.updateTAGTable(tag_table, tag, urlHash,YMarkStatics.TABLE_TAGS_ACTION_REMOVE); + sb.tables.bookmarks.updateTAGTable(tag_table, tag, urlHash,YMarkTables.TABLE_TAGS_ACTION_REMOVE); } } sb.tables.delete(bmk_table,urlHash); @@ -55,7 +55,7 @@ public class delete_ymark { Log.logException(e); } } else { - prop.put(YMarkStatics.TABLE_BOOKMARKS_USER_AUTHENTICATE,YMarkStatics.TABLE_BOOKMARKS_USER_AUTHENTICATE_MSG); + prop.put(YMarkTables.TABLE_BOOKMARKS_USER_AUTHENTICATE,YMarkTables.TABLE_BOOKMARKS_USER_AUTHENTICATE_MSG); } // return rewrite properties return prop; diff --git a/htroot/api/ymarks/get_ymark.java b/htroot/api/ymarks/get_ymark.java index 75aee74d8..ae807b13b 100644 --- a/htroot/api/ymarks/get_ymark.java +++ b/htroot/api/ymarks/get_ymark.java @@ -5,7 +5,7 @@ import net.yacy.cora.protocol.RequestHeader; import net.yacy.kelondro.blob.Tables; import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.logging.Log; -import de.anomic.data.YMarkStatics; +import de.anomic.data.YMarkTables; import de.anomic.data.userDB; import de.anomic.search.Switchboard; import de.anomic.server.serverObjects; @@ -22,16 +22,16 @@ public class get_ymark { final boolean isAuthUser = user!= null && user.hasRight(userDB.Entry.BOOKMARK_RIGHT); if(isAdmin || isAuthUser) { - final String bmk_table = (isAuthUser ? user.getUserName() : YMarkStatics.TABLE_BOOKMARKS_USER_ADMIN)+YMarkStatics.TABLE_BOOKMARKS_BASENAME; - final String tag_table = (isAuthUser ? user.getUserName() : YMarkStatics.TABLE_BOOKMARKS_USER_ADMIN)+YMarkStatics.TABLE_TAGS_BASENAME; + final String bmk_table = (isAuthUser ? user.getUserName() : YMarkTables.TABLE_BOOKMARKS_USER_ADMIN)+YMarkTables.TABLE_BOOKMARKS_BASENAME; + final String tag_table = (isAuthUser ? user.getUserName() : YMarkTables.TABLE_BOOKMARKS_USER_ADMIN)+YMarkTables.TABLE_TAGS_BASENAME; - if(post.containsKey(YMarkStatics.TABLE_TAGS_COL_TAG)) { - final byte[] tagHash = YMarkStatics.getTagHash(post.get(YMarkStatics.TABLE_TAGS_COL_TAG)); + if(post.containsKey(YMarkTables.TABLE_TAGS_COL_TAG)) { + final byte[] tagHash = YMarkTables.getTagId(post.get(YMarkTables.TABLE_TAGS_COL_TAG)); Tables.Row tag_row = null; try { tag_row = sb.tables.select(tag_table, tagHash); if (tag_row != null) { - final IteratorurlIter = (YMarkStatics.keysStringToKeySet(new String(tag_row.get(YMarkStatics.TABLE_TAGS_COL_URLS)))).iterator(); + final IteratorurlIter = (YMarkTables.keysStringToKeySet(new String(tag_row.get(YMarkTables.TABLE_TAGS_COL_URLS)))).iterator(); int count = 0; while(urlIter.hasNext()) { final byte[] urlHash = urlIter.next().getBytes(); @@ -39,14 +39,14 @@ public class get_ymark { bmk_row = sb.tables.select(bmk_table, urlHash); if (bmk_row != null) { prop.putXML("bookmarks_"+count+"_id", new String(urlHash)); - prop.putXML("bookmarks_"+count+"_url", new String(bmk_row.get(YMarkStatics.TABLE_BOOKMARKS_COL_URL,YMarkStatics.TABLE_BOOKMARKS_COL_DEFAULT))); - prop.putXML("bookmarks_"+count+"_title", new String(bmk_row.get(YMarkStatics.TABLE_BOOKMARKS_COL_TITLE,YMarkStatics.TABLE_BOOKMARKS_COL_DEFAULT))); - prop.putXML("bookmarks_"+count+"_desc", new String(bmk_row.get(YMarkStatics.TABLE_BOOKMARKS_COL_DESC,YMarkStatics.TABLE_BOOKMARKS_COL_DEFAULT))); - prop.putXML("bookmarks_"+count+"_added", new String(bmk_row.get(YMarkStatics.TABLE_BOOKMARKS_COL_DATE_ADDED,YMarkStatics.TABLE_BOOKMARKS_COL_DEFAULT))); - prop.putXML("bookmarks_"+count+"_modified", new String(bmk_row.get(YMarkStatics.TABLE_BOOKMARKS_COL_DATE_MODIFIED,YMarkStatics.TABLE_BOOKMARKS_COL_DEFAULT))); - prop.putXML("bookmarks_"+count+"_visited", new String(bmk_row.get(YMarkStatics.TABLE_BOOKMARKS_COL_DATE_VISITED,YMarkStatics.TABLE_BOOKMARKS_COL_DEFAULT))); - prop.putXML("bookmarks_"+count+"_public", new String(bmk_row.get(YMarkStatics.TABLE_BOOKMARKS_COL_PUBLIC,YMarkStatics.TABLE_BOOKMARKS_COL_PUBLIC_FALSE))); - prop.putXML("bookmarks_"+count+"_tags", new String(bmk_row.get(YMarkStatics.TABLE_BOOKMARKS_COL_TAGS,YMarkStatics.TABLE_BOOKMARKS_COL_DEFAULT))); + prop.putXML("bookmarks_"+count+"_url", new String(bmk_row.get(YMarkTables.TABLE_BOOKMARKS_COL_URL,YMarkTables.TABLE_BOOKMARKS_COL_DEFAULT))); + prop.putXML("bookmarks_"+count+"_title", new String(bmk_row.get(YMarkTables.TABLE_BOOKMARKS_COL_TITLE,YMarkTables.TABLE_BOOKMARKS_COL_DEFAULT))); + prop.putXML("bookmarks_"+count+"_desc", new String(bmk_row.get(YMarkTables.TABLE_BOOKMARKS_COL_DESC,YMarkTables.TABLE_BOOKMARKS_COL_DEFAULT))); + prop.putXML("bookmarks_"+count+"_added", new String(bmk_row.get(YMarkTables.TABLE_BOOKMARKS_COL_DATE_ADDED,YMarkTables.TABLE_BOOKMARKS_COL_DEFAULT))); + prop.putXML("bookmarks_"+count+"_modified", new String(bmk_row.get(YMarkTables.TABLE_BOOKMARKS_COL_DATE_MODIFIED,YMarkTables.TABLE_BOOKMARKS_COL_DEFAULT))); + prop.putXML("bookmarks_"+count+"_visited", new String(bmk_row.get(YMarkTables.TABLE_BOOKMARKS_COL_DATE_VISITED,YMarkTables.TABLE_BOOKMARKS_COL_DEFAULT))); + prop.putXML("bookmarks_"+count+"_public", new String(bmk_row.get(YMarkTables.TABLE_BOOKMARKS_COL_PUBLIC,YMarkTables.TABLE_BOOKMARKS_COL_PUBLIC_FALSE))); + prop.putXML("bookmarks_"+count+"_tags", new String(bmk_row.get(YMarkTables.TABLE_BOOKMARKS_COL_TAGS,YMarkTables.TABLE_BOOKMARKS_COL_DEFAULT))); count++; } } @@ -63,7 +63,7 @@ public class get_ymark { } } } else { - prop.put(YMarkStatics.TABLE_BOOKMARKS_USER_AUTHENTICATE,YMarkStatics.TABLE_BOOKMARKS_USER_AUTHENTICATE_MSG); + prop.put(YMarkTables.TABLE_BOOKMARKS_USER_AUTHENTICATE,YMarkTables.TABLE_BOOKMARKS_USER_AUTHENTICATE_MSG); } // return rewrite properties return prop; diff --git a/htroot/api/ymarks/import.html b/htroot/api/ymarks/import.html new file mode 100644 index 000000000..c98cbb246 --- /dev/null +++ b/htroot/api/ymarks/import.html @@ -0,0 +1,37 @@ + + + + YaCy Bookmarks + + +
+
+ Import HTML Bookmarks +
+
+ +
+
+ +
+
+ +
+
+ +
+
+ +
+
+ +
+
+ +
+
+ + \ No newline at end of file diff --git a/htroot/api/ymarks/import_html.java b/htroot/api/ymarks/import_html.java new file mode 100644 index 000000000..e0891302d --- /dev/null +++ b/htroot/api/ymarks/import_html.java @@ -0,0 +1,58 @@ +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.UnsupportedEncodingException; + +import javax.swing.text.html.parser.ParserDelegator; + +import net.yacy.cora.protocol.RequestHeader; +import net.yacy.kelondro.logging.Log; +import de.anomic.data.YMarksHTMLImporter; +import de.anomic.data.YMarkTables; +import de.anomic.data.userDB; +import de.anomic.search.Switchboard; +import de.anomic.server.serverObjects; +import de.anomic.server.serverSwitch; + + +public class import_html { + + public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) { + final Switchboard sb = (Switchboard) env; + final serverObjects prop = new serverObjects(); + final userDB.Entry user = sb.userDB.getUser(header); + final boolean isAdmin = (sb.verifyAuthentication(header, true)); + final boolean isAuthUser = user!= null && user.hasRight(userDB.Entry.BOOKMARK_RIGHT); + + if(isAdmin || isAuthUser) { + final String bmk_table = (isAuthUser ? user.getUserName() : YMarkTables.TABLE_BOOKMARKS_USER_ADMIN)+YMarkTables.TABLE_BOOKMARKS_BASENAME; + if(post.containsKey("htmlfile")){ + try { + final ByteArrayInputStream byteIn = new ByteArrayInputStream(post.get("htmlfile$file").getBytes("UTF-8")); + if(byteIn !=null) { + final InputStreamReader reader = new InputStreamReader(byteIn,"UTF-8"); + final ParserDelegator delegator = new ParserDelegator(); + final YMarksHTMLImporter htmlHandler = new YMarksHTMLImporter(sb.tables, bmk_table); + delegator.parse(reader, htmlHandler, true); + } + } catch (UnsupportedEncodingException e) { + Log.logException(e); + } catch (IOException e) { + Log.logException(e); + } + prop.put("result", "1"); + } + } + // return rewrite properties + return prop; + } +} + + + + + + + + + diff --git a/htroot/api/ymarks/import_html.xml b/htroot/api/ymarks/import_html.xml new file mode 100644 index 000000000..d6880fb21 --- /dev/null +++ b/htroot/api/ymarks/import_html.xml @@ -0,0 +1,2 @@ + + \ No newline at end of file diff --git a/source/de/anomic/data/WorkTables.java b/source/de/anomic/data/WorkTables.java index cbcdd5f59..a0afcbfdc 100644 --- a/source/de/anomic/data/WorkTables.java +++ b/source/de/anomic/data/WorkTables.java @@ -31,7 +31,6 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.Date; -import java.util.HashSet; import java.util.LinkedHashMap; import java.util.Map; @@ -43,7 +42,7 @@ import net.yacy.kelondro.util.DateFormatter; import de.anomic.server.serverObjects; public class WorkTables extends Tables { - + public final static String TABLE_API_NAME = "api"; public final static String TABLE_API_TYPE_STEERING = "steering"; public final static String TABLE_API_TYPE_CONFIGURATION = "configuration"; @@ -66,9 +65,11 @@ public class WorkTables extends Tables { public final static String TABLE_ACTIVECRAWLS_NAME = "crawljobsActive"; public final static String TABLE_PASSIVECRAWLS_NAME = "crawljobsPassive"; + public YMarkTables bookmarks; public WorkTables(final File workPath) { super(workPath, 12); + this.bookmarks = new YMarkTables(this); } /** @@ -277,62 +278,4 @@ public class WorkTables extends Tables { d -= d % 60000; // remove seconds row.put(WorkTables.TABLE_API_COL_DATE_NEXT_EXEC, new Date(d)); } - - /** - * YMark function that updates the tag index - * @param tag_table is the user specific tag index - * @param tag is a single tag - * @param url is the url has as returned by DigestURI.hash() - * @param action is either add (1) or remove (2) - * @return - */ - public int updateTAGTable(final String tag_table, final String tag, final byte[] url, final int action) { - Tables.Row tag_row = null; - final byte[] tagHash = YMarkStatics.getTagHash(tag); - final String urlHash = new String(url); - HashSeturlSet = new HashSet(); - try { - tag_row = this.select(tag_table, tagHash); - if(tag_row == null) { - switch (action) { - case YMarkStatics.TABLE_TAGS_ACTION_ADD: - urlSet.add(urlHash); - break; - default: - return 0; - } - Data tagEntry = new Data(); - tagEntry.put(YMarkStatics.TABLE_TAGS_COL_TAG, tag.getBytes()); - tagEntry.put(YMarkStatics.TABLE_TAGS_COL_URLS, YMarkStatics.keySetToBytes(urlSet)); - this.insert(tag_table, tagHash, tagEntry); - return 1; - } else { - urlSet = YMarkStatics.keysStringToKeySet(new String(tag_row.get(YMarkStatics.TABLE_TAGS_COL_URLS))); - if(urlSet.contains(urlHash)) - Log.logInfo(YMarkStatics.TABLE_BOOKMARKS_LOG, "ok, urlHash found!"); - switch (action) { - case YMarkStatics.TABLE_TAGS_ACTION_ADD: - urlSet.add(urlHash); - break; - case YMarkStatics.TABLE_TAGS_ACTION_REMOVE: - urlSet.remove(urlHash); - if(urlSet.isEmpty()) { - this.delete(tag_table, tagHash); - return 1; - } - break; - default: - return 1; - } - tag_row.put(YMarkStatics.TABLE_TAGS_COL_URLS, YMarkStatics.keySetToBytes(urlSet)); - this.update(tag_table, tag_row); - return 1; - } - } catch (IOException e) { - Log.logException(e); - } catch (RowSpaceExceededException e) { - Log.logException(e); - } - return 0; - } } diff --git a/source/de/anomic/data/YMarkTables.java b/source/de/anomic/data/YMarkTables.java new file mode 100644 index 000000000..ec9ce63d4 --- /dev/null +++ b/source/de/anomic/data/YMarkTables.java @@ -0,0 +1,209 @@ +package de.anomic.data; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Set; + +import net.yacy.kelondro.blob.Tables; +import net.yacy.kelondro.blob.Tables.Data; +import net.yacy.kelondro.data.meta.DigestURI; +import net.yacy.kelondro.data.word.Word; +import net.yacy.kelondro.index.RowSpaceExceededException; +import net.yacy.kelondro.logging.Log; + +public class YMarkTables { + + public final static String TABLE_BOOKMARKS_BASENAME = "_bookmarks"; + public final static String TABLE_BOOKMARKS_LOG = "BOOKMARKS"; + + public final static String TABLE_BOOKMARKS_USER_ADMIN = "admin"; + public final static String TABLE_BOOKMARKS_USER_AUTHENTICATE = "AUTHENTICATE"; + public final static String TABLE_BOOKMARKS_USER_AUTHENTICATE_MSG = "Authentication required!"; + + public final static String TABLE_BOOKMARKS_URL_PROTOCOL_HTTP = "http://"; + public final static String TABLE_BOOKMARKS_URL_PROTOCOL_HTTPS = "https://"; + + public final static String TABLE_BOOKMARKS_COL_ID = "id"; + public final static String TABLE_BOOKMARKS_COL_URL = "url"; + public final static String TABLE_BOOKMARKS_COL_TITLE = "title"; + public final static String TABLE_BOOKMARKS_COL_DESC = "desc"; + public final static String TABLE_BOOKMARKS_COL_DATE_ADDED = "added"; + public final static String TABLE_BOOKMARKS_COL_DATE_MODIFIED = "modified"; + public final static String TABLE_BOOKMARKS_COL_DATE_VISITED = "visited"; + public final static String TABLE_BOOKMARKS_COL_PUBLIC = "public"; + public final static String TABLE_BOOKMARKS_COL_TAGS = "tags"; + public final static String TABLE_BOOKMARKS_COL_VISITS = "visits"; + public final static String TABLE_BOOKMARKS_COL_FOLDER = "folder"; + + public final static String TABLE_BOOKMARKS_COL_DEFAULT = ""; + public final static String TABLE_BOOKMARKS_COL_PUBLIC_TRUE = "true"; + public final static String TABLE_BOOKMARKS_COL_PUBLIC_FALSE = "false"; + public final static String TABLE_BOOKMARKS_COL_VISITS_ZERO = "0"; + + public final static String TABLE_TAGS_BASENAME = "_tags"; + public final static String TABLE_TAGS_SEPARATOR = ","; + + public final static String TABLE_TAGS_COL_ID = "id"; + public final static String TABLE_TAGS_COL_TAG = "tag"; + public final static String TABLE_TAGS_COL_URLS = "urls"; + + public final static int TABLE_TAGS_ACTION_ADD = 1; + public final static int TABLE_TAGS_ACTION_REMOVE = 2; + + public final static String TABLE_FOLDERS_SEPARATOR = "/"; + public final static String TABLE_FOLDERS_ROOT = "/"; + public final static String TABLE_FOLDERS_UNSORTED = "/unsorted"; + public final static String TABLE_FOLDERS_IMPORTED = "/imported"; + + private Tables worktables; + + public YMarkTables(final Tables wt) { + this.worktables = wt; + } + + public final static byte[] getBookmarkId(String url) throws MalformedURLException { + return (new DigestURI(url, null)).hash(); + } + + public final static byte[] getTagId(final String tag) { + return Word.word2hash(tag.toLowerCase()); + } + + public final static HashSet getTagSet(final String tagsString, boolean clean) { + HashSettagSet = new HashSet(); + final String[] tagArray = clean ? cleanTagsString(tagsString).split(TABLE_TAGS_SEPARATOR) : tagsString.split(TABLE_TAGS_SEPARATOR); + for (final String tag : tagArray) { + tagSet.add(tag); + } + return tagSet; + } + + public final static HashSet getTagSet(final String tagsString) { + return getTagSet(tagsString, true); + } + + public final static HashSet getTagIdSet(final String tagsString, boolean clean) { + HashSettagSet = new HashSet(); + final String[] tagArray = clean ? cleanTagsString(tagsString).split(TABLE_TAGS_SEPARATOR) : tagsString.split(TABLE_TAGS_SEPARATOR); + for (final String tag : tagArray) { + tagSet.add(getTagId(tag)); + } + return tagSet; + } + + public final static Set getTagIdSet(final String tagsString) { + return getTagIdSet(tagsString, true); + } + + public final static byte[] keySetToBytes(final HashSet urlSet) { + final Iterator urlIter = urlSet.iterator(); + final + StringBuilder urls = new StringBuilder(); + while(urlIter.hasNext()) { + urls.append(TABLE_TAGS_SEPARATOR); + urls.append(urlIter.next()); + } + urls.deleteCharAt(0); + return urls.toString().getBytes(); + } + + public final static HashSet keysStringToKeySet(final String keysString) { + HashSet keySet = new HashSet(); + final String[] keyArray = keysString.split(TABLE_TAGS_SEPARATOR); + for (final String key : keyArray) { + keySet.add(key); + } + return keySet; + } + + public final static String cleanTagsString(String tagsString) { + // get rid of heading, trailing and double commas since they are useless + while (tagsString.length() > 0 && tagsString.charAt(0) == TABLE_TAGS_SEPARATOR.charAt(0)) { + tagsString = tagsString.substring(1); + } + while (tagsString.endsWith(TABLE_TAGS_SEPARATOR)) { + tagsString = tagsString.substring(0,tagsString.length() -1); + } + while (tagsString.contains(",,")){ + tagsString = tagsString.replaceAll(",,", TABLE_TAGS_SEPARATOR); + } + // space characters following a comma are removed + tagsString = tagsString.replaceAll(",\\s+", TABLE_TAGS_SEPARATOR); + return tagsString; + } + + public final static String cleanFoldersString(String foldersString) { + foldersString = cleanTagsString(foldersString); + // get rid of double and trailing slashes + while (foldersString.endsWith("/")){ + foldersString = foldersString.substring(0, foldersString.length() -1); + } + while (foldersString.contains("/,")){ + foldersString = foldersString.replaceAll("/,", TABLE_TAGS_SEPARATOR); + } + while (foldersString.contains("//")){ + foldersString = foldersString.replaceAll("//", "/"); + } + return foldersString; + } + + /** + * YMark function that updates the tag index + * @param tag_table is the user specific tag index + * @param tag is a single tag + * @param url is the url has as returned by DigestURI.hash() + * @param action is either add (1) or remove (2) + * @return + */ + public int updateTAGTable(final String tag_table, final String tag, final byte[] url, final int action) { + Tables.Row tag_row = null; + final byte[] tagHash = YMarkTables.getTagId(tag); + final String urlHash = new String(url); + HashSeturlSet = new HashSet(); + try { + tag_row = this.worktables.select(tag_table, tagHash); + if(tag_row == null) { + switch (action) { + case YMarkTables.TABLE_TAGS_ACTION_ADD: + urlSet.add(urlHash); + break; + default: + return 0; + } + Data tagEntry = new Data(); + tagEntry.put(YMarkTables.TABLE_TAGS_COL_TAG, tag.getBytes()); + tagEntry.put(YMarkTables.TABLE_TAGS_COL_URLS, YMarkTables.keySetToBytes(urlSet)); + this.worktables.insert(tag_table, tagHash, tagEntry); + return 1; + } else { + urlSet = YMarkTables.keysStringToKeySet(new String(tag_row.get(YMarkTables.TABLE_TAGS_COL_URLS))); + if(urlSet.contains(urlHash)) + Log.logInfo(YMarkTables.TABLE_BOOKMARKS_LOG, "ok, urlHash found!"); + switch (action) { + case YMarkTables.TABLE_TAGS_ACTION_ADD: + urlSet.add(urlHash); + break; + case YMarkTables.TABLE_TAGS_ACTION_REMOVE: + urlSet.remove(urlHash); + if(urlSet.isEmpty()) { + this.worktables.delete(tag_table, tagHash); + return 1; + } + break; + default: + return 1; + } + tag_row.put(YMarkTables.TABLE_TAGS_COL_URLS, YMarkTables.keySetToBytes(urlSet)); + this.worktables.update(tag_table, tag_row); + return 1; + } + } catch (IOException e) { + Log.logException(e); + } catch (RowSpaceExceededException e) { + Log.logException(e); + } + return 0; + } +} diff --git a/source/de/anomic/data/YMarksHTMLImporter.java b/source/de/anomic/data/YMarksHTMLImporter.java new file mode 100644 index 000000000..67965960c --- /dev/null +++ b/source/de/anomic/data/YMarksHTMLImporter.java @@ -0,0 +1,95 @@ +package de.anomic.data; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.util.Date; + +import javax.swing.text.MutableAttributeSet; +import javax.swing.text.html.HTML; +import javax.swing.text.html.HTMLEditorKit; + +import net.yacy.kelondro.blob.Tables; +import net.yacy.kelondro.blob.Tables.Data; +import net.yacy.kelondro.logging.Log; + +public class YMarksHTMLImporter extends HTMLEditorKit.ParserCallback { + + private static final int NOTHING = 0; + private static final int BOOKMARK = 2; + private static final int FOLDER = 3; + + private Tables worktables; + private String bmk_table; + + private int state; + private String folder = YMarkTables.TABLE_FOLDERS_IMPORTED; + private String href; + private Date date_added; + private Date date_visited; + private Date date_modified; + + public YMarksHTMLImporter(final Tables worktables, final String bmk_table) { + this.bmk_table = bmk_table; + this.worktables = worktables; + } + + public void handleText(char[] data, int pos) { + switch (state) { + case NOTHING: + break; + case BOOKMARK: + Data bmk = new Data(); + byte[] urlHash; + try { + if(href.toLowerCase().startsWith("http://") || href.toLowerCase().startsWith("http://")) { + urlHash = YMarkTables.getBookmarkId(this.href); + // create and insert new entry + bmk.put(YMarkTables.TABLE_BOOKMARKS_COL_URL, this.href.getBytes()); + bmk.put(YMarkTables.TABLE_BOOKMARKS_COL_TITLE, (new String(data)).getBytes()); + bmk.put(YMarkTables.TABLE_BOOKMARKS_COL_DESC, YMarkTables.TABLE_BOOKMARKS_COL_DEFAULT.getBytes()); + bmk.put(YMarkTables.TABLE_BOOKMARKS_COL_PUBLIC, YMarkTables.TABLE_BOOKMARKS_COL_PUBLIC_FALSE.getBytes()); + bmk.put(YMarkTables.TABLE_BOOKMARKS_COL_TAGS, YMarkTables.TABLE_BOOKMARKS_COL_DEFAULT.getBytes()); + bmk.put(YMarkTables.TABLE_BOOKMARKS_COL_VISITS, YMarkTables.TABLE_BOOKMARKS_COL_VISITS_ZERO.getBytes()); + bmk.put(YMarkTables.TABLE_BOOKMARKS_COL_FOLDER, this.folder.getBytes()); + // bmk.put(YMarkStatics.TABLE_BOOKMARKS_COL_DATE_ADDED, DateFormatter.formatShortMilliSecond(this.date_added).getBytes()); + // bmk.put(YMarkStatics.TABLE_BOOKMARKS_COL_DATE_MODIFIED, DateFormatter.formatShortMilliSecond(this.date_modified).getBytes()); + // bmk.put(YMarkStatics.TABLE_BOOKMARKS_COL_DATE_VISITED, DateFormatter.formatShortMilliSecond(this.date_visited).getBytes()); + worktables.insert(bmk_table, urlHash, bmk); + } + break; + } catch (MalformedURLException e) { + Log.logException(e); + } catch (IOException e) { + Log.logException(e); + } + break; + case FOLDER: + this.folder = this.folder + YMarkTables.TABLE_FOLDERS_SEPARATOR + new String(data); + Log.logInfo("IMPORT folder:", folder); + break; + default: + break; + } + state = NOTHING; + } + + public void handleStartTag(HTML.Tag t, MutableAttributeSet a, int pos) { + if (t == HTML.Tag.A) { + this.href = (String)a.getAttribute(HTML.Attribute.HREF); + // this.date_added = new Date(Long.parseLong((String)a.getAttribute("add_date"))*1000l); + // this.date_visited = new Date(Long.parseLong((String)a.getAttribute("last_visit"))*1000l); + // this.date_modified = new Date(Long.parseLong((String)a.getAttribute("last_modified"))*1000l); + state = BOOKMARK; + } else if (t == HTML.Tag.H3) { + state = FOLDER; + } + } + + public void handleEndTag(HTML.Tag t, int pos) { + if (t == HTML.Tag.DL) { + if(!folder.equals(YMarkTables.TABLE_FOLDERS_IMPORTED)) { + folder = folder.replaceAll("(/.[^/]*$)", ""); + } + } + } +}