From 808edffaf6d6dfe0a313127324b9bb64b6b93864 Mon Sep 17 00:00:00 2001 From: apfelmaennchen Date: Sat, 6 Nov 2010 20:26:13 +0000 Subject: [PATCH] ymarks - some refactoring - working xbel and html import (/api/ymarks/test_import.html) - working treeview (/api/ymarks/test_treeview.html) git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7312 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- ...{get_ymark_tree.java => get_treeview.java} | 24 +-- ...{get_ymark_tree.json => get_treeview.json} | 0 .../{get_ymark_tree.xml => get_treeview.xml} | 0 htroot/api/ymarks/get_xbel.java | 80 ++++++++ htroot/api/ymarks/get_xbel.xml | 8 + htroot/api/ymarks/get_ymark_list.java | 91 --------- htroot/api/ymarks/get_ymark_list.xml | 6 - .../{import_html.java => import_ymark.java} | 7 +- .../{import_html.xml => import_ymark.xml} | 0 .../ymarks/{import.html => test_import.html} | 4 +- .../ymarks/{test.html => test_treeview.html} | 2 +- htroot/yacy/ui/css/jquery.treeview.css | 4 +- source/de/anomic/data/YMarkIndex.java | 28 ++- source/de/anomic/data/YMarkStatics.java | 136 -------------- source/de/anomic/data/YMarkTables.java | 37 +++- source/de/anomic/data/YMarksHTMLImporter.java | 17 +- source/de/anomic/data/YMarksXBELImporter.java | 172 +++++++++++------- 17 files changed, 265 insertions(+), 351 deletions(-) rename htroot/api/ymarks/{get_ymark_tree.java => get_treeview.java} (91%) rename htroot/api/ymarks/{get_ymark_tree.json => get_treeview.json} (100%) rename htroot/api/ymarks/{get_ymark_tree.xml => get_treeview.xml} (100%) create mode 100644 htroot/api/ymarks/get_xbel.java create mode 100644 htroot/api/ymarks/get_xbel.xml delete mode 100644 htroot/api/ymarks/get_ymark_list.java delete mode 100644 htroot/api/ymarks/get_ymark_list.xml rename htroot/api/ymarks/{import_html.java => import_ymark.java} (94%) rename htroot/api/ymarks/{import_html.xml => import_ymark.xml} (100%) rename htroot/api/ymarks/{import.html => test_import.html} (89%) rename htroot/api/ymarks/{test.html => test_treeview.html} (94%) delete mode 100644 source/de/anomic/data/YMarkStatics.java diff --git a/htroot/api/ymarks/get_ymark_tree.java b/htroot/api/ymarks/get_treeview.java similarity index 91% rename from htroot/api/ymarks/get_ymark_tree.java rename to htroot/api/ymarks/get_treeview.java index d48f45627..3eb718d4b 100644 --- a/htroot/api/ymarks/get_ymark_tree.java +++ b/htroot/api/ymarks/get_treeview.java @@ -13,24 +13,11 @@ import de.anomic.search.Switchboard; import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; -public class get_ymark_tree { +public class get_treeview { public static final String ROOT = "root"; public static final String SOURCE = "source"; - - // for some reason enums don't work in servlet classes ?!?! - /* - private static final Map TYPE = createMap(); - private static Map createMap() { - Map result = new HashMap(); - result.put("tags", "tag"); - result.put("url", "link"); - result.put("public", "lock"); - result.put("visits", "stat"); - return Collections.unmodifiableMap(result); - } - */ - + static serverObjects prop; public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) { @@ -74,12 +61,11 @@ public class get_ymark_tree { } int n = root.split(YMarkTables.FOLDERS_SEPARATOR).length; if (n == 0) n = 1; - while (it.hasNext()) { String folder = it.next(); foldername = folder.split(YMarkTables.FOLDERS_SEPARATOR); if (foldername.length == n+1) { - prop.put("folders_"+count+"_foldername", foldername[n]); + prop.put("folders_"+count+"_foldername", foldername[n]); prop.put("folders_"+count+"_expanded", "false"); prop.put("folders_"+count+"_type", "folder"); prop.put("folders_"+count+"_hash", folder); //TODO: switch from pathString to folderHash @@ -161,7 +147,9 @@ public class get_ymark_tree { Log.logException(e); } } - } + } else { + prop.put(YMarkTables.USER_AUTHENTICATE,YMarkTables.USER_AUTHENTICATE_MSG); + } // return rewrite properties return prop; } diff --git a/htroot/api/ymarks/get_ymark_tree.json b/htroot/api/ymarks/get_treeview.json similarity index 100% rename from htroot/api/ymarks/get_ymark_tree.json rename to htroot/api/ymarks/get_treeview.json diff --git a/htroot/api/ymarks/get_ymark_tree.xml b/htroot/api/ymarks/get_treeview.xml similarity index 100% rename from htroot/api/ymarks/get_ymark_tree.xml rename to htroot/api/ymarks/get_treeview.xml diff --git a/htroot/api/ymarks/get_xbel.java b/htroot/api/ymarks/get_xbel.java new file mode 100644 index 000000000..90f07f914 --- /dev/null +++ b/htroot/api/ymarks/get_xbel.java @@ -0,0 +1,80 @@ +import java.io.IOException; +import java.util.Iterator; + +import net.yacy.cora.protocol.RequestHeader; +import net.yacy.document.parser.html.CharacterCoding; +import net.yacy.kelondro.blob.Tables; +import net.yacy.kelondro.logging.Log; +import de.anomic.data.BookmarkHelper; +import de.anomic.data.YMarkIndex; +import de.anomic.data.YMarkTables; +import de.anomic.data.userDB; +import de.anomic.search.Switchboard; +import de.anomic.server.serverObjects; +import de.anomic.server.serverSwitch; + +public class get_xbel { + public static final String ROOT = "root"; + public static final String SOURCE = "source"; + + static serverObjects prop; + + public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) { + final Switchboard sb = (Switchboard) env; + prop = new serverObjects(); + final userDB.Entry user = sb.userDB.getUser(header); + final boolean isAdmin = (sb.verifyAuthentication(header, true)); + final boolean isAuthUser = user!= null && user.hasRight(userDB.Entry.BOOKMARK_RIGHT); + + if(isAdmin || isAuthUser) { + final String bmk_user = (isAuthUser ? user.getUserName() : YMarkTables.USER_ADMIN); + + String root = YMarkTables.FOLDERS_ROOT; + String[] foldername = null; + + if (post != null){ + if (post.containsKey(ROOT)) { + if (post.get(ROOT).equals(SOURCE) || post.get(ROOT).equals(YMarkTables.FOLDERS_ROOT)) { + root = ""; + } else if (post.get(ROOT).startsWith(YMarkTables.FOLDERS_ROOT)) { + root = post.get(ROOT); + } else { + root = YMarkTables.FOLDERS_ROOT + post.get(ROOT); + } + } + } + + Iterator it = null; + Tables.Row bmk_row = null; + int count = 0; + + // + // loop through folderList + try { + it = sb.tables.bookmarks.folders.getFolders(bmk_user, root); + } catch (IOException e) { + Log.logException(e); + } + int n = YMarkIndex.getFolderDepth(root);; + + + while (it.hasNext()) { + String folder = it.next(); + foldername = folder.split(YMarkTables.FOLDERS_SEPARATOR); + if (foldername.length == n+1) { + prop.put("xbel_"+count+"_elements", ""); + count++; + prop.put("xbel_"+count+"_elements", "" + CharacterCoding.unicode2xml(foldername[n], true) + ""); + count++; + // print bookmars + } + } + + } else { + prop.put(YMarkTables.USER_AUTHENTICATE,YMarkTables.USER_AUTHENTICATE_MSG); + } + // return rewrite properties + return prop; + } +} diff --git a/htroot/api/ymarks/get_xbel.xml b/htroot/api/ymarks/get_xbel.xml new file mode 100644 index 000000000..69b9a7b45 --- /dev/null +++ b/htroot/api/ymarks/get_xbel.xml @@ -0,0 +1,8 @@ + + +#(style)#::#(/style)# + + #{xbel}# + #[elements]# + #{/xbel}# + \ No newline at end of file diff --git a/htroot/api/ymarks/get_ymark_list.java b/htroot/api/ymarks/get_ymark_list.java deleted file mode 100644 index 7b83e0e57..000000000 --- a/htroot/api/ymarks/get_ymark_list.java +++ /dev/null @@ -1,91 +0,0 @@ -import java.io.IOException; -import java.util.Iterator; -import java.util.TreeSet; - -import net.yacy.cora.protocol.RequestHeader; -import net.yacy.kelondro.blob.Tables; -import net.yacy.kelondro.index.RowSpaceExceededException; -import net.yacy.kelondro.logging.Log; -import de.anomic.data.YMarkTables; -import de.anomic.data.userDB; -import de.anomic.search.Switchboard; -import de.anomic.server.serverObjects; -import de.anomic.server.serverSwitch; - - -public class get_ymark_list { - - private static Switchboard sb = null; - private static serverObjects prop = null; - - public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) { - sb = (Switchboard) env; - prop = new serverObjects(); - - boolean tags = false; - - final userDB.Entry user = sb.userDB.getUser(header); - final boolean isAdmin = (sb.verifyAuthentication(header, true)); - final boolean isAuthUser = user!= null && user.hasRight(userDB.Entry.BOOKMARK_RIGHT); - final TreeSet bookmarks = new TreeSet(); - - if(isAdmin || isAuthUser) { - final String bmk_user = (isAuthUser ? user.getUserName() : YMarkTables.USER_ADMIN); - - if(post.containsKey(YMarkTables.BOOKMARK.TAGS.key())) { - tags = true; - final String[] tagArray = YMarkTables.cleanTagsString(post.get(YMarkTables.BOOKMARK.TAGS.key())).split(YMarkTables.TAGS_SEPARATOR); - try { - bookmarks.addAll(sb.tables.bookmarks.tags.getBookmarks(bmk_user, tagArray)); - } catch (IOException e) { - Log.logException(e); - } catch (RowSpaceExceededException e) { - Log.logException(e); - } - } - if(post.containsKey(YMarkTables.BOOKMARK.FOLDERS.key())) { - final String[] folderArray = YMarkTables.cleanFoldersString(post.get(YMarkTables.BOOKMARK.FOLDERS.key())).split(YMarkTables.TAGS_SEPARATOR); - try { - if(tags) - bookmarks.retainAll(sb.tables.bookmarks.folders.getBookmarks(bmk_user, folderArray)); - else - bookmarks.addAll(sb.tables.bookmarks.folders.getBookmarks(bmk_user, folderArray)); - } catch (IOException e) { - Log.logException(e); - } catch (RowSpaceExceededException e) { - Log.logException(e); - } - } - putBookmarks(bookmarks, YMarkTables.TABLES.BOOKMARKS.tablename(bmk_user)); - - } else { - prop.put(YMarkTables.USER_AUTHENTICATE,YMarkTables.USER_AUTHENTICATE_MSG); - } - // return rewrite properties - return prop; - } - - private static void putBookmarks(final TreeSet urlSet, final String bmk_table) { - final IteratorurlIter = urlSet.iterator(); - int count = 0; - while(urlIter.hasNext()) { - final byte[] urlHash = urlIter.next().getBytes(); - Tables.Row bmk_row = null; - try { - bmk_row = sb.tables.select(bmk_table, urlHash); - if (bmk_row != null) { - prop.putXML("bookmarks_"+count+"_id", new String(urlHash)); - for (YMarkTables.BOOKMARK bmk : YMarkTables.BOOKMARK.values()) { - prop.putXML("bookmarks_"+count+"_"+bmk.key(), new String(bmk_row.get(bmk.key(),bmk.deflt()))); - } - count++; - } - } catch (IOException e) { - Log.logException(e); - } catch (RowSpaceExceededException e) { - Log.logException(e); - } - } - prop.put("bookmarks", count); - } -} diff --git a/htroot/api/ymarks/get_ymark_list.xml b/htroot/api/ymarks/get_ymark_list.xml deleted file mode 100644 index fbc05ea0b..000000000 --- a/htroot/api/ymarks/get_ymark_list.xml +++ /dev/null @@ -1,6 +0,0 @@ - - -#{bookmarks}# - -#{/bookmarks}# - \ No newline at end of file diff --git a/htroot/api/ymarks/import_html.java b/htroot/api/ymarks/import_ymark.java similarity index 94% rename from htroot/api/ymarks/import_html.java rename to htroot/api/ymarks/import_ymark.java index 941316032..80c9a9f45 100644 --- a/htroot/api/ymarks/import_html.java +++ b/htroot/api/ymarks/import_ymark.java @@ -15,7 +15,7 @@ import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; -public class import_html { +public class import_ymark { public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) { final Switchboard sb = (Switchboard) env; @@ -49,6 +49,7 @@ public class import_html { } } } + prop.put("result", "1"); } if(post.containsKey("xbelfile")){ try { @@ -73,7 +74,9 @@ public class import_html { } prop.put("result", "1"); } - } + } else { + prop.put(YMarkTables.USER_AUTHENTICATE,YMarkTables.USER_AUTHENTICATE_MSG); + } // return rewrite properties return prop; } diff --git a/htroot/api/ymarks/import_html.xml b/htroot/api/ymarks/import_ymark.xml similarity index 100% rename from htroot/api/ymarks/import_html.xml rename to htroot/api/ymarks/import_ymark.xml diff --git a/htroot/api/ymarks/import.html b/htroot/api/ymarks/test_import.html similarity index 89% rename from htroot/api/ymarks/import.html rename to htroot/api/ymarks/test_import.html index a5d1ec422..66cb6228d 100644 --- a/htroot/api/ymarks/import.html +++ b/htroot/api/ymarks/test_import.html @@ -4,7 +4,7 @@ YaCy Bookmarks -
+
Import HTML Bookmarks
@@ -33,7 +33,7 @@
-
+
Import XBEL Bookmarks
diff --git a/htroot/api/ymarks/test.html b/htroot/api/ymarks/test_treeview.html similarity index 94% rename from htroot/api/ymarks/test.html rename to htroot/api/ymarks/test_treeview.html index 5e4e39747..c0d729f8d 100644 --- a/htroot/api/ymarks/test.html +++ b/htroot/api/ymarks/test_treeview.html @@ -10,7 +10,7 @@ // getFolders(final String user, final String root) throws IOException { final String index_table = user + this.table_basename; final TreeSet folders = new TreeSet(); final Pattern r = Pattern.compile(PATTERN_PREFIX + root + PATTERN_POSTFIX); final Iterator it = this.worktables.iterator(index_table, INDEX.NAME.key(), r); - - String path = ""; + final StringBuilder path = new StringBuilder(100); Row folder; while (it.hasNext()) { folder = it.next(); - path = new String(folder.get(INDEX.NAME.key(), INDEX.NAME.deflt())); - while(path.length() > 0 && !path.equals(root)){ - folders.add(path); - path = path.replaceAll(PATTERN_REPLACE, ""); + path.setLength(0); + path.append(new String(folder.get(INDEX.NAME.key(), INDEX.NAME.deflt()))); + //TODO: get rid of .toString.equals() + while(path.length() > 0 && !path.toString().equals(root)){ + folders.add(path.toString()); + path.setLength(path.lastIndexOf(YMarkTables.FOLDERS_SEPARATOR)); } } if (!root.equals(YMarkTables.FOLDERS_ROOT)) { folders.add(root); } diff --git a/source/de/anomic/data/YMarkStatics.java b/source/de/anomic/data/YMarkStatics.java deleted file mode 100644 index 6040b02ed..000000000 --- a/source/de/anomic/data/YMarkStatics.java +++ /dev/null @@ -1,136 +0,0 @@ -package de.anomic.data; - -import java.net.MalformedURLException; -import java.util.HashSet; -import java.util.Iterator; -import java.util.Set; - -import net.yacy.kelondro.data.meta.DigestURI; -import net.yacy.kelondro.data.word.Word; - -public class YMarkStatics { - - public final static String TABLE_BOOKMARKS_BASENAME = "_bookmarks"; - public final static String TABLE_BOOKMARKS_LOG = "BOOKMARKS"; - - public final static String TABLE_BOOKMARKS_USER_ADMIN = "admin"; - public final static String TABLE_BOOKMARKS_USER_AUTHENTICATE = "AUTHENTICATE"; - public final static String TABLE_BOOKMARKS_USER_AUTHENTICATE_MSG = "Authentication required!"; - - public final static String TABLE_BOOKMARKS_URL_PROTOCOL_HTTP = "http://"; - public final static String TABLE_BOOKMARKS_URL_PROTOCOL_HTTPS = "https://"; - - public final static String TABLE_BOOKMARKS_COL_ID = "id"; - public final static String TABLE_BOOKMARKS_COL_URL = "url"; - public final static String TABLE_BOOKMARKS_COL_TITLE = "title"; - public final static String TABLE_BOOKMARKS_COL_DESC = "desc"; - public final static String TABLE_BOOKMARKS_COL_DATE_ADDED = "added"; - public final static String TABLE_BOOKMARKS_COL_DATE_MODIFIED = "modified"; - public final static String TABLE_BOOKMARKS_COL_DATE_VISITED = "visited"; - public final static String TABLE_BOOKMARKS_COL_PUBLIC = "public"; - public final static String TABLE_BOOKMARKS_COL_TAGS = "tags"; - - public final static String TABLE_BOOKMARKS_COL_DEFAULT = ""; - public final static String TABLE_BOOKMARKS_COL_PUBLIC_TRUE = "true"; - public final static String TABLE_BOOKMARKS_COL_PUBLIC_FALSE = "false"; - - public final static String TABLE_TAGS_BASENAME = "_tags"; - public final static String TABLE_TAGS_SEPARATOR = ","; - - public final static String TABLE_TAGS_COL_ID = "id"; - public final static String TABLE_TAGS_COL_TAG = "tag"; - public final static String TABLE_TAGS_COL_URLS = "urls"; - - public final static int TABLE_TAGS_ACTION_ADD = 1; - public final static int TABLE_TAGS_ACTION_REMOVE = 2; - - public final static byte[] getBookmarkId(String url) throws MalformedURLException { - if (!url.toLowerCase().startsWith(TABLE_BOOKMARKS_URL_PROTOCOL_HTTP) && !url.toLowerCase().startsWith(TABLE_BOOKMARKS_URL_PROTOCOL_HTTPS)) { - url=TABLE_BOOKMARKS_URL_PROTOCOL_HTTP+url; - } - return (new DigestURI(url, null)).hash(); - } - - public final static byte[] getTagHash(final String tag) { - return Word.word2hash(tag.toLowerCase()); - } - - public final static HashSet getTagSet(final String tagsString, boolean clean) { - HashSettagSet = new HashSet(); - final String[] tagArray = clean ? cleanTagsString(tagsString).split(TABLE_TAGS_SEPARATOR) : tagsString.split(TABLE_TAGS_SEPARATOR); - for (final String tag : tagArray) { - tagSet.add(tag); - } - return tagSet; - } - - public final static HashSet getTagSet(final String tagsString) { - return getTagSet(tagsString, true); - } - - public final static HashSet getTagIdSet(final String tagsString, boolean clean) { - HashSettagSet = new HashSet(); - final String[] tagArray = clean ? cleanTagsString(tagsString).split(TABLE_TAGS_SEPARATOR) : tagsString.split(TABLE_TAGS_SEPARATOR); - for (final String tag : tagArray) { - tagSet.add(getTagHash(tag)); - } - return tagSet; - } - - public final static Set getTagIdSet(final String tagsString) { - return getTagIdSet(tagsString, true); - } - - public final static byte[] keySetToBytes(final HashSet urlSet) { - final Iterator urlIter = urlSet.iterator(); - final - StringBuilder urls = new StringBuilder(); - while(urlIter.hasNext()) { - urls.append(TABLE_TAGS_SEPARATOR); - urls.append(urlIter.next()); - } - urls.deleteCharAt(0); - return urls.toString().getBytes(); - } - - public final static HashSet keysStringToKeySet(final String keysString) { - HashSet keySet = new HashSet(); - final String[] keyArray = keysString.split(TABLE_TAGS_SEPARATOR); - for (final String key : keyArray) { - keySet.add(key); - } - return keySet; - } - - public final static String cleanTagsString(String tagsString) { - // get rid of heading, trailing and double commas since they are useless - while (tagsString.length() > 0 && tagsString.charAt(0) == TABLE_TAGS_SEPARATOR.charAt(0)) { - tagsString = tagsString.substring(1); - } - while (tagsString.endsWith(TABLE_TAGS_SEPARATOR)) { - tagsString = tagsString.substring(0,tagsString.length() -1); - } - while (tagsString.contains(",,")){ - tagsString = tagsString.replaceAll(",,", TABLE_TAGS_SEPARATOR); - } - // space characters following a comma are removed - tagsString = tagsString.replaceAll(",\\s+", TABLE_TAGS_SEPARATOR); - return tagsString; - } - - public final static String cleanFoldersString(String foldersString) { - foldersString = cleanTagsString(foldersString); - // get rid of double and trailing slashes - while (foldersString.endsWith("/")){ - foldersString = foldersString.substring(0, foldersString.length() -1); - } - while (foldersString.contains("/,")){ - foldersString = foldersString.replaceAll("/,", TABLE_TAGS_SEPARATOR); - } - while (foldersString.contains("//")){ - foldersString = foldersString.replaceAll("//", "/"); - } - return foldersString; - } - -} diff --git a/source/de/anomic/data/YMarkTables.java b/source/de/anomic/data/YMarkTables.java index 1e02e86da..cde593d9a 100644 --- a/source/de/anomic/data/YMarkTables.java +++ b/source/de/anomic/data/YMarkTables.java @@ -2,6 +2,9 @@ package de.anomic.data; import java.io.IOException; import java.net.MalformedURLException; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Date; import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; @@ -13,6 +16,7 @@ import net.yacy.kelondro.blob.Tables.Data; import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.data.word.Word; import net.yacy.kelondro.index.RowSpaceExceededException; +import net.yacy.kelondro.logging.Log; public class YMarkTables { @@ -50,7 +54,7 @@ public class YMarkTables { return this.protocol+s; } } - + public static enum BOOKMARK { URL ("url", "", "href", "href", "link"), TITLE ("title", "", "", "", "meta"), @@ -113,16 +117,14 @@ public class YMarkTables { public final static String FOLDERS_ROOT = "/"; public final static String FOLDERS_UNSORTED = "/unsorted"; public final static String FOLDERS_IMPORTED = "/imported"; - + public static final int FOLDER_BUFFER_SIZE = 100; + public final static String BOOKMARKS_LOG = "BOOKMARKS"; public final static String BOOKMARKS_ID = "id"; public final static String USER_ADMIN = "admin"; public final static String USER_AUTHENTICATE = "AUTHENTICATE"; public final static String USER_AUTHENTICATE_MSG = "Authentication required!"; - - - private WorkTables worktables; public YMarkIndex tags; @@ -134,6 +136,21 @@ public class YMarkTables { this.tags = new YMarkIndex(this.worktables, TABLES.TAGS.basename()); } + public static Date parseISO8601(final String s) throws ParseException { + if(s == null) + throw new ParseException("parseISO8601 - NPE", 0); + StringBuilder date = new StringBuilder(s); + SimpleDateFormat dateformat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssz"); + if(date.charAt(date.length()) == 'Z') { + date.deleteCharAt(date.length()); + date.append("GMT-00:00"); + } else { + date.insert(date.length()-6, "GMT"); + } + Log.logInfo(YMarkTables.BOOKMARKS_LOG, "ISO8601: "+s+" =? "+dateformat.toString()); + return dateformat.parse(date.toString()); + } + public final static byte[] getBookmarkId(String url) throws MalformedURLException { return (new DigestURI(url, null)).hash(); } @@ -166,7 +183,7 @@ public class YMarkTables { } return keySet; } - + public final static String cleanTagsString(final String tagsString) { StringBuilder ts = new StringBuilder(tagsString); // get rid of double commas and space characters following a comma @@ -244,14 +261,14 @@ public class YMarkTables { switch(b) { case DATE_ADDED: case DATE_MODIFIED: - if(bmk.containsKey(b.key())) { + if(bmk.containsKey(b.key()) && bmk.get(b.key()) != null) { data.put(b.key(), bmk.get(b.key())); } else { data.put(b.key(), String.valueOf(System.currentTimeMillis()).getBytes()); } break; case TAGS: - if(bmk.containsKey(b.key())) { + if(bmk.containsKey(b.key()) && bmk.get(b.key()) != null) { this.tags.insertIndexEntry(bmk_user, bmk.get(b.key()), urlHash); data.put(b.key(), bmk.get(b.key())); } else { @@ -260,7 +277,7 @@ public class YMarkTables { } break; case FOLDERS: - if(bmk.containsKey(b.key())) { + if(bmk.containsKey(b.key()) && bmk.get(b.key()) != null) { this.folders.insertIndexEntry(bmk_user, bmk.get(b.key()), urlHash); data.put(b.key(), bmk.get(b.key())); } else { @@ -269,7 +286,7 @@ public class YMarkTables { } break; default: - if(bmk.containsKey(b.key())) { + if(bmk.containsKey(b.key()) && bmk.get(b.key()) != null) { data.put(b.key(), bmk.get(b.key())); } } diff --git a/source/de/anomic/data/YMarksHTMLImporter.java b/source/de/anomic/data/YMarksHTMLImporter.java index 6d90d5790..da5f802aa 100644 --- a/source/de/anomic/data/YMarksHTMLImporter.java +++ b/source/de/anomic/data/YMarksHTMLImporter.java @@ -29,7 +29,7 @@ public class YMarksHTMLImporter extends HTMLEditorKit.ParserCallback implements private STATE state; private HTML.Tag prevTag; private HashMap bmk; - private String folder; + private StringBuilder folder; private final InputStream input; private final BlockingQueue> bookmarks; @@ -39,7 +39,8 @@ public class YMarksHTMLImporter extends HTMLEditorKit.ParserCallback implements this.state = STATE.NOTHING; this.prevTag = null; this.bmk = new HashMap(); - this.folder = YMarkTables.FOLDERS_IMPORTED; + this.folder = new StringBuilder(YMarkTables.FOLDER_BUFFER_SIZE); + this.folder.append(YMarkTables.FOLDERS_IMPORTED); this.bookmarks = new ArrayBlockingQueue>(queueSize); this.input = input; this.htmlParser = new ParserDelegator(); @@ -70,12 +71,13 @@ public class YMarksHTMLImporter extends HTMLEditorKit.ParserCallback implements break; case BOOKMARK: this.bmk.put(YMarkTables.BOOKMARK.TITLE.key(), new String(data)); - this.bmk.put(YMarkTables.BOOKMARK.FOLDERS.key(), this.folder); + this.bmk.put(YMarkTables.BOOKMARK.FOLDERS.key(), this.folder.toString()); this.bmk.put(YMarkTables.BOOKMARK.PUBLIC.key(), YMarkTables.BOOKMARK.PUBLIC.deflt()); this.bmk.put(YMarkTables.BOOKMARK.VISITS.key(), YMarkTables.BOOKMARK.VISITS.deflt()); break; case FOLDER: - this.folder = this.folder + YMarkTables.FOLDERS_SEPARATOR + new String(data); + this.folder.append(YMarkTables.FOLDERS_SEPARATOR); + this.folder.append(data); break; case FOLDER_DESC: Log.logInfo(YMarkTables.BOOKMARKS_LOG, "YMarksHTMLImporter - folder: "+this.folder+" desc: "+new String(data)); @@ -133,9 +135,10 @@ public class YMarksHTMLImporter extends HTMLEditorKit.ParserCallback implements if (t == HTML.Tag.H3) { state = STATE.FOLDER_DESC; } else if (t == HTML.Tag.DL) { - if(!folder.equals(YMarkTables.FOLDERS_IMPORTED)) { - folder = folder.replaceAll(YMarkIndex.PATTERN_REPLACE, ""); - } + //TODO: get rid of .toString.equals() + if(!this.folder.toString().equals(YMarkTables.FOLDERS_IMPORTED)) { + folder.setLength(folder.lastIndexOf(YMarkTables.FOLDERS_SEPARATOR)); + } } else { state = STATE.NOTHING; } diff --git a/source/de/anomic/data/YMarksXBELImporter.java b/source/de/anomic/data/YMarksXBELImporter.java index 9b7f0a40c..8d37fa506 100644 --- a/source/de/anomic/data/YMarksXBELImporter.java +++ b/source/de/anomic/data/YMarksXBELImporter.java @@ -2,6 +2,7 @@ package de.anomic.data; import java.io.IOException; import java.io.InputStream; +import java.text.ParseException; import java.util.HashMap; import java.util.concurrent.ArrayBlockingQueue; @@ -19,6 +20,7 @@ import org.xml.sax.helpers.DefaultHandler; public class YMarksXBELImporter extends DefaultHandler implements Runnable { public static enum XBEL { + NOTHING, XBEL, TITLE, DESC, @@ -34,29 +36,23 @@ public class YMarksXBELImporter extends DefaultHandler implements Runnable { } } - public static enum STATE { - NOTHING, - BOOKMARK, - INFO, - FOLDER, - FOLDER_DESC - } - private HashMap bmk; - private boolean parsingValue; - private STATE state; - private String keyname; - private String folder; - private final InputStream input; + private XBEL outer_state; // BOOKMARK, FOLDER, NOTHING + private XBEL inner_state; // DESC, TITLE, INFO, ALIAS, (METADATA), NOTHING + private boolean parse_value; private final StringBuilder buffer; + private final StringBuilder folder; + + private final InputStream input; private final ArrayBlockingQueue> bookmarks; private final SAXParser saxParser; public YMarksXBELImporter (final InputStream input, int queueSize) throws IOException { - this.buffer = new StringBuilder(); this.bmk = null; - this.folder = YMarkTables.FOLDERS_IMPORTED; + this.buffer = new StringBuilder(); + this.folder = new StringBuilder(YMarkTables.FOLDER_BUFFER_SIZE); + this.folder.append(YMarkTables.FOLDERS_IMPORTED); this.bookmarks = new ArrayBlockingQueue>(queueSize); this.input = input; final SAXParserFactory factory = SAXParserFactory.newInstance(); @@ -95,47 +91,61 @@ public class YMarksXBELImporter extends DefaultHandler implements Runnable { } public void startElement(final String uri, final String name, String tag, final Attributes atts) throws SAXException { - if (tag == null) return; + String date; + if (tag == null) return; tag = tag.toLowerCase(); if (XBEL.BOOKMARK.tag().equals(tag)) { this.bmk = new HashMap(); this.bmk.put(YMarkTables.BOOKMARK.URL.key(), atts.getValue(uri, YMarkTables.BOOKMARK.URL.xbel_attrb())); - this.bmk.put(YMarkTables.BOOKMARK.DATE_ADDED.key(), atts.getValue(uri, YMarkTables.BOOKMARK.DATE_ADDED.xbel_attrb())); - this.bmk.put(YMarkTables.BOOKMARK.DATE_VISITED.key(), atts.getValue(uri, YMarkTables.BOOKMARK.DATE_VISITED.xbel_attrb())); - this.bmk.put(YMarkTables.BOOKMARK.DATE_MODIFIED.key(), atts.getValue(uri, YMarkTables.BOOKMARK.DATE_MODIFIED.xbel_attrb())); - state = STATE.BOOKMARK; - this.parsingValue = false; + try { + date = String.valueOf(YMarkTables.parseISO8601(atts.getValue(uri, YMarkTables.BOOKMARK.DATE_ADDED.xbel_attrb()))); + } catch (ParseException e) { + date = String.valueOf(System.currentTimeMillis()); + } + this.bmk.put(YMarkTables.BOOKMARK.DATE_ADDED.key(), date); + try { + date = String.valueOf(YMarkTables.parseISO8601(atts.getValue(uri, YMarkTables.BOOKMARK.DATE_VISITED.xbel_attrb()))); + this.bmk.put(YMarkTables.BOOKMARK.DATE_VISITED.key(), date); + } catch (ParseException e) { + } + try { + date = String.valueOf(YMarkTables.parseISO8601(atts.getValue(uri, YMarkTables.BOOKMARK.DATE_MODIFIED.xbel_attrb()))); + } catch (ParseException e) { + date = String.valueOf(System.currentTimeMillis()); + } + this.bmk.put(YMarkTables.BOOKMARK.DATE_MODIFIED.key(), date); + outer_state = XBEL.BOOKMARK; + inner_state = XBEL.NOTHING; + this.parse_value = false; } else if(XBEL.FOLDER.tag().equals(tag)) { - this.state = STATE.FOLDER; + this.outer_state = XBEL.FOLDER; + this.inner_state = XBEL.NOTHING; } else if (XBEL.DESC.tag().equals(tag)) { - if(this.state == STATE.FOLDER) { - this.keyname = null; - this.state = STATE.FOLDER_DESC; - } else if (this.state == STATE.BOOKMARK) { - this.keyname = YMarkTables.BOOKMARK.DESC.key(); - } else { - Log.logInfo(YMarkTables.BOOKMARKS_LOG, "YMarksXBELImporter - state: "+this.state+" tag: "+tag); - this.parsingValue = false; - return; - } - this.parsingValue = true; + this.inner_state = XBEL.DESC; + this.parse_value = true; } else if (XBEL.TITLE.tag().equals(tag)) { - if(this.state == STATE.FOLDER) { - this.keyname = null; - } else if (this.state == STATE.BOOKMARK) { - this.keyname = YMarkTables.BOOKMARK.TITLE.key(); - } else { - Log.logInfo(YMarkTables.BOOKMARKS_LOG, "YMarksXBELImporter - state: "+this.state+" tag: "+tag); - this.parsingValue = false; - return; - } - this.parsingValue = true; + this.inner_state = XBEL.TITLE; + this.parse_value = true; } else if (XBEL.INFO.tag().equals(tag)) { - this.parsingValue = false; - this.state = STATE.INFO; - } else { - this.parsingValue = false; - this.state = STATE.NOTHING; + this.inner_state = XBEL.INFO; + this.parse_value = false; + } else if (XBEL.METADATA.tag().equals(tag)) { + /* + this.meta_owner = atts.getValue(uri, "owner"); + this.inner_state = XBEL.METADATA; + this.parse_value = true; + */ + } else if (XBEL.ALIAS.tag().equals(tag)) { + /* + this.alias_ref = atts.getValue(uri, "ref"); + this.inner_state = XBEL.ALIAS; + this.parse_value = false; + */ + } + else { + this.outer_state = XBEL.NOTHING; + this.inner_state = XBEL.NOTHING; + this.parse_value = false; } } @@ -146,7 +156,7 @@ public class YMarksXBELImporter extends DefaultHandler implements Runnable { if(XBEL.BOOKMARK.tag().equals(tag)) { // write bookmark if (!this.bmk.isEmpty()) { - this.bmk.put(YMarkTables.BOOKMARK.FOLDERS.key(), this.folder); + this.bmk.put(YMarkTables.BOOKMARK.FOLDERS.key(), this.folder.toString()); try { this.bookmarks.put(this.bmk); bmk = new HashMap(); @@ -154,32 +164,60 @@ public class YMarksXBELImporter extends DefaultHandler implements Runnable { Log.logException(e); } } - this.state = STATE.FOLDER; + this.outer_state = XBEL.FOLDER; } else if (XBEL.FOLDER.tag().equals(tag)) { - this.state = STATE.NOTHING; // go up one folder - if(!folder.equals(YMarkTables.FOLDERS_IMPORTED)) { - folder = folder.replaceAll(YMarkIndex.PATTERN_REPLACE, ""); - this.state = STATE.FOLDER; - } + //TODO: get rid of .toString.equals() + if(!this.folder.toString().equals(YMarkTables.FOLDERS_IMPORTED)) { + folder.setLength(folder.lastIndexOf(YMarkTables.FOLDERS_SEPARATOR)); + } + this.outer_state = XBEL.FOLDER; } else if (XBEL.INFO.tag().equals(tag)) { - this.state = STATE.BOOKMARK; + this.inner_state = XBEL.NOTHING; + } else if (XBEL.METADATA.tag().equals(tag)) { + this.inner_state = XBEL.INFO; } } public void characters(final char ch[], final int start, final int length) { - if (parsingValue) { + if (parse_value) { buffer.append(ch, start, length); - if (this.state == STATE.BOOKMARK) { - this.bmk.put(this.keyname, this.buffer.toString()); - } else if (this.state == STATE.FOLDER) { - this.folder = this.folder + YMarkTables.FOLDERS_SEPARATOR + this.buffer.toString(); - } else if (this.state == STATE.FOLDER_DESC) { - Log.logInfo(YMarkTables.BOOKMARKS_LOG, "YMarksXBELImporter - folder: "+this.folder+" desc: "+this.buffer.toString()); - this.state = STATE.FOLDER; - } - this.buffer.setLength(0); - this.parsingValue = false; + switch(outer_state) { + case BOOKMARK: + switch(inner_state) { + case DESC: + this.bmk.put(YMarkTables.BOOKMARK.DESC.key(), this.buffer.toString()); + break; + case TITLE: + this.bmk.put(YMarkTables.BOOKMARK.TITLE.key(), this.buffer.toString()); + break; + case METADATA: + // this.meta_data = this.buffer.toString(); + break; + default: + break; + } + break; + case FOLDER: + switch(inner_state) { + case DESC: + break; + case TITLE: + this.folder.append(YMarkTables.FOLDERS_SEPARATOR); + this.folder.append(this.buffer); + break; + case METADATA: + // this.meta_data = this.buffer.toString(); + break; + default: + break; + } + break; + default: + break; + } + this.buffer.setLength(0); + this.parse_value = false; } }