package de.anomic.data; import java.io.IOException; import java.net.MalformedURLException; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import net.yacy.cora.storage.ConcurrentARC; import net.yacy.kelondro.blob.Tables; import net.yacy.kelondro.blob.Tables.Data; import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.data.word.Word; import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.logging.Log; public class YMarkTables { public static enum TABLES { BOOKMARKS ("_bookmarks"), TAGS ("_tags"), FOLDERS ("_folders"); private String basename; private TABLES(String b) { this.basename = b; } public String basename() { return this.basename; } public String tablename(String user) { return user+this.basename; } } public static enum PROTOCOLS { HTTP ("http://"), HTTPS ("https://"); private String protocol; private PROTOCOLS(String s) { this.protocol = s; } public String protocol() { return this.protocol; } public String protocol(String s) { return this.protocol+s; } } public static enum BOOKMARK { URL ("url", "", "href", "href"), TITLE ("title", "", "", ""), DESC ("desc", "", "", ""), DATE_ADDED ("date_added", "", "add_date", "added"), DATE_MODIFIED ("date_modified", "", "last_modified", "modified"), DATE_VISITED ("date_visited", "", "last_visited", "visited"), PUBLIC ("public", "flase", "", ""), TAGS ("tags", "unsorted", "shortcuturl", ""), VISITS ("visits", "0", "", ""), FOLDERS ("folders", "/unsorted", "", ""); private String key; private String dflt; private String html_attrb; private String xbel_attrb; private BOOKMARK(String k, String s, String a, String x) { this.key = k; this.dflt = s; this.html_attrb = a; this.xbel_attrb = x; } public String key() { return this.key; } public String deflt() { return this.dflt; } public String html_attrb() { return this.html_attrb; } public String xbel_attrb() { return this.xbel_attrb; } } public static enum INDEX { ID ("id", ""), NAME ("name", ""), DESC ("desc", ""), URLS ("urls", ""); private String key; private String dflt; private INDEX(String k, String s) { this.key = k; this.dflt = s; } public String key() { return this.key; } public String deflt() { return this.dflt; } public byte[] b_deflt() { return dflt.getBytes(); } } public static enum INDEX_ACTION { ADD, REMOVE } public final static HashMap POISON = new HashMap(); public final static String TAGS_SEPARATOR = ","; public final static String FOLDERS_SEPARATOR = "/"; public final static String FOLDERS_ROOT = "/"; public final static String FOLDERS_UNSORTED = "/unsorted"; public final static String FOLDERS_IMPORTED = "/imported"; public final static String BOOKMARKS_LOG = "BOOKMARKS"; public final static String BOOKMARKS_ID = "id"; public final static String USER_ADMIN = "admin"; public final static String USER_AUTHENTICATE = "AUTHENTICATE"; public final static String USER_AUTHENTICATE_MSG = "Authentication required!"; private WorkTables worktables; public ConcurrentARC cache; public YMarkTables(final Tables wt) { this.worktables = (WorkTables)wt; this.cache = new ConcurrentARC(50,1); } public final static byte[] getBookmarkId(String url) throws MalformedURLException { return (new DigestURI(url, null)).hash(); } public final static byte[] getKeyId(final String tag) { return Word.word2hash(tag.toLowerCase()); } public final static byte[] keySetToBytes(final HashSet urlSet) { final Iterator urlIter = urlSet.iterator(); final StringBuilder urls = new StringBuilder(urlSet.size()*20); while(urlIter.hasNext()) { urls.append(TAGS_SEPARATOR); urls.append(urlIter.next()); } urls.deleteCharAt(0); return urls.toString().getBytes(); } public final static HashSet keysStringToSet(final String keysString) { HashSet keySet = new HashSet(); final String[] keyArray = keysString.split(TAGS_SEPARATOR); for (final String key : keyArray) { keySet.add(key); } return keySet; } public final static String cleanTagsString(final String tagsString) { StringBuilder ts = new StringBuilder(tagsString); // get rid of double commas and space characters following a comma for (int i = 0; i < ts.length()-1; i++) { if (ts.charAt(i) == TAGS_SEPARATOR.charAt(0)) { if (ts.charAt(i+1) == TAGS_SEPARATOR.charAt(0) || ts.charAt(i+1) == ' ') { ts.deleteCharAt(i+1); i--; } } } // get rid of heading and trailing comma if (ts.charAt(0) == TAGS_SEPARATOR.charAt(0)) ts.deleteCharAt(0); if (ts.charAt(ts.length()-1) == TAGS_SEPARATOR.charAt(0)) ts.deleteCharAt(ts.length()-1); return ts.toString(); } public final static String cleanFoldersString(final String foldersString) { StringBuilder fs = new StringBuilder(cleanTagsString(foldersString)); for (int i = 0; i < fs.length()-1; i++) { if (fs.charAt(i) == FOLDERS_SEPARATOR.charAt(0)) { if (fs.charAt(i+1) == TAGS_SEPARATOR.charAt(0) || fs.charAt(i+1) == FOLDERS_SEPARATOR.charAt(0)) { fs.deleteCharAt(i); i--; } else if (fs.charAt(i+1) == ' ') { fs.deleteCharAt(i+1); i--; } } } if (fs.charAt(fs.length()-1) == FOLDERS_SEPARATOR.charAt(0)) { fs.deleteCharAt(fs.length()-1); } return fs.toString(); } public void cleanCache(final String tablename) { final Iterator iter = this.cache.keySet().iterator(); while(iter.hasNext()) { final String key = iter.next(); if (key.startsWith(tablename)) { this.cache.remove(key); } } } public void createIndexEntry(final String index_table, final String keyname, final HashSet urlSet) throws IOException { final byte[] key = YMarkTables.getKeyId(keyname); final String cacheKey = index_table+":"+keyname; final byte[] BurlSet = keySetToBytes(urlSet); Data tagEntry = new Data(); this.cache.insert(cacheKey, BurlSet); tagEntry.put(INDEX.NAME.key, keyname); tagEntry.put(INDEX.URLS.key, BurlSet); this.worktables.insert(index_table, key, tagEntry); } public HashSet getBookmarks(final String index_table, final String keyname) throws IOException, RowSpaceExceededException { final String cacheKey = index_table+":"+keyname; if (this.cache.containsKey(cacheKey)) { return keysStringToSet(new String(this.cache.get(cacheKey))); } else { final Tables.Row idx_row = this.worktables.select(index_table, getKeyId(keyname)); if (idx_row != null) { final byte[] keys = idx_row.get(INDEX.URLS.key); this.cache.put(cacheKey, keys); return keysStringToSet(new String(keys)); } } return new HashSet(); } public HashSet getBookmarks(final String index_table, final String[] keyArray) throws IOException, RowSpaceExceededException { final HashSet urlSet = new HashSet(); urlSet.addAll(getBookmarks(index_table, keyArray[0])); if (urlSet.isEmpty()) return urlSet; if (keyArray.length > 1) { for (final String keyname : keyArray) { urlSet.retainAll(getBookmarks(index_table, keyname)); if (urlSet.isEmpty()) return urlSet; } } return urlSet; } /** * YMark function that updates the tag/folder index * @param index_table is the user specific index * @param keyname * @param url is the url has as returned by DigestURI.hash() * @param action is either add (1) or remove (2) */ public void updateIndexTable(final String index_table, final String keyname, final byte[] url, final INDEX_ACTION action) { final byte[] key = YMarkTables.getKeyId(keyname); final String urlHash = new String(url); Tables.Row row = null; // try to load urlSet from cache final String cacheKey = index_table+":"+keyname; HashSeturlSet = this.cache.containsKey(cacheKey) ? keysStringToSet(new String(this.cache.get(cacheKey))) : new HashSet(); try { row = this.worktables.select(index_table, key); // key has no index_table entry if(row == null) { switch (action) { case ADD: urlSet.add(urlHash); createIndexEntry(index_table, keyname, urlSet); break; case REMOVE: // key has no index_table entry but a cache entry // TODO: this shouldn't happen if(!urlSet.isEmpty()) { urlSet.remove(urlHash); createIndexEntry(index_table, keyname, urlSet); } break; default: break; } } // key has an existing index_table entry else { byte[] BurlSet = null; // key has no cache entry if (urlSet.isEmpty()) { // load urlSet from index_table urlSet = keysStringToSet(new String(row.get(INDEX.URLS.key))); } switch (action) { case ADD: urlSet.add(urlHash); break; case REMOVE: urlSet.remove(urlHash); break; default: break; } if (urlSet.isEmpty()) { this.cache.remove(cacheKey); this.worktables.delete(index_table, key); } else { BurlSet = keySetToBytes(urlSet); this.cache.insert(cacheKey, BurlSet); row.put(INDEX.URLS.key, BurlSet); this.worktables.update(index_table, row); } } } catch (IOException e) { Log.logException(e); } catch (RowSpaceExceededException e) { Log.logException(e); } } public void addBookmark(final HashMap bmk, final String bmk_user) { final String bmk_table = bmk_user + TABLES.BOOKMARKS.basename(); final String folder_table = bmk_user + TABLES.FOLDERS.basename(); final String tag_table = bmk_user + TABLES.TAGS.basename(); Tables.Row bmk_row = null; byte[] urlHash = null; try { urlHash = getBookmarkId(bmk.get(BOOKMARK.URL.key())); } catch (MalformedURLException e) { Log.logInfo(BOOKMARKS_LOG, "Malformed URL:"+bmk.get(BOOKMARK.URL.key())); return; } if (urlHash != null) { try { bmk_row = this.worktables.select(bmk_table, urlHash); } catch (IOException e) { Log.logException(e); } catch (RowSpaceExceededException e) { Log.logException(e); } if (bmk_row == null) { Data data = new Data(); for (BOOKMARK b : BOOKMARK.values()) { switch(b) { case DATE_ADDED: case DATE_MODIFIED: if(bmk.containsKey(b.key())) { data.put(b.key(), bmk.get(b.key())); } else { data.put(b.key(), String.valueOf(System.currentTimeMillis()).getBytes()); } break; case TAGS: if(bmk.containsKey(b.key())) { final String[] tagArray = bmk.get(b.key()).split(TAGS_SEPARATOR); for (final String tag : tagArray) { this.worktables.bookmarks.updateIndexTable(tag_table, tag, urlHash, INDEX_ACTION.ADD); } data.put(b.key(), bmk.get(b.key())); } break; case FOLDERS: if(bmk.containsKey(b.key())) { final String[] folderArray = bmk.get(b.key()).split(TAGS_SEPARATOR); for (final String folder : folderArray) { this.worktables.bookmarks.updateIndexTable(folder_table, folder, urlHash, INDEX_ACTION.ADD); } data.put(b.key(), bmk.get(b.key())); } break; default: if(bmk.containsKey(b.key())) { data.put(b.key(), bmk.get(b.key())); } } } try { Log.logInfo(BOOKMARKS_LOG, "Add URL:"+bmk.get(BOOKMARK.URL.key())); this.worktables.insert(bmk_table, urlHash, data); } catch (IOException e) { Log.logException(e); } } } } }