diff --git a/htroot/Table_YMark_p.html b/htroot/Table_YMark_p.html new file mode 100644 index 000000000..1f2b89503 --- /dev/null +++ b/htroot/Table_YMark_p.html @@ -0,0 +1,202 @@ + + + + YaCy '#[clientname]#': Table Viewer + #(showtable)#:: + + #(/showtable)# + #%env/templates/metas.template%# + + + + + #%env/templates/header.template%# + #%env/templates/submenuConfig.template%# + #(showselection)#:: +

YMark Table Administration

+
+ #(/showselection)# +
+ #(showtable)#:: + +
+ +
+

+ + + + + + + +

+
+
+ + + + + #{columns}# + + #{/columns}# + + #{list}# + + + + #{columns}# + + #{/columns}# + + #{/list}# +
PK#[header]#
#[pk]##[cell]#
+
+
+ + #(/showtable)# + #(showedit)#:: +
+
+
Row Editor +
+
Primary Key
+
#[pk]#
+ #{list}# +
#[key]#
+
+ +
+ #{/list}# +
+
+ + + +
+
+
+
+
+ #(/showedit)# +
+ #(showselection)#:: +
+
Table Selection +
+
+ +
+
+ +
+
+ +
+
+ + entries +
+
+ +
+
+ +
+
+
+
+ Search Table +
+
+ +
+
+ + +
+
+
+
+ Filter Table +
+ +
+
+ +
+
+ +
+
+ +
+ +
+
+ Import Bookmarks +
+
+ +
+
+ +
+
+ +
+
+ + +
+
+ +
+ +
+ #(/showselection)# + #%env/templates/footer.template%# + + diff --git a/htroot/Table_YMark_p.java b/htroot/Table_YMark_p.java new file mode 100644 index 000000000..b5030a407 --- /dev/null +++ b/htroot/Table_YMark_p.java @@ -0,0 +1,255 @@ +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.regex.Pattern; + +import net.yacy.cora.protocol.RequestHeader; +import net.yacy.kelondro.blob.Tables; +import net.yacy.kelondro.index.RowSpaceExceededException; +import net.yacy.kelondro.logging.Log; +import de.anomic.data.YMarkTables; +import de.anomic.search.Switchboard; +import de.anomic.server.serverObjects; +import de.anomic.server.serverSwitch; + + +public class Table_YMark_p { + public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) { + final Switchboard sb = (Switchboard) env; + final serverObjects prop = new serverObjects(); + + prop.put("showtable", 0); + prop.put("showedit", 0); + prop.put("showselection", 0); + + String table = (post == null) ? null : post.get("table", null); + if (table != null && !sb.tables.hasHeap(table)) table = null; + + // get the user name for the selected table + String bmk_user = null; + if (table != null) + bmk_user = table.substring(0,table.indexOf('_')); + + // show table selection + int count = 0; + Iterator ti = sb.tables.tables(); + String tablename; + prop.put("showselection", 1); + while (ti.hasNext()) { + tablename = ti.next(); + if(tablename.endsWith(YMarkTables.TABLES.BOOKMARKS.basename())) { + prop.put("showselection_tables_" + count + "_name", tablename); + prop.put("showselection_tables_" + count + "_selected", (table != null && table.equals(tablename)) ? 1 : 0); + count++; + } + } + prop.put("showselection_tables", count); + prop.put("showselection_pattern", ""); + + if (post == null) return prop; // return rewrite properties + + // get available tags and folders + count = 0; + byte[] key; + try { + Iterator iter = sb.tables.keys(YMarkTables.TABLES.TAGS.tablename(bmk_user)); + while(iter.hasNext()) { + key = iter.next(); + prop.put("showselection_tags_" + count + "_tagHash", new String(key)); + prop.put("showselection_tags_" + count + "_tagName", sb.tables.bookmarks.tags.getKeyname(bmk_user, key)); + count++; + } + prop.put("showselection_tags", count); + count = 0; + iter = sb.tables.keys(YMarkTables.TABLES.FOLDERS.tablename(bmk_user)); + while(iter.hasNext()) { + key = iter.next(); + prop.put("showselection_folders_" + count + "_folderHash", new String(key)); + prop.put("showselection_folders_" + count + "_folderName", sb.tables.bookmarks.folders.getKeyname(bmk_user, key)); + count++; + } + prop.put("showselection_folders", count); + } catch (IOException e) { + Log.logException(e); + } catch (RowSpaceExceededException e) { + Log.logException(e); + } + + String counts = post.get("count", null); + int maxcount = (counts == null || counts.equals("all")) ? Integer.MAX_VALUE : Integer.parseInt(counts); + String pattern = post.get("search", ""); + Pattern matcher = (pattern.length() == 0 || pattern.equals(".*")) ? null : Pattern.compile(".*" + pattern + ".*"); + prop.put("pattern", pattern); + + List columns = null; + if (table != null) try { + columns = sb.tables.columns(table); + } catch (IOException e) { + Log.logException(e); + columns = new ArrayList(); + } + + final Iterator cit = columns.iterator(); + count = 0; + while(cit.hasNext()) { + prop.put("showselection_columns_" + count + "_col", cit.next()); + count++; + } + prop.put("showselection_columns", count); + + // apply deletion requests + if (post.get("deletetable", "").length() > 0) try { + sb.tables.clear(table); + sb.tables.clear(YMarkTables.TABLES.FOLDERS.tablename(bmk_user)); + sb.tables.clear(YMarkTables.TABLES.TAGS.tablename(bmk_user)); + } catch (IOException e) { + Log.logException(e); + } + + if (post.get("deleterows", "").length() > 0) { + for (Map.Entry entry: post.entrySet()) { + if (entry.getValue().startsWith("mark_")) try { + sb.tables.bookmarks.deleteBookmark(bmk_user, entry.getValue().substring(5).getBytes()); + } catch (IOException e) { + Log.logException(e); + } catch (RowSpaceExceededException e) { + Log.logException(e); + } + } + } + + if (post.get("commitrow", "").length() > 0) { + final HashMap bmk = new HashMap(); + for (Map.Entry entry: post.entrySet()) { + if (entry.getKey().startsWith("col_")) { + bmk.put(entry.getKey().substring(4), entry.getValue()); + } + } + try { + sb.tables.bookmarks.addBookmark(bmk_user, bmk, false); + } catch (IOException e) { + Log.logException(e); + } catch (RowSpaceExceededException e) { + Log.logException(e); + } + } + + // generate table + prop.put("showtable", 0); + prop.put("showedit", 0); + + if (table != null) { + + if (post.containsKey("editrow")) { + // check if we can find a key + String pk = null; + for (Map.Entry entry: post.entrySet()) { + if (entry.getValue().startsWith("mark_")) { + pk = entry.getValue().substring(5); + break; + } + } + try { + if (pk != null && sb.tables.has(table, pk.getBytes())) { + setEdit(sb, prop, table, pk, columns); + } + } catch (IOException e) { + Log.logException(e); + } catch (RowSpaceExceededException e) { + Log.logException(e); + } + } else if (post.containsKey("addrow")) try { + // get a new key + String pk = new String(sb.tables.createRow(table)); + setEdit(sb, prop, table, pk, columns); + } catch (IOException e) { + Log.logException(e); + } catch (RowSpaceExceededException e) { + Log.logException(e); + } else { + prop.put("showtable", 1); + prop.put("showtable_table", table); + + // insert the columns + + for (int i = 0; i < columns.size(); i++) { + prop.putHTML("showtable_columns_" + i + "_header", columns.get(i)); + } + prop.put("showtable_columns", columns.size()); + + // insert all rows + try { + maxcount = Math.min(maxcount, sb.tables.size(table)); + } catch (IOException e) { + Log.logException(e); + maxcount = 0; + } + count = 0; + try { + Iterator mapIterator; + + if(post.containsKey("folders") && !post.get("folders").isEmpty()) { + mapIterator = sb.tables.bookmarks.folders.getBookmarks(bmk_user, post.get("folders")); + } else if(post.containsKey("tags") && !post.get("tags").isEmpty()) { + mapIterator = sb.tables.bookmarks.tags.getBookmarks(bmk_user, post.get("tags")); + } else { + final Iterator plainIterator = sb.tables.iterator(table, matcher); + mapIterator = sb.tables.orderByPK(plainIterator, maxcount).iterator(); + } + + Tables.Row row; + boolean dark = true; + byte[] cell; + while (mapIterator.hasNext() && count < maxcount) { + row = mapIterator.next(); + if (row == null) continue; + + // write table content + prop.put("showtable_list_" + count + "_dark", ((dark) ? 1 : 0) ); dark=!dark; + prop.put("showtable_list_" + count + "_pk", new String(row.getPK())); + prop.put("showtable_list_" + count + "_count", count); + for (int i = 0; i < columns.size(); i++) { + cell = row.get(columns.get(i)); + prop.putHTML("showtable_list_" + count + "_columns_" + i + "_cell", cell == null ? "" : new String(cell)); + } + prop.put("showtable_list_" + count + "_columns", columns.size()); + count++; + } + } catch (IOException e) { + Log.logException(e); + } catch (RowSpaceExceededException e) { + Log.logException(e); + } + prop.put("showtable_list", count); + prop.put("showtable_num", count); + } + + } + + // adding the peer address + prop.put("address", sb.peers.mySeed().getPublicAddress()); + + // return rewrite properties + return prop; + } + + private static void setEdit(final Switchboard sb, final serverObjects prop, final String table, final String pk, List columns) throws IOException, RowSpaceExceededException { + prop.put("showedit", 1); + prop.put("showedit_table", table); + prop.put("showedit_pk", pk); + Tables.Row row = sb.tables.select(table, pk.getBytes()); + if (row == null) return; + int count = 0; + byte[] cell; + for (String col: columns) { + cell = row.get(col); + prop.put("showedit_list_" + count + "_key", col); + prop.put("showedit_list_" + count + "_value", cell == null ? "" : new String(cell)); + count++; + } + prop.put("showedit_list", count); + } +} diff --git a/htroot/api/ymarks/get_treeview.java b/htroot/api/ymarks/get_treeview.java index de1a27a07..37a0f90b4 100644 --- a/htroot/api/ymarks/get_treeview.java +++ b/htroot/api/ymarks/get_treeview.java @@ -77,7 +77,7 @@ public class get_treeview { } // loop through bookmarkList try { - it = sb.tables.bookmarks.folders.getBookmarks(bmk_user, root).iterator(); + it = sb.tables.bookmarks.folders.getBookmarkIds(bmk_user, root).iterator(); while (it.hasNext()) { final String urlHash = new String(it.next()); bmk_row = sb.tables.select(YMarkTables.TABLES.BOOKMARKS.tablename(bmk_user), urlHash.getBytes()); diff --git a/htroot/api/ymarks/get_xbel.java b/htroot/api/ymarks/get_xbel.java index 54ca1ae1b..17bba203a 100644 --- a/htroot/api/ymarks/get_xbel.java +++ b/htroot/api/ymarks/get_xbel.java @@ -65,8 +65,7 @@ public class get_xbel { while (fit.hasNext()) { String folder = fit.next(); - foldername = folder.split(YMarkTables.FOLDERS_SEPARATOR); - Log.logInfo(YMarkTables.BOOKMARKS_LOG, "folder: "+folder+" n: "+n+" folder_length: "+foldername.length); + foldername = folder.split(YMarkTables.FOLDERS_SEPARATOR); if (n != root_depth && foldername.length <= n) { prop.put("xbel_"+count+"_elements", ""); count++; @@ -80,7 +79,7 @@ public class get_xbel { count++; } try { - bit = sb.tables.bookmarks.folders.getBookmarks(bmk_user, folder).iterator(); + bit = sb.tables.bookmarks.folders.getBookmarkIds(bmk_user, folder).iterator(); Tables.Row bmk_row = null; String urlHash; while(bit.hasNext()){ diff --git a/htroot/api/ymarks/get_ymark.java b/htroot/api/ymarks/get_ymark.java index 35698c4f9..40996af11 100644 --- a/htroot/api/ymarks/get_ymark.java +++ b/htroot/api/ymarks/get_ymark.java @@ -36,7 +36,7 @@ public class get_ymark { tags = true; final String[] tagArray = YMarkTables.cleanTagsString(post.get(YMarkTables.BOOKMARK.TAGS.key())).split(YMarkTables.TAGS_SEPARATOR); try { - bookmarks.addAll(sb.tables.bookmarks.tags.getBookmarks(bmk_user, tagArray)); + bookmarks.addAll(sb.tables.bookmarks.tags.getBookmarkIds(bmk_user, tagArray)); } catch (IOException e) { Log.logException(e); } catch (RowSpaceExceededException e) { @@ -47,9 +47,9 @@ public class get_ymark { final String[] folderArray = YMarkTables.cleanFoldersString(post.get(YMarkTables.BOOKMARK.FOLDERS.key())).split(YMarkTables.TAGS_SEPARATOR); try { if(tags) - bookmarks.retainAll(sb.tables.bookmarks.folders.getBookmarks(bmk_user, folderArray)); + bookmarks.retainAll(sb.tables.bookmarks.folders.getBookmarkIds(bmk_user, folderArray)); else - bookmarks.addAll(sb.tables.bookmarks.folders.getBookmarks(bmk_user, folderArray)); + bookmarks.addAll(sb.tables.bookmarks.folders.getBookmarkIds(bmk_user, folderArray)); } catch (IOException e) { Log.logException(e); } catch (RowSpaceExceededException e) { diff --git a/htroot/api/ymarks/import_ymark.java b/htroot/api/ymarks/import_ymark.java index f4f6225b1..f6040ccef 100644 --- a/htroot/api/ymarks/import_ymark.java +++ b/htroot/api/ymarks/import_ymark.java @@ -2,8 +2,6 @@ import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.UnsupportedEncodingException; import java.util.HashMap; -import java.util.Iterator; - import org.xml.sax.SAXException; import net.yacy.cora.protocol.RequestHeader; @@ -62,7 +60,8 @@ public class import_ymark { } else if(post.get("importer").equals("xbel") && byteIn != null) { final YMarksXBELImporter xbelImporter; try { - xbelImporter = new YMarksXBELImporter(byteIn, 100); + //TODO: make RootFold + xbelImporter = new YMarksXBELImporter(byteIn, 100, YMarkTables.FOLDERS_IMPORTED); } catch (SAXException e) { //TODO: display an error message Log.logException(e); @@ -83,21 +82,6 @@ public class import_ymark { prop.put("result", "0"); return prop; } - } - // update bookmarks with aliases - final Iterator> it = xbelImporter.getAliases().iterator(); - while (it.hasNext()) { - try { - sb.tables.bookmarks.addBookmark(bmk_user, it.next(), true); - } catch (IOException e) { - Log.logWarning(YMarkTables.BOOKMARKS_LOG.toString(), "XBEL Importer - IOException for URL: "+bmk.get(YMarkTables.BOOKMARK.URL.key())); - continue; - } catch (RowSpaceExceededException e) { - //TODO: display an error message - Log.logException(e); - prop.put("result", "0"); - return prop; - } } prop.put("result", "1"); } diff --git a/source/de/anomic/data/YMarkIndex.java b/source/de/anomic/data/YMarkIndex.java index 52db17983..9144a8d35 100644 --- a/source/de/anomic/data/YMarkIndex.java +++ b/source/de/anomic/data/YMarkIndex.java @@ -127,7 +127,7 @@ public class YMarkIndex { } } - public HashSet getBookmarks(final String user, final String keyname) throws IOException, RowSpaceExceededException { + public HashSet getBookmarkIds(final String user, final String keyname) throws IOException, RowSpaceExceededException { final String index_table = user + this.table_basename; final String cacheKey = index_table+":"+keyname; if (this.cache.containsKey(cacheKey)) { @@ -143,14 +143,23 @@ public class YMarkIndex { return new HashSet(); } - public HashSet getBookmarks(final String user, final String[] keyArray) throws IOException, RowSpaceExceededException { + public Iterator getBookmarks(final String user, final String keyname) throws IOException, RowSpaceExceededException { + final Iterator bit = getBookmarkIds(user, keyname).iterator(); + final HashSet bookmarks = new HashSet(); + while(bit.hasNext()) { + bookmarks.add(this.worktables.select(YMarkTables.TABLES.BOOKMARKS.tablename(user), bit.next().getBytes())); + } + return bookmarks.iterator(); + } + + public HashSet getBookmarkIds(final String user, final String[] keyArray) throws IOException, RowSpaceExceededException { final HashSet urlSet = new HashSet(); - urlSet.addAll(getBookmarks(user, keyArray[0])); + urlSet.addAll(getBookmarkIds(user, keyArray[0])); if (urlSet.isEmpty()) return urlSet; if (keyArray.length > 1) { for (final String keyname : keyArray) { - urlSet.retainAll(getBookmarks(user, keyname)); + urlSet.retainAll(getBookmarkIds(user, keyname)); if (urlSet.isEmpty()) return urlSet; } diff --git a/source/de/anomic/data/YMarkTables.java b/source/de/anomic/data/YMarkTables.java index e282986c0..f1f290a6d 100644 --- a/source/de/anomic/data/YMarkTables.java +++ b/source/de/anomic/data/YMarkTables.java @@ -33,8 +33,8 @@ public class YMarkTables { public String basename() { return this.basename; } - public String tablename(String user) { - return user+this.basename; + public String tablename(String bmk_user) { + return bmk_user+this.basename; } } @@ -129,7 +129,7 @@ public class YMarkTables { public final static String FOLDERS_SEPARATOR = "/"; public final static String FOLDERS_ROOT = "/"; public final static String FOLDERS_UNSORTED = "/unsorted"; - public final static String FOLDERS_IMPORTED = ""; + public final static String FOLDERS_IMPORTED = "/imported"; public static final int FOLDER_BUFFER_SIZE = 100; public final static String BOOKMARKS_LOG = "BOOKMARKS"; @@ -150,11 +150,11 @@ public class YMarkTables { } public static Date parseISO8601(final String s) throws ParseException { - StringBuilder date = new StringBuilder(s); - SimpleDateFormat dateformat; - if(s == null || s.isEmpty()) { + if(s == null || s.isEmpty()) { throw new ParseException("parseISO8601 - empty string, nothing to parse", 0); } + SimpleDateFormat dateformat; + StringBuilder date = new StringBuilder(s); if(s.length()==10) dateformat = new SimpleDateFormat("yyyy-MM-dd"); else { diff --git a/source/de/anomic/data/YMarksXBELImporter.java b/source/de/anomic/data/YMarksXBELImporter.java index 87413e926..015fe6f7e 100644 --- a/source/de/anomic/data/YMarksXBELImporter.java +++ b/source/de/anomic/data/YMarksXBELImporter.java @@ -74,17 +74,23 @@ public class YMarksXBELImporter extends DefaultHandler implements Runnable { private final InputSource input; private final ArrayBlockingQueue> bookmarks; private final XMLReader xmlReader; + private final String RootFolder; - public YMarksXBELImporter (final InputStream input, int queueSize) throws SAXException { + public YMarksXBELImporter (final InputStream input, int queueSize, String root) throws SAXException { this.bmk = null; - this.buffer = new StringBuilder(); + this.RootFolder = root; + + this.buffer = new StringBuilder(); this.foldersString = new StringBuilder(YMarkTables.FOLDER_BUFFER_SIZE); this.folder = new StringBuilder(YMarkTables.FOLDER_BUFFER_SIZE); - this.folder.append(YMarkTables.FOLDERS_IMPORTED); - this.bmkRef = new HashMap>(); + + this.folder.append(this.RootFolder); + + this.input = new InputSource(input); + this.bmkRef = new HashMap>(); this.aliasRef = new HashSet>(); this.bookmarks = new ArrayBlockingQueue>(queueSize); - this.input = new InputSource(input); + this.xmlReader = XMLReaderFactory.createXMLReader(); this.xmlReader.setContentHandler(this); this.xmlReader.setFeature("http://xml.org/sax/features/namespace-prefixes", false); @@ -103,6 +109,7 @@ public class YMarksXBELImporter extends DefaultHandler implements Runnable { Log.logException(e); } finally { try { + Log.logInfo(YMarkTables.BOOKMARKS_LOG, "XBEL Importer inserted poison pill in queue"); this.bookmarks.put(YMarkTables.POISON); } catch (InterruptedException e1) { Log.logException(e1); @@ -111,8 +118,11 @@ public class YMarksXBELImporter extends DefaultHandler implements Runnable { } public void endDocument() throws SAXException { - // no need to keep the bookmark references any longer - // this.bmkRef.clear(); + // put alias references in the bookmark queue to ensure that folders get updated + // we do that at endDocument to ensure all referenced bookmarks already exist + this.bookmarks.addAll(this.aliasRef); + this.aliasRef.clear(); + this.bmkRef.clear(); } public void startElement(final String uri, final String name, String tag, final Attributes atts) throws SAXException { @@ -140,7 +150,6 @@ public class YMarksXBELImporter extends DefaultHandler implements Runnable { date = String.valueOf(System.currentTimeMillis()); } this.bmk.put(YMarkTables.BOOKMARK.DATE_MODIFIED.key(), date); - UpdateBmkRef(atts.getValue(uri, "id"), true); outer_state = XBEL.BOOKMARK; inner_state = XBEL.NOTHING; @@ -162,7 +171,6 @@ public class YMarksXBELImporter extends DefaultHandler implements Runnable { atts.getValue(uri, "owner"); */ } else if (XBEL.ALIAS.tag().equals(tag)) { - Log.logInfo(YMarkTables.BOOKMARKS_LOG, "ALIAS: "+this.ref.get(YMarkTables.BOOKMARK.URL.key())); final String r = atts.getValue(uri, "ref"); UpdateBmkRef(r, false); this.aliasRef.add(this.bmkRef.get(r)); @@ -192,8 +200,8 @@ public class YMarksXBELImporter extends DefaultHandler implements Runnable { } else if (XBEL.FOLDER.tag().equals(tag)) { // go up one folder //TODO: get rid of .toString.equals() - if(!this.folder.toString().equals(YMarkTables.FOLDERS_IMPORTED)) { - folder.setLength(folder.lastIndexOf(YMarkTables.FOLDERS_SEPARATOR)); + if(!this.folder.toString().equals(this.RootFolder)) { + folder.setLength(folder.lastIndexOf(YMarkTables.FOLDERS_SEPARATOR)); } this.outer_state = XBEL.FOLDER; } else if (XBEL.INFO.tag().equals(tag)) { @@ -205,7 +213,7 @@ public class YMarksXBELImporter extends DefaultHandler implements Runnable { public void characters(final char ch[], final int start, final int length) { if (parse_value) { - buffer.append(ch, start, length); + buffer.append(ch, start, length); switch(outer_state) { case BOOKMARK: switch(inner_state) { @@ -254,10 +262,6 @@ public class YMarksXBELImporter extends DefaultHandler implements Runnable { } } - public HashSet> getAliases() { - return this.aliasRef; - } - private void UpdateBmkRef(final String id, final boolean url) { this.foldersString.setLength(0);