boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

YMark:

Browse Source 
- some improvements to firefox json bookmark importer
- test import with: /api/ymarks/test_import.html
- view ymarks with: /api/ymarks/test_treeview.html


git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7660 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
apfelmaennchen 14 years ago
parent 0abd99621c
commit 667e912b19
3 changed files with 91 additions and 61 deletions
Show Stats Download Patch File Download Diff File

9
htroot/api/ymarks/import_ymark.java
Unescape View File

@ -1,5 +1,6 @@
import java.io.ByteArrayInputStream; import java.io.ByteArrayInputStream;
import java.io.IOException; import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.util.HashMap; import java.util.HashMap;
import net.yacy.cora.document.UTF8; import net.yacy.cora.document.UTF8;
@ -68,7 +69,13 @@ public class import_ymark {
} }
prop.put("result", "1"); prop.put("result", "1");
} else if(post.get("importer").equals("json") && byteIn != null) { } else if(post.get("importer").equals("json") && byteIn != null) {
final YMarkJSONImporter jsonImporter = new YMarkJSONImporter(byteIn, 10); YMarkJSONImporter jsonImporter;
try {
jsonImporter = new YMarkJSONImporter(byteIn, 10);
} catch (UnsupportedEncodingException e) {
prop.put("result", "1");
return prop;
}
t = new Thread(jsonImporter, "YMarks - JSON Importer"); t = new Thread(jsonImporter, "YMarks - JSON Importer");
t.start(); t.start();
while ((bmk = jsonImporter.take()) != YMarkTables.POISON) { while ((bmk = jsonImporter.take()) != YMarkTables.POISON) {

51
source/de/anomic/data/ymark/YMarkJSONImporter.java
Unescape View File

@ -4,6 +4,7 @@ import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.io.InputStreamReader; import java.io.InputStreamReader;
import java.io.Reader; import java.io.Reader;
import java.io.UnsupportedEncodingException;
import java.util.HashMap; import java.util.HashMap;
import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.ArrayBlockingQueue;
@ -17,36 +18,29 @@ public class YMarkJSONImporter implements Runnable, ContentHandler{
public final static String FOLDER = "text/x-moz-place-container"; public final static String FOLDER = "text/x-moz-place-container";
public final static String BOOKMARK = "text/x-moz-place"; public final static String BOOKMARK = "text/x-moz-place";
public final static String ANNOS = "annos";
public static enum JSON_KEY { public final static String TYPE = "type";
annos, public final static String CHILDREN = "children";
type, private final static String MILLIS = "000";
title,
children,
keyword,
dateAdded,
lastModified,
uri;
}
private final JSONParser parser; private final JSONParser parser;
private final ArrayBlockingQueue<HashMap<String,String>> bookmarks;
private final Reader json; private final Reader json;
private final StringBuilder folderstring; private final StringBuilder folderstring;
private final StringBuilder value; private final StringBuilder value;
private final StringBuilder key; private final StringBuilder key;
private final HashMap<String,String> obj; private final HashMap<String,String> obj;
private final ArrayBlockingQueue<HashMap<String,String>> bookmarks;
private HashMap<String,String> bmk; private HashMap<String,String> bmk;
private int depth; private int depth;
private Boolean isFolder; private Boolean isFolder;
private Boolean isBookmark; private Boolean isBookmark;
private Boolean isAnnos; private Boolean isAnnos;
public YMarkJSONImporter(final InputStream input, int queueSize) { public YMarkJSONImporter(final InputStream input, int queueSize) throws UnsupportedEncodingException {
this.parser = new JSONParser(); this.parser = new JSONParser();
this.bookmarks = new ArrayBlockingQueue<HashMap<String,String>>(queueSize); this.bookmarks = new ArrayBlockingQueue<HashMap<String,String>>(queueSize);
this.json = new InputStreamReader(input); this.json = new InputStreamReader(input, "UTF-8");
this.folderstring = new StringBuilder(256); this.folderstring = new StringBuilder(256);
this.key = new StringBuilder(16); this.key = new StringBuilder(16);
this.value = new StringBuilder(128); this.value = new StringBuilder(128);
@ -66,13 +60,13 @@ public class YMarkJSONImporter implements Runnable, ContentHandler{
public boolean startArray() throws ParseException, IOException { public boolean startArray() throws ParseException, IOException {
final String key = this.key.toString(); final String key = this.key.toString();
if(key.equals(JSON_KEY.children.toString()) && this.isFolder) { if(key.equals(CHILDREN) && this.isFolder) {
if(this.depth > 0) { if(this.depth > 0) {
this.folderstring.append(YMarkUtil.FOLDERS_SEPARATOR); this.folderstring.append(YMarkUtil.FOLDERS_SEPARATOR);
this.folderstring.append(this.obj.get(JSON_KEY.title.toString())); this.folderstring.append(this.obj.get(YMarkTables.BOOKMARK.TITLE.json_attrb()));
} }
this.depth++; this.depth++;
} else if(key.equals(JSON_KEY.annos.toString())) { } else if(key.equals(ANNOS)) {
this.isAnnos = true; this.isAnnos = true;
} }
return true; return true;
@ -82,7 +76,10 @@ public class YMarkJSONImporter implements Runnable, ContentHandler{
if(this.isAnnos) { if(this.isAnnos) {
this.isAnnos = false; this.isAnnos = false;
} else if(this.depth > 0) { } else if(this.depth > 0) {
folderstring.setLength(folderstring.lastIndexOf(YMarkUtil.FOLDERS_SEPARATOR)); if(this.depth == 1)
folderstring.setLength(0);
else
folderstring.setLength(folderstring.lastIndexOf(YMarkUtil.FOLDERS_SEPARATOR));
this.depth--; this.depth--;
} }
return true; return true;
@ -97,13 +94,13 @@ public class YMarkJSONImporter implements Runnable, ContentHandler{
public boolean endObject() throws ParseException, IOException { public boolean endObject() throws ParseException, IOException {
if(this.isBookmark) { if(this.isBookmark) {
this.bmk.put(YMarkTables.BOOKMARK.TITLE.key(),obj.get(JSON_KEY.title.toString())); this.bmk.put(YMarkTables.BOOKMARK.TITLE.key(),obj.get(YMarkTables.BOOKMARK.TITLE.json_attrb()));
this.bmk.put(YMarkTables.BOOKMARK.URL.key(),obj.get(JSON_KEY.uri.toString())); this.bmk.put(YMarkTables.BOOKMARK.URL.key(),obj.get(YMarkTables.BOOKMARK.URL.json_attrb()));
this.bmk.put(YMarkTables.BOOKMARK.DATE_ADDED.key(),obj.get(JSON_KEY.dateAdded.toString())); this.bmk.put(YMarkTables.BOOKMARK.DATE_ADDED.key(),obj.get(YMarkTables.BOOKMARK.DATE_ADDED.json_attrb())+MILLIS);
this.bmk.put(YMarkTables.BOOKMARK.DATE_MODIFIED.key(),obj.get(JSON_KEY.lastModified.toString())); this.bmk.put(YMarkTables.BOOKMARK.DATE_MODIFIED.key(),obj.get(YMarkTables.BOOKMARK.DATE_MODIFIED.json_attrb())+MILLIS);
this.bmk.put(YMarkTables.BOOKMARK.FOLDERS.key(),this.folderstring.toString()); this.bmk.put(YMarkTables.BOOKMARK.FOLDERS.key(),this.folderstring.toString());
if(this.obj.containsKey(JSON_KEY.keyword.toString())) { if(this.obj.containsKey(YMarkTables.BOOKMARK.TAGS.json_attrb())) {
this.bmk.put(YMarkTables.BOOKMARK.TAGS.key(),obj.get(JSON_KEY.keyword.toString())); this.bmk.put(YMarkTables.BOOKMARK.TAGS.key(),obj.get(YMarkTables.BOOKMARK.TAGS.json_attrb()));
} }
try { try {
this.bookmarks.put(this.bmk); this.bookmarks.put(this.bmk);
@ -142,7 +139,7 @@ public class YMarkJSONImporter implements Runnable, ContentHandler{
if(!this.isAnnos) { if(!this.isAnnos) {
final String key = this.key.toString(); final String key = this.key.toString();
final String value = this.value.toString(); final String value = this.value.toString();
if(key.equals(JSON_KEY.type.toString())) { if(key.equals(TYPE)) {
if(value.equals(FOLDER)) { if(value.equals(FOLDER)) {
this.isFolder = true; this.isFolder = true;
} else if(value.equals(BOOKMARK)) { } else if(value.equals(BOOKMARK)) {

92
source/de/anomic/data/ymark/YMarkTables.java
Unescape View File

@ -43,7 +43,7 @@ import net.yacy.kelondro.index.RowSpaceExceededException;
public class YMarkTables { public class YMarkTables {
public static enum TABLES { public static enum TABLES {
BOOKMARKS ("_bookmarks"), BOOKMARKS ("_bookmarks"),
TAGS ("_tags"), TAGS ("_tags"),
FOLDERS ("_folders"); FOLDERS ("_folders");
@ -77,24 +77,25 @@ public class YMarkTables {
return this.protocol+s; return this.protocol+s;
} }
} }
public static enum BOOKMARK { public static enum BOOKMARK {
// key dflt html_attrb xbel_attrb type // key dflt html_attrb xbel_attrb json_attrb type
URL ("url", "", "href", "href", "link"), URL ("url", "", "href", "href", "uri", "link"),
TITLE ("title", "", "", "", "meta"), TITLE ("title", "", "", "", "title", "meta"),
DESC ("desc", "", "", "", "comment"), DESC ("desc", "", "", "", "", "comment"),
DATE_ADDED ("date_added", "", "add_date", "added", "date"), DATE_ADDED ("date_added", "", "add_date", "added", "dateAdded", "date"),
DATE_MODIFIED ("date_modified", "", "last_modified", "modified", "date"), DATE_MODIFIED ("date_modified", "", "last_modified", "modified", "lastModified", "date"),
DATE_VISITED ("date_visited", "", "last_visited", "visited", "date"), DATE_VISITED ("date_visited", "", "last_visited", "visited", "", "date"),
PUBLIC ("public", "flase", "", "yacy:public", "lock"), PUBLIC ("public", "flase", "", "yacy:public", "", "lock"),
TAGS ("tags", "unsorted", "shortcuturl", "yacy:tags", "tag"), TAGS ("tags", "unsorted", "shortcuturl", "yacy:tags", "keyword", "tag"),
VISITS ("visits", "0", "", "yacy:visits", "stat"), VISITS ("visits", "0", "", "yacy:visits", "", "stat"),
FOLDERS ("folders", "/unsorted", "", "", "folder"); FOLDERS ("folders", "/unsorted", "", "", "", "folder");
private String key; private String key;
private String dflt; private String dflt;
private String html_attrb; private String html_attrb;
private String xbel_attrb; private String xbel_attrb;
private String json_attrb;
private String type; private String type;
private static final Map<String,BOOKMARK> lookup = new HashMap<String,BOOKMARK>(); private static final Map<String,BOOKMARK> lookup = new HashMap<String,BOOKMARK>();
@ -105,11 +106,12 @@ public class YMarkTables {
private static StringBuilder buffer = new StringBuilder(25);; private static StringBuilder buffer = new StringBuilder(25);;
private BOOKMARK(String k, String s, String a, String x, String t) { private BOOKMARK(String k, String s, String a, String x, String j, String t) {
this.key = k; this.key = k;
this.dflt = s; this.dflt = s;
this.html_attrb = a; this.html_attrb = a;
this.xbel_attrb = x; this.xbel_attrb = x;
this.json_attrb = j;
this.type = t; this.type = t;
} }
public static BOOKMARK get(String key) { public static BOOKMARK get(String key) {
@ -130,6 +132,9 @@ public class YMarkTables {
public String xbel_attrb() { public String xbel_attrb() {
return this.xbel_attrb; return this.xbel_attrb;
} }
public String json_attrb() {
return this.json_attrb;
}
public String xbel() { public String xbel() {
buffer.setLength(0); buffer.setLength(0);
buffer.append('"'); buffer.append('"');
@ -159,10 +164,21 @@ public class YMarkTables {
public final static String USER_AUTHENTICATE = "AUTHENTICATE"; public final static String USER_AUTHENTICATE = "AUTHENTICATE";
public final static String USER_AUTHENTICATE_MSG = "Authentication required!"; public final static String USER_AUTHENTICATE_MSG = "Authentication required!";
private WorkTables worktables; public final static String p1 = "(?:^|.*,)";
public final static String p2 = "\\Q";
public final static String p3 = "\\E";
public final static String p4 = "(?:,.*|$)";
public final static String p5 = "((?:";
public final static String p6 = "),*.*){";
public final static String p7 = "/.*)";
public final static String p8 = "(?:,|$)";
private final WorkTables worktables;
private final StringBuffer patternBuilder;
public YMarkTables(final Tables wt) { public YMarkTables(final Tables wt) {
this.worktables = (WorkTables)wt; this.worktables = (WorkTables)wt;
this.patternBuilder = new StringBuffer(512);
} }
public void deleteBookmark(final String bmk_user, final byte[] urlHash) throws IOException, RowSpaceExceededException { public void deleteBookmark(final String bmk_user, final byte[] urlHash) throws IOException, RowSpaceExceededException {
@ -180,7 +196,13 @@ public class YMarkTables {
public TreeSet<String> getFolders(final String bmk_user, final String root) throws IOException { public TreeSet<String> getFolders(final String bmk_user, final String root) throws IOException {
final String bmk_table = TABLES.BOOKMARKS.tablename(bmk_user); final String bmk_table = TABLES.BOOKMARKS.tablename(bmk_user);
final Pattern r = Pattern.compile("(?:^|.*,)("+root+"/.*)(?:,|$)"); this.patternBuilder.setLength(0);
this.patternBuilder.append(p1);
this.patternBuilder.append('(');
this.patternBuilder.append(root);
this.patternBuilder.append(p7);
this.patternBuilder.append(p8);
final Pattern r = Pattern.compile(this.patternBuilder.toString());
final Iterator<Tables.Row> bit = this.worktables.iterator(bmk_table, YMarkTables.BOOKMARK.FOLDERS.key(), r); final Iterator<Tables.Row> bit = this.worktables.iterator(bmk_table, YMarkTables.BOOKMARK.FOLDERS.key(), r);
final TreeSet<String> folders = new TreeSet<String>(); final TreeSet<String> folders = new TreeSet<String>();
final StringBuilder path = new StringBuilder(200); final StringBuilder path = new StringBuilder(200);
@ -210,30 +232,34 @@ public class YMarkTables {
public Iterator<Tables.Row> getBookmarksByFolder(final String bmk_user, final String folder) throws IOException { public Iterator<Tables.Row> getBookmarksByFolder(final String bmk_user, final String folder) throws IOException {
final String bmk_table = TABLES.BOOKMARKS.tablename(bmk_user); final String bmk_table = TABLES.BOOKMARKS.tablename(bmk_user);
final StringBuffer buffer = new StringBuffer(folder.length()+30); this.patternBuilder.setLength(0);
buffer.append("(?:^|.*,)(\\Q"); this.patternBuilder.append(p1);
buffer.append(folder); this.patternBuilder.append('(');
buffer.append("\\E)(?:,|$)"); this.patternBuilder.append(p2);
final Pattern p = Pattern.compile(buffer.toString()); this.patternBuilder.append(folder);
this.patternBuilder.append(p3);
this.patternBuilder.append(')');
this.patternBuilder.append(p4);
final Pattern p = Pattern.compile(this.patternBuilder.toString());
return this.worktables.iterator(bmk_table, YMarkTables.BOOKMARK.FOLDERS.key(), p); return this.worktables.iterator(bmk_table, YMarkTables.BOOKMARK.FOLDERS.key(), p);
} }
public Iterator<Tables.Row> getBookmarksByTag(final String bmk_user, final String[] tagArray) throws IOException { public Iterator<Tables.Row> getBookmarksByTag(final String bmk_user, final String[] tagArray) throws IOException {
// "(?:^|.*,)((?:tag4|tag2|tag5),*.*){3}"
final String bmk_table = TABLES.BOOKMARKS.tablename(bmk_user); final String bmk_table = TABLES.BOOKMARKS.tablename(bmk_user);
final StringBuffer buffer = new StringBuffer((tagArray.length * 25)+25); this.patternBuilder.setLength(0);
buffer.append("(?:^|.*,)((?:"); this.patternBuilder.append(p1);
this.patternBuilder.append(p5);
for (final String tag : tagArray) { for (final String tag : tagArray) {
buffer.append("\\Q"); this.patternBuilder.append(p2);
buffer.append(tag); this.patternBuilder.append(tag);
buffer.append("\\E"); this.patternBuilder.append(p3);
buffer.append("|"); this.patternBuilder.append('|');
} }
buffer.deleteCharAt(buffer.length()-1); this.patternBuilder.deleteCharAt(this.patternBuilder.length()-1);
buffer.append("),*.*){"); this.patternBuilder.append(p6);
buffer.append(tagArray.length); this.patternBuilder.append(tagArray.length);
buffer.append("}"); this.patternBuilder.append('}');
final Pattern p = Pattern.compile(buffer.toString()); final Pattern p = Pattern.compile(this.patternBuilder.toString());
return this.worktables.iterator(bmk_table, YMarkTables.BOOKMARK.TAGS.key(), p); return this.worktables.iterator(bmk_table, YMarkTables.BOOKMARK.TAGS.key(), p);
} }

Write Preview
Loading…
Cancel
Save