boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

YMark:

Browse Source 
- some improvements to firefox json bookmark importer
- test import with: /api/ymarks/test_import.html
- view ymarks with: /api/ymarks/test_treeview.html


git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7660 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
apfelmaennchen 14 years ago
parent 0abd99621c
commit 667e912b19
3 changed files with 91 additions and 61 deletions
Show Stats Download Patch File Download Diff File

9
htroot/api/ymarks/import_ymark.java
Unescape View File

@ -1,5 +1,6 @@
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.util.HashMap;
import net.yacy.cora.document.UTF8;
@ -68,7 +69,13 @@ public class import_ymark {
}
prop.put("result", "1");
} else if(post.get("importer").equals("json") && byteIn != null) {
final YMarkJSONImporter jsonImporter = new YMarkJSONImporter(byteIn, 10);
YMarkJSONImporter jsonImporter;
try {
jsonImporter = new YMarkJSONImporter(byteIn, 10);
} catch (UnsupportedEncodingException e) {
prop.put("result", "1");
return prop;
}
t = new Thread(jsonImporter, "YMarks - JSON Importer");
t.start();
while ((bmk = jsonImporter.take()) != YMarkTables.POISON) {

51
source/de/anomic/data/ymark/YMarkJSONImporter.java
Unescape View File

@ -4,6 +4,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.io.UnsupportedEncodingException;
import java.util.HashMap;
import java.util.concurrent.ArrayBlockingQueue;
@ -17,36 +18,29 @@ public class YMarkJSONImporter implements Runnable, ContentHandler{
public final static String FOLDER = "text/x-moz-place-container";
public final static String BOOKMARK = "text/x-moz-place";
public static enum JSON_KEY {
annos,
type,
title,
children,
keyword,
dateAdded,
lastModified,
uri;
}
public final static String ANNOS = "annos";
public final static String TYPE = "type";
public final static String CHILDREN = "children";
private final static String MILLIS = "000";
private final JSONParser parser;
private final ArrayBlockingQueue<HashMap<String,String>> bookmarks;
private final Reader json;
private final StringBuilder folderstring;
private final StringBuilder value;
private final StringBuilder key;
private final HashMap<String,String> obj;
private final ArrayBlockingQueue<HashMap<String,String>> bookmarks;
private HashMap<String,String> bmk;
private int depth;
private Boolean isFolder;
private Boolean isBookmark;
private Boolean isAnnos;
public YMarkJSONImporter(final InputStream input, int queueSize) {
public YMarkJSONImporter(final InputStream input, int queueSize) throws UnsupportedEncodingException {
this.parser = new JSONParser();
this.bookmarks = new ArrayBlockingQueue<HashMap<String,String>>(queueSize);
this.json = new InputStreamReader(input);
this.json = new InputStreamReader(input, "UTF-8");
this.folderstring = new StringBuilder(256);
this.key = new StringBuilder(16);
this.value = new StringBuilder(128);
@ -66,13 +60,13 @@ public class YMarkJSONImporter implements Runnable, ContentHandler{
public boolean startArray() throws ParseException, IOException {
final String key = this.key.toString();
if(key.equals(JSON_KEY.children.toString()) && this.isFolder) {
if(key.equals(CHILDREN) && this.isFolder) {
if(this.depth > 0) {
this.folderstring.append(YMarkUtil.FOLDERS_SEPARATOR);
this.folderstring.append(this.obj.get(JSON_KEY.title.toString()));
this.folderstring.append(this.obj.get(YMarkTables.BOOKMARK.TITLE.json_attrb()));
}
this.depth++;
} else if(key.equals(JSON_KEY.annos.toString())) {
} else if(key.equals(ANNOS)) {
this.isAnnos = true;
}
return true;
@ -82,7 +76,10 @@ public class YMarkJSONImporter implements Runnable, ContentHandler{
if(this.isAnnos) {
this.isAnnos = false;
} else if(this.depth > 0) {
folderstring.setLength(folderstring.lastIndexOf(YMarkUtil.FOLDERS_SEPARATOR));
if(this.depth == 1)
folderstring.setLength(0);
else
folderstring.setLength(folderstring.lastIndexOf(YMarkUtil.FOLDERS_SEPARATOR));
this.depth--;
}
return true;
@ -97,13 +94,13 @@ public class YMarkJSONImporter implements Runnable, ContentHandler{
public boolean endObject() throws ParseException, IOException {
if(this.isBookmark) {
this.bmk.put(YMarkTables.BOOKMARK.TITLE.key(),obj.get(JSON_KEY.title.toString()));
this.bmk.put(YMarkTables.BOOKMARK.URL.key(),obj.get(JSON_KEY.uri.toString()));
this.bmk.put(YMarkTables.BOOKMARK.DATE_ADDED.key(),obj.get(JSON_KEY.dateAdded.toString()));
this.bmk.put(YMarkTables.BOOKMARK.DATE_MODIFIED.key(),obj.get(JSON_KEY.lastModified.toString()));
this.bmk.put(YMarkTables.BOOKMARK.TITLE.key(),obj.get(YMarkTables.BOOKMARK.TITLE.json_attrb()));
this.bmk.put(YMarkTables.BOOKMARK.URL.key(),obj.get(YMarkTables.BOOKMARK.URL.json_attrb()));
this.bmk.put(YMarkTables.BOOKMARK.DATE_ADDED.key(),obj.get(YMarkTables.BOOKMARK.DATE_ADDED.json_attrb())+MILLIS);
this.bmk.put(YMarkTables.BOOKMARK.DATE_MODIFIED.key(),obj.get(YMarkTables.BOOKMARK.DATE_MODIFIED.json_attrb())+MILLIS);
this.bmk.put(YMarkTables.BOOKMARK.FOLDERS.key(),this.folderstring.toString());
if(this.obj.containsKey(JSON_KEY.keyword.toString())) {
this.bmk.put(YMarkTables.BOOKMARK.TAGS.key(),obj.get(JSON_KEY.keyword.toString()));
if(this.obj.containsKey(YMarkTables.BOOKMARK.TAGS.json_attrb())) {
this.bmk.put(YMarkTables.BOOKMARK.TAGS.key(),obj.get(YMarkTables.BOOKMARK.TAGS.json_attrb()));
}
try {
this.bookmarks.put(this.bmk);
@ -142,7 +139,7 @@ public class YMarkJSONImporter implements Runnable, ContentHandler{
if(!this.isAnnos) {
final String key = this.key.toString();
final String value = this.value.toString();
if(key.equals(JSON_KEY.type.toString())) {
if(key.equals(TYPE)) {
if(value.equals(FOLDER)) {
this.isFolder = true;
} else if(value.equals(BOOKMARK)) {

92
source/de/anomic/data/ymark/YMarkTables.java
Unescape View File

@ -43,7 +43,7 @@ import net.yacy.kelondro.index.RowSpaceExceededException;
public class YMarkTables {
public static enum TABLES {
public static enum TABLES {
BOOKMARKS ("_bookmarks"),
TAGS ("_tags"),
FOLDERS ("_folders");
@ -77,24 +77,25 @@ public class YMarkTables {
return this.protocol+s;
}
}
public static enum BOOKMARK {
// key dflt html_attrb xbel_attrb type
URL ("url", "", "href", "href", "link"),
TITLE ("title", "", "", "", "meta"),
DESC ("desc", "", "", "", "comment"),
DATE_ADDED ("date_added", "", "add_date", "added", "date"),
DATE_MODIFIED ("date_modified", "", "last_modified", "modified", "date"),
DATE_VISITED ("date_visited", "", "last_visited", "visited", "date"),
PUBLIC ("public", "flase", "", "yacy:public", "lock"),
TAGS ("tags", "unsorted", "shortcuturl", "yacy:tags", "tag"),
VISITS ("visits", "0", "", "yacy:visits", "stat"),
FOLDERS ("folders", "/unsorted", "", "", "folder");
// key dflt html_attrb xbel_attrb json_attrb type
URL ("url", "", "href", "href", "uri", "link"),
TITLE ("title", "", "", "", "title", "meta"),
DESC ("desc", "", "", "", "", "comment"),
DATE_ADDED ("date_added", "", "add_date", "added", "dateAdded", "date"),
DATE_MODIFIED ("date_modified", "", "last_modified", "modified", "lastModified", "date"),
DATE_VISITED ("date_visited", "", "last_visited", "visited", "", "date"),
PUBLIC ("public", "flase", "", "yacy:public", "", "lock"),
TAGS ("tags", "unsorted", "shortcuturl", "yacy:tags", "keyword", "tag"),
VISITS ("visits", "0", "", "yacy:visits", "", "stat"),
FOLDERS ("folders", "/unsorted", "", "", "", "folder");
private String key;
private String dflt;
private String html_attrb;
private String xbel_attrb;
private String json_attrb;
private String type;
private static final Map<String,BOOKMARK> lookup = new HashMap<String,BOOKMARK>();
@ -105,11 +106,12 @@ public class YMarkTables {
private static StringBuilder buffer = new StringBuilder(25);;
private BOOKMARK(String k, String s, String a, String x, String t) {
private BOOKMARK(String k, String s, String a, String x, String j, String t) {
this.key = k;
this.dflt = s;
this.html_attrb = a;
this.xbel_attrb = x;
this.json_attrb = j;
this.type = t;
}
public static BOOKMARK get(String key) {
@ -130,6 +132,9 @@ public class YMarkTables {
public String xbel_attrb() {
return this.xbel_attrb;
}
public String json_attrb() {
return this.json_attrb;
}
public String xbel() {
buffer.setLength(0);
buffer.append('"');
@ -159,10 +164,21 @@ public class YMarkTables {
public final static String USER_AUTHENTICATE = "AUTHENTICATE";
public final static String USER_AUTHENTICATE_MSG = "Authentication required!";
private WorkTables worktables;
public final static String p1 = "(?:^|.*,)";
public final static String p2 = "\\Q";
public final static String p3 = "\\E";
public final static String p4 = "(?:,.*|$)";
public final static String p5 = "((?:";
public final static String p6 = "),*.*){";
public final static String p7 = "/.*)";
public final static String p8 = "(?:,|$)";
private final WorkTables worktables;
private final StringBuffer patternBuilder;
public YMarkTables(final Tables wt) {
this.worktables = (WorkTables)wt;
this.patternBuilder = new StringBuffer(512);
}
public void deleteBookmark(final String bmk_user, final byte[] urlHash) throws IOException, RowSpaceExceededException {
@ -180,7 +196,13 @@ public class YMarkTables {
public TreeSet<String> getFolders(final String bmk_user, final String root) throws IOException {
final String bmk_table = TABLES.BOOKMARKS.tablename(bmk_user);
final Pattern r = Pattern.compile("(?:^|.*,)("+root+"/.*)(?:,|$)");
this.patternBuilder.setLength(0);
this.patternBuilder.append(p1);
this.patternBuilder.append('(');
this.patternBuilder.append(root);
this.patternBuilder.append(p7);
this.patternBuilder.append(p8);
final Pattern r = Pattern.compile(this.patternBuilder.toString());
final Iterator<Tables.Row> bit = this.worktables.iterator(bmk_table, YMarkTables.BOOKMARK.FOLDERS.key(), r);
final TreeSet<String> folders = new TreeSet<String>();
final StringBuilder path = new StringBuilder(200);
@ -210,30 +232,34 @@ public class YMarkTables {
public Iterator<Tables.Row> getBookmarksByFolder(final String bmk_user, final String folder) throws IOException {
final String bmk_table = TABLES.BOOKMARKS.tablename(bmk_user);
final StringBuffer buffer = new StringBuffer(folder.length()+30);
buffer.append("(?:^|.*,)(\\Q");
buffer.append(folder);
buffer.append("\\E)(?:,|$)");
final Pattern p = Pattern.compile(buffer.toString());
this.patternBuilder.setLength(0);
this.patternBuilder.append(p1);
this.patternBuilder.append('(');
this.patternBuilder.append(p2);
this.patternBuilder.append(folder);
this.patternBuilder.append(p3);
this.patternBuilder.append(')');
this.patternBuilder.append(p4);
final Pattern p = Pattern.compile(this.patternBuilder.toString());
return this.worktables.iterator(bmk_table, YMarkTables.BOOKMARK.FOLDERS.key(), p);
}
public Iterator<Tables.Row> getBookmarksByTag(final String bmk_user, final String[] tagArray) throws IOException {
// "(?:^|.*,)((?:tag4|tag2|tag5),*.*){3}"
final String bmk_table = TABLES.BOOKMARKS.tablename(bmk_user);
final StringBuffer buffer = new StringBuffer((tagArray.length * 25)+25);
buffer.append("(?:^|.*,)((?:");
this.patternBuilder.setLength(0);
this.patternBuilder.append(p1);
this.patternBuilder.append(p5);
for (final String tag : tagArray) {
buffer.append("\\Q");
buffer.append(tag);
buffer.append("\\E");
buffer.append("|");
this.patternBuilder.append(p2);
this.patternBuilder.append(tag);
this.patternBuilder.append(p3);
this.patternBuilder.append('|');
}
buffer.deleteCharAt(buffer.length()-1);
buffer.append("),*.*){");
buffer.append(tagArray.length);
buffer.append("}");
final Pattern p = Pattern.compile(buffer.toString());
this.patternBuilder.deleteCharAt(this.patternBuilder.length()-1);
this.patternBuilder.append(p6);
this.patternBuilder.append(tagArray.length);
this.patternBuilder.append('}');
final Pattern p = Pattern.compile(this.patternBuilder.toString());
return this.worktables.iterator(bmk_table, YMarkTables.BOOKMARK.TAGS.key(), p);
}

Write Preview
Loading…
Cancel
Save