boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

YMark: code clean up and some small fixes

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7661 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
apfelmaennchen 14 years ago
parent 667e912b19
commit 62855f9567
6 changed files with 159 additions and 121 deletions
Show Stats Download Patch File Download Diff File

44
htroot/api/ymarks/import_ymark.java
Unescape View File

@ -1,5 +1,6 @@
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException;
import java.util.HashMap;
@ -33,29 +34,43 @@ public class import_ymark {
final boolean isAuthUser = user!= null && user.hasRight(UserDB.AccessRight.BOOKMARK_RIGHT);
Thread t;
HashMap<String,String> bmk;
ByteArrayInputStream byteIn = null;
String root = YMarkTables.FOLDERS_IMPORTED;
InputStreamReader reader = null;
if(isAdmin || isAuthUser) {
String bmk_user = (isAuthUser ? user.getUserName() : YMarkTables.USER_ADMIN);
if(isAdmin && post.containsKey("table") && post.get("table").length() > 0) {
bmk_user = post.get("table").substring(0, post.get("table").indexOf('_'));
}
if(post.containsKey("redirect") && post.get("redirect").length() > 0) {
prop.put("redirect_url", post.get("redirect"));
prop.put("redirect", "1");
}
if(post.containsKey("root") && post.get("root").length() > 0) {
root = post.get("root");
}
if(post.containsKey("bmkfile") && post.containsKey("importer")){
byteIn = new ByteArrayInputStream(UTF8.getBytes(post.get("bmkfile$file")));
if(post.get("importer").equals("html") && byteIn != null) {
final YMarkHTMLImporter htmlImporter = new YMarkHTMLImporter(byteIn, 10);
try {
reader = new InputStreamReader(new ByteArrayInputStream(UTF8.getBytes(post.get("bmkfile$file"))),"UTF-8");
} catch (UnsupportedEncodingException e1) {
Log.logException(e1);
// return rewrite properties
prop.put("result", "0");
return prop;
}
if(post.get("importer").equals("html") && reader != null) {
final YMarkHTMLImporter htmlImporter = new YMarkHTMLImporter(reader, 10, root);
t = new Thread(htmlImporter, "YMarks - HTML Importer");
t.start();
while ((bmk = htmlImporter.take()) != YMarkTables.POISON) {
putBookmark(sb, bmk_user, bmk);
}
prop.put("result", "1");
} else if(post.get("importer").equals("xbel") && byteIn != null) {
} else if(post.get("importer").equals("xbel") && reader != null) {
final YMarkXBELImporter xbelImporter;
try {
//TODO: make RootFold
xbelImporter = new YMarkXBELImporter(byteIn, 100, YMarkTables.FOLDERS_IMPORTED);
xbelImporter = new YMarkXBELImporter(reader, 10, root);
} catch (SAXException e) {
//TODO: display an error message
Log.logException(e);
@ -68,14 +83,9 @@ public class import_ymark {
putBookmark(sb, bmk_user, bmk);
}
prop.put("result", "1");
} else if(post.get("importer").equals("json") && byteIn != null) {
} else if(post.get("importer").equals("json") && reader != null) {
YMarkJSONImporter jsonImporter;
try {
jsonImporter = new YMarkJSONImporter(byteIn, 10);
} catch (UnsupportedEncodingException e) {
prop.put("result", "1");
return prop;
}
jsonImporter = new YMarkJSONImporter(reader, 10, root);
t = new Thread(jsonImporter, "YMarks - JSON Importer");
t.start();
while ((bmk = jsonImporter.take()) != YMarkTables.POISON) {
@ -87,10 +97,6 @@ public class import_ymark {
} else {
prop.put(YMarkTables.USER_AUTHENTICATE,YMarkTables.USER_AUTHENTICATE_MSG);
}
if(post.containsKey("redirect") && post.get("redirect").length() > 0) {
prop.put("redirect_url", post.get("redirect"));
prop.put("redirect", "1");
}
// return rewrite properties
return prop;
}

28
htroot/api/ymarks/test_import.html
Unescape View File

@ -9,15 +9,31 @@
<legend>Import Bookmarks</legend>
<dl>
<dt>
<label for="importer">Importer:</label>
<label for="importer">Bookmark File Format:</label>
</dt>
<dd>
<select name="importer" id="importer">
<option value="xbel">XBEL Importer</option>
<option value="html">Netscape HTML Importer</option>
<option value="json">Firefox JSON Importer</option>
</select>
<input type="radio" name="importer" value="html" /> Netscape HTML,
<small>
bookmark.htm (Firefox, IE, Safari, Opera)
</small>
<br />
<input type="radio" name="importer" value="json" /> Firefox JSON
<br />
<input type="radio" name="importer" value="xbel" /> XBEL,
<small>
<a href="http://pyxml.sourceforge.net/topics/xbel/">
XML Bookmark Exchange Language
</a>
</small>
</dd>
<br />
<dt>
<label for="root">Target Folder:</label>
</dt>
<dd>
<input type="text" name="root" id="root" value="/Imported Bookmarks" />
</dd>
<br />
<dt>
<label for="file">File:</label>
</dt>

70
source/de/anomic/data/ymark/YMarkHTMLImporter.java
Unescape View File

@ -27,11 +27,9 @@
package de.anomic.data.ymark;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.util.HashMap;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
import javax.swing.text.MutableAttributeSet;
import javax.swing.text.html.HTML;
@ -41,40 +39,46 @@ import javax.swing.text.html.parser.ParserDelegator;
import net.yacy.kelondro.logging.Log;
public class YMarkHTMLImporter extends HTMLEditorKit.ParserCallback implements Runnable {
// Importer Variables
private final ArrayBlockingQueue<HashMap<String,String>> bookmarks;
private final Reader bmk_file;
private final String RootFolder;
private final StringBuilder folderstring;
private HashMap<String,String> bmk;
private final ParserDelegator htmlParser;
// Statics
public static enum STATE {
NOTHING,
BOOKMARK,
FOLDER,
BMK_DESC,
FOLDER_DESC
NOTHING,
BOOKMARK,
FOLDER,
BMK_DESC,
FOLDER_DESC
}
public static final String MILLIS = "000";
private static final String MILLIS = "000";
private STATE state;
// Parser variables
private STATE state;
private HTML.Tag prevTag;
private HashMap<String,String> bmk;
private StringBuilder folder;
private final InputStream input;
private final BlockingQueue<HashMap<String,String>> bookmarks;
private final ParserDelegator htmlParser;
public YMarkHTMLImporter(final InputStream input, int queueSize) {
this.state = STATE.NOTHING;
public YMarkHTMLImporter(final Reader bmk_file, final int queueSize, final String root) {
this.bookmarks = new ArrayBlockingQueue<HashMap<String,String>>(queueSize);
this.bmk_file = bmk_file;
this.RootFolder = root;
this.folderstring = new StringBuilder(YMarkTables.FOLDER_BUFFER_SIZE);
this.folderstring.append(this.RootFolder);
this.bmk = new HashMap<String,String>();
this.htmlParser = new ParserDelegator();
this.state = STATE.NOTHING;
this.prevTag = null;
this.bmk = new HashMap<String,String>();
this.folder = new StringBuilder(YMarkTables.FOLDER_BUFFER_SIZE);
this.folder.append(YMarkTables.FOLDERS_IMPORTED);
this.bookmarks = new ArrayBlockingQueue<HashMap<String,String>>(queueSize);
this.input = input;
this.htmlParser = new ParserDelegator();
}
public void run() {
try {
this.htmlParser.parse(new InputStreamReader(this.input,"UTF-8"), this, true);
this.htmlParser.parse(this.bmk_file, this, true);
} catch (IOException e) {
Log.logException(e);
} finally {
@ -84,7 +88,7 @@ public class YMarkHTMLImporter extends HTMLEditorKit.ParserCallback implements R
Log.logException(e);
}
try {
this.input.close();
this.bmk_file.close();
} catch (IOException e) {
Log.logException(e);
}
@ -97,16 +101,16 @@ public class YMarkHTMLImporter extends HTMLEditorKit.ParserCallback implements R
break;
case BOOKMARK:
this.bmk.put(YMarkTables.BOOKMARK.TITLE.key(), new String(data));
this.bmk.put(YMarkTables.BOOKMARK.FOLDERS.key(), this.folder.toString());
this.bmk.put(YMarkTables.BOOKMARK.FOLDERS.key(), this.folderstring.toString());
this.bmk.put(YMarkTables.BOOKMARK.PUBLIC.key(), YMarkTables.BOOKMARK.PUBLIC.deflt());
this.bmk.put(YMarkTables.BOOKMARK.VISITS.key(), YMarkTables.BOOKMARK.VISITS.deflt());
break;
case FOLDER:
this.folder.append(YMarkUtil.FOLDERS_SEPARATOR);
this.folder.append(data);
this.folderstring.append(YMarkUtil.FOLDERS_SEPARATOR);
this.folderstring.append(data);
break;
case FOLDER_DESC:
Log.logInfo(YMarkTables.BOOKMARKS_LOG, "YMarksHTMLImporter - folder: "+this.folder+" desc: " + new String(data));
Log.logInfo(YMarkTables.BOOKMARKS_LOG, "YMarksHTMLImporter - folder: "+this.folderstring+" desc: " + new String(data));
break;
case BMK_DESC:
this.bmk.put(YMarkTables.BOOKMARK.DESC.key(), new String(data));
@ -163,8 +167,8 @@ public class YMarkHTMLImporter extends HTMLEditorKit.ParserCallback implements R
state = STATE.FOLDER_DESC;
} else if (t == HTML.Tag.DL) {
//TODO: get rid of .toString.equals()
if(!this.folder.toString().equals(YMarkTables.FOLDERS_IMPORTED)) {
folder.setLength(folder.lastIndexOf(YMarkUtil.FOLDERS_SEPARATOR));
if(!this.folderstring.toString().equals(YMarkTables.FOLDERS_IMPORTED)) {
folderstring.setLength(folderstring.lastIndexOf(YMarkUtil.FOLDERS_SEPARATOR));
}
} else {
state = STATE.NOTHING;

55
source/de/anomic/data/ymark/YMarkJSONImporter.java
Unescape View File

@ -1,10 +1,7 @@
package de.anomic.data.ymark;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.io.UnsupportedEncodingException;
import java.util.HashMap;
import java.util.concurrent.ArrayBlockingQueue;
@ -15,38 +12,48 @@ import org.json.simple.parser.JSONParser;
import org.json.simple.parser.ParseException;
public class YMarkJSONImporter implements Runnable, ContentHandler{
public final static String FOLDER = "text/x-moz-place-container";
public final static String BOOKMARK = "text/x-moz-place";
public final static String ANNOS = "annos";
public final static String TYPE = "type";
public final static String CHILDREN = "children";
private final static String MILLIS = "000";
private final JSONParser parser;
// Importer Variables
private final ArrayBlockingQueue<HashMap<String,String>> bookmarks;
private final Reader bmk_file;
private final String RootFolder;
private final StringBuilder folderstring;
private HashMap<String,String> bmk;
private final JSONParser parser;
private final Reader json;
private final StringBuilder folderstring;
// Statics
public final static String FOLDER = "text/x-moz-place-container";
public final static String BOOKMARK = "text/x-moz-place";
public final static String ANNOS = "annos";
public final static String TYPE = "type";
public final static String CHILDREN = "children";
public final static String MILLIS = "000";
// Parser Variables
private final StringBuilder value;
private final StringBuilder key;
private final HashMap<String,String> obj;
private final ArrayBlockingQueue<HashMap<String,String>> bookmarks;
private HashMap<String,String> bmk;
private int depth;
private Boolean isFolder;
private Boolean isBookmark;
private Boolean isAnnos;
public YMarkJSONImporter(final InputStream input, int queueSize) throws UnsupportedEncodingException {
this.parser = new JSONParser();
this.bookmarks = new ArrayBlockingQueue<HashMap<String,String>>(queueSize);
this.json = new InputStreamReader(input, "UTF-8");
this.folderstring = new StringBuilder(256);
this.key = new StringBuilder(16);
this.value = new StringBuilder(128);
public YMarkJSONImporter(final Reader bmk_file, final int queueSize, final String root) {
this.bookmarks = new ArrayBlockingQueue<HashMap<String,String>>(queueSize);
this.bmk_file = bmk_file;
this.RootFolder = root;
this.folderstring = new StringBuilder(YMarkTables.FOLDER_BUFFER_SIZE);
this.folderstring.append(this.RootFolder);
this.bmk = new HashMap<String,String>();
this.parser = new JSONParser();
this.value = new StringBuilder(128);
this.key = new StringBuilder(16);
this.obj = new HashMap<String,String>();
this.bmk = new HashMap<String,String>();
this.depth = 0;
this.isAnnos = false;
this.isBookmark = false;
this.isFolder = true;
@ -154,7 +161,7 @@ public class YMarkJSONImporter implements Runnable, ContentHandler{
public void run() {
try {
Log.logInfo(YMarkTables.BOOKMARKS_LOG, "JSON Importer run()");
this.parser.parse(json, this, true);
this.parser.parse(this.bmk_file, this, true);
} catch (IOException e) {
Log.logException(e);
} catch (ParseException e) {

8
source/de/anomic/data/ymark/YMarkTables.java
Unescape View File

@ -202,24 +202,26 @@ public class YMarkTables {
this.patternBuilder.append(root);
this.patternBuilder.append(p7);
this.patternBuilder.append(p8);
final Pattern r = Pattern.compile(this.patternBuilder.toString());
final Iterator<Tables.Row> bit = this.worktables.iterator(bmk_table, YMarkTables.BOOKMARK.FOLDERS.key(), r);
final TreeSet<String> folders = new TreeSet<String>();
final StringBuilder path = new StringBuilder(200);
Tables.Row bmk_row = null;
while(bit.hasNext()) {
bmk_row = bit.next();
if(bmk_row.containsKey(BOOKMARK.FOLDERS.key())) {
final String[] folderArray = (new String(bmk_row.get(BOOKMARK.FOLDERS.key()),"UTF8")).split(YMarkUtil.TAGS_SEPARATOR);
for (final String folder : folderArray) {
if(folder.startsWith(root)) {
if(!folders.contains(folder)) {
if(folder.substring(0, root.length()+1).equals(root+'/')) {
if(!folders.contains(folder)) {
path.setLength(0);
path.append(folder);
//TODO: get rid of .toString.equals()
while(path.length() > 0 && !path.toString().equals(root)){
folders.add(path.toString());
path.setLength(path.lastIndexOf(YMarkUtil.FOLDERS_SEPARATOR));
path.setLength(path.lastIndexOf(YMarkUtil.FOLDERS_SEPARATOR));
}
}
}

75
source/de/anomic/data/ymark/YMarkXBELImporter.java
Unescape View File

@ -27,7 +27,7 @@
package de.anomic.data.ymark;
import java.io.IOException;
import java.io.InputStream;
import java.io.Reader;
import java.text.ParseException;
import java.util.HashMap;
import java.util.HashSet;
@ -45,7 +45,16 @@ import org.xml.sax.helpers.XMLReaderFactory;
public class YMarkXBELImporter extends DefaultHandler implements Runnable {
public static enum XBEL {
// Importer Variables
private final ArrayBlockingQueue<HashMap<String,String>> bookmarks;
private final Reader bmk_file;
private final String RootFolder;
private final StringBuilder folderstring;
private HashMap<String,String> bmk;
private final XMLReader xmlReader;
// Statics
public static enum XBEL {
NOTHING (""),
XBEL ("<xbel"),
TITLE ("<title"),
@ -86,47 +95,41 @@ public class YMarkXBELImporter extends DefaultHandler implements Runnable {
}
}
private HashMap<String,String> ref;
private HashMap<String,String> bmk;
private XBEL outer_state; // BOOKMARK, FOLDER, NOTHING
private XBEL inner_state; // DESC, TITLE, INFO, ALIAS, (METADATA), NOTHING
private boolean parse_value;
// Parser Variables
private final HashMap<String,HashMap<String,String>> bmkRef;
private final HashSet<HashMap<String,String>> aliasRef;
private final StringBuilder buffer;
private final StringBuilder folder;
private final StringBuilder foldersString;
private final InputSource input;
private final ArrayBlockingQueue<HashMap<String,String>> bookmarks;
private final XMLReader xmlReader;
private final String RootFolder;
private final StringBuilder folder;
private HashMap<String,String> ref;
private XBEL outer_state; // BOOKMARK, FOLDER, NOTHING
private XBEL inner_state; // DESC, TITLE, INFO, ALIAS, (METADATA), NOTHING
private boolean parse_value;
public YMarkXBELImporter (final InputStream input, int queueSize, String root) throws SAXException {
this.bmk = null;
this.RootFolder = root;
this.buffer = new StringBuilder();
this.foldersString = new StringBuilder(YMarkTables.FOLDER_BUFFER_SIZE);
this.folder = new StringBuilder(YMarkTables.FOLDER_BUFFER_SIZE);
this.folder.append(this.RootFolder);
this.input = new InputSource(input);
this.bmkRef = new HashMap<String,HashMap<String,String>>();
this.aliasRef = new HashSet<HashMap<String,String>>();
this.bookmarks = new ArrayBlockingQueue<HashMap<String,String>>(queueSize);
this.xmlReader = XMLReaderFactory.createXMLReader();
public YMarkXBELImporter (final Reader bmk_file, final int queueSize, final String root) throws SAXException {
this.bookmarks = new ArrayBlockingQueue<HashMap<String,String>>(queueSize);
this.bmk_file = bmk_file;
this.RootFolder = root;
this.folderstring = new StringBuilder(YMarkTables.FOLDER_BUFFER_SIZE);
this.folderstring.append(this.RootFolder);
this.bmk = new HashMap<String,String>();
this.xmlReader = XMLReaderFactory.createXMLReader();
this.xmlReader.setContentHandler(this);
this.xmlReader.setFeature("http://xml.org/sax/features/namespace-prefixes", false);
this.xmlReader.setFeature("http://xml.org/sax/features/namespaces", false);
this.xmlReader.setFeature("http://xml.org/sax/features/validation", false);
this.bmkRef = new HashMap<String,HashMap<String,String>>();
this.aliasRef = new HashSet<HashMap<String,String>>();
this.buffer = new StringBuilder();
this.folder = new StringBuilder(YMarkTables.FOLDER_BUFFER_SIZE);
this.folder.append(this.RootFolder);
}
public void run() {
try {
this.xmlReader.parse(this.input);
this.xmlReader.parse(new InputSource(this.bmk_file));
} catch (SAXParseException e) {
Log.logException(e);
} catch (SAXException e) {
@ -295,19 +298,19 @@ public class YMarkXBELImporter extends DefaultHandler implements Runnable {
}
private void UpdateBmkRef(final String id, final boolean url) {
this.foldersString.setLength(0);
this.folderstring.setLength(0);
if(this.bmkRef.containsKey(id)) {
this.foldersString.append(this.bmkRef.get(id).get(YMarkTables.BOOKMARK.FOLDERS.key()));
this.foldersString.append(',');
this.folderstring.append(this.bmkRef.get(id).get(YMarkTables.BOOKMARK.FOLDERS.key()));
this.folderstring.append(',');
this.ref = this.bmkRef.get(id);
} else {
this.ref = new HashMap<String,String>();
}
this.foldersString.append(this.folder);
this.folderstring.append(this.folder);
if(url)
this.ref.put(YMarkTables.BOOKMARK.URL.key(), this.bmk.get(YMarkTables.BOOKMARK.URL.key()));
this.ref.put(YMarkTables.BOOKMARK.FOLDERS.key(), this.foldersString.toString());
this.ref.put(YMarkTables.BOOKMARK.FOLDERS.key(), this.folderstring.toString());
this.bmkRef.put(id, ref);
}
}

Write Preview
Loading…
Cancel
Save