yacy_search_server/source/net/yacy/data/ymark/YMarkHTMLImporter.java

// YMarkHTMLImporter.java
// (C) 2011 by Stefan F<><46>rster, sof@gmx.de, Norderstedt, Germany
// first published 2010 on http://yacy.net
//
// This is a part of YaCy, a peer-to-peer based web search engine
//
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
// LICENSE
// 
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

package net.yacy.data.ymark;

import javax.swing.text.MutableAttributeSet;
import javax.swing.text.html.HTML;
import javax.swing.text.html.HTMLEditorKit;
import javax.swing.text.html.parser.ParserDelegator;

import net.yacy.cora.util.ConcurrentLog;

public class YMarkHTMLImporter extends YMarkImporter {
	
    // Importer Variables
    private final ParserDelegator htmlParser;
    
    // Statics
	public static String IMPORTER = "HTML";
    public static enum STATE {
        NOTHING,
        BOOKMARK,
        FOLDER,
        BMK_DESC,
        FOLDER_DESC
    }
    public static final String MILLIS = "000";

	public YMarkHTMLImporter(final MonitoredReader bmk_file, final int queueSize, final String targetFolder, final String sourceFolder) {		
		super(bmk_file, queueSize, targetFolder, sourceFolder);
		setImporter(IMPORTER);
		this.htmlParser = new ParserDelegator();
	}
	
	public YMarkHTMLImporter (final MonitoredReader bmk_file, final int queueSize, final String targetFolder) {
		this(bmk_file, queueSize, targetFolder, "");
	}	
	
	@Override
    public void parse() throws Exception {
		htmlParser.parse(bmk_file, new HTMLParser(), true);
	}
	
	public class HTMLParser extends HTMLEditorKit.ParserCallback {
	    
	    private YMarkEntry bmk;
	    private final StringBuilder folderstring;
	    private STATE state;
		private HTML.Tag prevTag;
		
	    public HTMLParser() {
	        this.folderstring = new StringBuilder(YMarkTables.BUFFER_LENGTH);
	        this.folderstring.append(targetFolder);        
	        this.bmk = new YMarkEntry();
		    this.state = STATE.NOTHING;
			this.prevTag = null;
	    }
		
		@Override
        public void handleText(char[] data, int pos) {
	    	switch (state) {
	    		case NOTHING:
	    			break;
	    		case BOOKMARK:
					this.bmk.put(YMarkEntry.BOOKMARK.TITLE.key(), new String(data));
					this.bmk.put(YMarkEntry.BOOKMARK.FOLDERS.key(), this.folderstring.toString());
					this.bmk.put(YMarkEntry.BOOKMARK.PUBLIC.key(), YMarkEntry.BOOKMARK.PUBLIC.deflt());
					this.bmk.put(YMarkEntry.BOOKMARK.VISITS.key(), YMarkEntry.BOOKMARK.VISITS.deflt());
					break;
	    		case FOLDER:
	    			this.folderstring.append(YMarkUtil.FOLDERS_SEPARATOR);
	    			this.folderstring.append(data);
	    			break;
	    		case FOLDER_DESC:
	    			ConcurrentLog.info(YMarkTables.BOOKMARKS_LOG, "YMarksHTMLImporter - folder: "+this.folderstring+" desc: " + new String(data));
	    			break;
	    		case BMK_DESC:
	    			this.bmk.put(YMarkEntry.BOOKMARK.DESC.key(), new String(data));
	    			break;    			
	    		default:
	    			break;
	    	}
		}

		@Override
        public void handleStartTag(HTML.Tag t, MutableAttributeSet a, int pos) {
			if (t == HTML.Tag.A) {
				if (!this.bmk.isEmpty()) {
					try {
						bookmarks.put(this.bmk);
						bmk = new YMarkEntry();
					} catch (final InterruptedException e) {
						ConcurrentLog.logException(e);
					}
				}
		    	final String url = (String)a.getAttribute(HTML.Attribute.HREF);
				this.bmk.put(YMarkEntry.BOOKMARK.URL.key(), url);	    	
				final StringBuilder sb = new StringBuilder(255);
				for (YMarkEntry.BOOKMARK bmk : YMarkEntry.BOOKMARK.values()) {    
					sb.setLength(0);   			
					if (a.isDefined(bmk.html_attrb())) {
						sb.append((String)a.getAttribute(bmk.html_attrb()));
						ConcurrentLog.info(YMarkTables.BOOKMARKS_LOG, bmk.key()+" : "+sb.toString());
					}
	    			switch(bmk) {	    					
	    				case TAGS:	    					
	    					// sb already contains the mozilla shortcuturl
	    					// add delicious.com tags that are stored in the tags attribute	    					
	    					if (a.isDefined(YMarkEntry.BOOKMARK.TAGS.key())) {		    					
	    						sb.append(YMarkUtil.TAGS_SEPARATOR);
	    						sb.append((String)a.getAttribute(YMarkEntry.BOOKMARK.TAGS.key()));		    					
	    					}
	    					this.bmk.put(bmk.key(), YMarkUtil.cleanTagsString(sb.toString()));
	    	    			break;
	    				case PUBLIC:
	    					// look for delicious.com private attribute
	    					if(sb.toString().equals("0"))
	    						this.bmk.put(bmk.key(), "true");
	    					break;
	    				case DATE_ADDED:
	    				case DATE_MODIFIED:
	    				case DATE_VISITED:
	   						sb.append(MILLIS);    					
	   						this.bmk.put(bmk.key(), sb.toString());
	    					break;
	    				default:
	    					break;		    					
		    		} 		
		    	}
		    	state = STATE.BOOKMARK;
		    } else if (t == HTML.Tag.H3) {
		    	state = STATE.FOLDER;
		    } else if (t == HTML.Tag.DD && this.prevTag == HTML.Tag.A) {
		    	state = STATE.BMK_DESC;	    	
		    } else {
		    	state = STATE.NOTHING;
		    }
		    this.prevTag = t;
		}

		@Override
        public void handleEndTag(HTML.Tag t, int pos) {
			// write the last bookmark, as no more <a> tags are following
			if (t == HTML.Tag.HTML) {
				if (!this.bmk.isEmpty()) {
					try {
						bookmarks.put(this.bmk);
					} catch (final InterruptedException e) {
						ConcurrentLog.logException(e);
					}
				}
			}
			if (t == HTML.Tag.H3) {
				state = STATE.FOLDER_DESC;
		    } else if (t == HTML.Tag.DL) {
	            //TODO: get rid of .toString.equals()
	        	if(!this.folderstring.toString().equals(targetFolder)) {
		    		folderstring.setLength(folderstring.lastIndexOf(YMarkUtil.FOLDERS_SEPARATOR));
	        	}
		    } else {
		    	state = STATE.NOTHING;
		    }
		}
	}
}