Conflicts: htroot/api/ymarks/import_ymark.java source/de/anomic/data/ymark/YMarkEntry.java source/de/anomic/data/ymark/YMarkTables.javapull/1/head
Michael Peter Christen
13 years ago
commit
8c099d2106
@ -1,27 +1,81 @@
|
||||
import java.io.IOException;
|
||||
import java.util.Iterator;
|
||||
|
||||
import net.yacy.cora.document.UTF8;
|
||||
import net.yacy.cora.protocol.HeaderFramework;
|
||||
import net.yacy.cora.protocol.RequestHeader;
|
||||
import net.yacy.cora.util.SpaceExceededException;
|
||||
import net.yacy.kelondro.blob.Tables;
|
||||
import net.yacy.kelondro.logging.Log;
|
||||
import net.yacy.search.Switchboard;
|
||||
import de.anomic.data.UserDB;
|
||||
import de.anomic.data.ymark.YMarkEntry;
|
||||
import de.anomic.data.ymark.YMarkRDF;
|
||||
import de.anomic.data.ymark.YMarkTables;
|
||||
import de.anomic.data.ymark.YMarkTables.TABLES;
|
||||
import de.anomic.server.serverObjects;
|
||||
import de.anomic.server.serverSwitch;
|
||||
|
||||
public class YMarks {
|
||||
public static serverObjects respond(final RequestHeader header, @SuppressWarnings("unused") final serverObjects post, final serverSwitch env) {
|
||||
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
|
||||
final Switchboard sb = (Switchboard) env;
|
||||
final serverObjects prop = new serverObjects();
|
||||
final UserDB.Entry user = sb.userDB.getUser(header);
|
||||
final boolean isAdmin = (sb.verifyAuthentication(header));
|
||||
final boolean isAuthUser = user!= null && user.hasRight(UserDB.AccessRight.BOOKMARK_RIGHT);
|
||||
|
||||
final String path = header.get(HeaderFramework.CONNECTION_PROP_PATH);
|
||||
if(path != null && path.endsWith(".rdf")) {
|
||||
YMarkRDF rdf = new YMarkRDF("http://"+sb.peers.myAlternativeAddress());
|
||||
|
||||
if(post != null && post.containsKey(YMarkEntry.BOOKMARKS_ID)) {
|
||||
final String id[] = post.get(YMarkEntry.BOOKMARKS_ID).split(":");
|
||||
if(id[1].equals("b")) {
|
||||
final String bmk_user = id[0];
|
||||
final String bmk_table = TABLES.BOOKMARKS.tablename(bmk_user);
|
||||
final byte[] urlHash = UTF8.getBytes(id[2]);
|
||||
Tables.Row bmk_row;
|
||||
try {
|
||||
bmk_row = sb.tables.select(bmk_table, urlHash);
|
||||
rdf.addBookmark(bmk_user, bmk_row);
|
||||
} catch (IOException e) {
|
||||
} catch (SpaceExceededException e) {
|
||||
}
|
||||
}
|
||||
} else {
|
||||
final Iterator<String> iter = sb.tables.iterator();
|
||||
while(iter.hasNext()) {
|
||||
final String bmk_table = iter.next();
|
||||
final int i = bmk_table.indexOf(TABLES.BOOKMARKS.basename());
|
||||
if(i > 0) {
|
||||
final String bmk_user = bmk_table.substring(0, i);
|
||||
try {
|
||||
// TODO select only public bookmarks
|
||||
rdf.addBookmarks(bmk_user, sb.tables.iterator(bmk_table));
|
||||
} catch (IOException e) {
|
||||
// TODO exception handling
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
prop.put("rdf", rdf.getRDF("RDF/XML-ABBREV"));
|
||||
return prop;
|
||||
}
|
||||
if(isAdmin || isAuthUser) {
|
||||
prop.put("login", 1);
|
||||
final String bmk_user = (isAuthUser ? user.getUserName() : YMarkTables.USER_ADMIN);
|
||||
prop.putHTML("user", bmk_user.substring(0,1).toUpperCase() + bmk_user.substring(1));
|
||||
|
||||
int size;
|
||||
try {
|
||||
size = sb.tables.bookmarks.getSize(bmk_user);
|
||||
} catch (IOException e) {
|
||||
Log.logException(e);
|
||||
size = 0;
|
||||
}
|
||||
prop.put("size", size);
|
||||
} else {
|
||||
prop.put("login", 0);
|
||||
}
|
||||
|
||||
}
|
||||
return prop;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1 @@
|
||||
#[rdf]#
|
||||
@ -0,0 +1,102 @@
|
||||
package de.anomic.data.ymark;
|
||||
|
||||
import java.io.FilterReader;
|
||||
import java.io.IOException;
|
||||
import java.io.Reader;
|
||||
import java.nio.CharBuffer;
|
||||
|
||||
import javax.swing.event.ChangeEvent;
|
||||
import javax.swing.event.ChangeListener;
|
||||
|
||||
/**
|
||||
* This class monitors the read progress
|
||||
*
|
||||
*/
|
||||
public class MonitoredReader extends FilterReader {
|
||||
private volatile long mark = 0;
|
||||
private volatile long location = 0;
|
||||
private final int threshold;
|
||||
private final long maxProgress;
|
||||
private long lastTriggeredLocation = 0;
|
||||
private ChangeListener listener = null;
|
||||
|
||||
public MonitoredReader(Reader in, int threshold, long maxProgress) {
|
||||
super(in);
|
||||
this.threshold = threshold;
|
||||
this.maxProgress = maxProgress;
|
||||
}
|
||||
|
||||
public void addChangeListener(ChangeListener l) {
|
||||
this.listener = l;
|
||||
}
|
||||
|
||||
protected void triggerChanged(final long location) {
|
||||
if ( threshold > 0 && Math.abs( location-lastTriggeredLocation ) < threshold )
|
||||
return;
|
||||
lastTriggeredLocation = location;
|
||||
if (listener == null)
|
||||
return;
|
||||
listener.stateChanged(new ChangeEvent(this));
|
||||
}
|
||||
|
||||
public long getProgress() {
|
||||
return this.location;
|
||||
}
|
||||
|
||||
public long maxProgress() {
|
||||
return this.maxProgress;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int read() throws IOException {
|
||||
final int i = super.read();
|
||||
if ( i != -1 )
|
||||
triggerChanged(location++);
|
||||
return i;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int read(char[] cbuf, int off, int len) throws IOException {
|
||||
final int i = super.read(cbuf, off, len);
|
||||
if ( i != -1 )
|
||||
triggerChanged(location+=i);
|
||||
return i;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int read(char[] cbuf) throws IOException {
|
||||
final int i = super.read(cbuf);
|
||||
if ( i != -1 )
|
||||
triggerChanged(location+=i);
|
||||
return i;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int read(CharBuffer target) throws IOException {
|
||||
final int i = super.read(target);
|
||||
if ( i != -1 )
|
||||
triggerChanged(location+=i);
|
||||
return i;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long skip(long n) throws IOException {
|
||||
final long i = super.skip(n);
|
||||
if ( i != -1 )
|
||||
triggerChanged(location+=i);
|
||||
return i;
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized void mark(int readlimit) throws IOException {
|
||||
super.mark(readlimit);
|
||||
mark = location;
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized void reset() throws IOException {
|
||||
super.reset();
|
||||
if ( location != mark )
|
||||
triggerChanged(location = mark);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,152 @@
|
||||
// YMarkDMOZImporter.java
|
||||
// (C) 2012 by Stefan Foerster (apfelmaennchen), sof@gmx.de, Norderstedt, Germany
|
||||
// first published 2012 on http://yacy.net
|
||||
//
|
||||
// This is a part of YaCy, a peer-to-peer based web search engine
|
||||
//
|
||||
// $LastChangedDate$
|
||||
// $LastChangedRevision$
|
||||
// $LastChangedBy$
|
||||
//
|
||||
// LICENSE
|
||||
//
|
||||
// This program is free software; you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation; either version 2 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with this program; if not, write to the Free Software
|
||||
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
|
||||
package de.anomic.data.ymark;
|
||||
|
||||
import net.yacy.cora.lod.vocabulary.DMOZ;
|
||||
import net.yacy.cora.lod.vocabulary.DublinCore;
|
||||
|
||||
import org.xml.sax.Attributes;
|
||||
import org.xml.sax.InputSource;
|
||||
import org.xml.sax.SAXException;
|
||||
import org.xml.sax.XMLReader;
|
||||
import org.xml.sax.helpers.DefaultHandler;
|
||||
import org.xml.sax.helpers.XMLReaderFactory;
|
||||
|
||||
public class YMarkDMOZImporter extends YMarkImporter {
|
||||
// Statics
|
||||
public static String IMPORTER = "DMOZ";
|
||||
|
||||
// Importer Variables
|
||||
private final XMLReader xmlReader;
|
||||
private int depth;
|
||||
|
||||
public YMarkDMOZImporter(final MonitoredReader dmoz_file, final int queueSize, final String targetFolder, final String sourceFolder) throws SAXException {
|
||||
super(dmoz_file, queueSize, sourceFolder, targetFolder);
|
||||
setImporter(IMPORTER);
|
||||
this.xmlReader = XMLReaderFactory.createXMLReader();
|
||||
this.xmlReader.setFeature(XML_NAMESPACE_PREFIXES, false);
|
||||
this.xmlReader.setFeature(XML_NAMESPACES, false);
|
||||
this.xmlReader.setFeature(XML_VALIDATION, false);
|
||||
this.xmlReader.setContentHandler(new DMOZParser());
|
||||
this.depth = Integer.MAX_VALUE;
|
||||
}
|
||||
|
||||
public void parse() throws Exception {
|
||||
xmlReader.parse(new InputSource(bmk_file));
|
||||
}
|
||||
|
||||
public void setDepth(int d) {
|
||||
this.depth = d + this.targetFolder.split(YMarkUtil.FOLDERS_SEPARATOR).length-1;
|
||||
}
|
||||
|
||||
public class DMOZParser extends DefaultHandler {
|
||||
|
||||
private YMarkEntry bmk;
|
||||
private boolean isNewEntry;
|
||||
private boolean isSubtopic;
|
||||
private String tag;
|
||||
private final StringBuilder buffer;
|
||||
|
||||
public DMOZParser() {
|
||||
this.bmk = new YMarkEntry();
|
||||
this.isNewEntry = false;
|
||||
this.isSubtopic = false;
|
||||
this.buffer = new StringBuilder(512);
|
||||
}
|
||||
|
||||
public void startElement(final String uri, String localName, final String qName, final Attributes attributes) throws SAXException {
|
||||
// get rid of namespace prefixes
|
||||
if (localName.isEmpty()) {
|
||||
localName = qName.substring(qName.indexOf(':')+1);
|
||||
}
|
||||
this.tag = null;
|
||||
if (localName.equals(DMOZ.ExternalPage.name())) {
|
||||
this.bmk = new YMarkEntry();
|
||||
this.bmk.put(YMarkEntry.BOOKMARK.URL.key(), attributes.getValue(0));
|
||||
this.isNewEntry = true;
|
||||
}
|
||||
if(isNewEntry && localName.equals(DublinCore.Title.name())) {
|
||||
this.tag = YMarkEntry.BOOKMARK.TITLE.key();
|
||||
}
|
||||
if(isNewEntry && localName.equals(DublinCore.Description.name())) {
|
||||
this.tag = YMarkEntry.BOOKMARK.DESC.key();
|
||||
}
|
||||
if(isNewEntry && localName.equals(DMOZ.topic.name())) {
|
||||
this.tag = YMarkEntry.BOOKMARK.FOLDERS.key();
|
||||
buffer.append(targetFolder);
|
||||
buffer.append(YMarkUtil.FOLDERS_SEPARATOR);
|
||||
}
|
||||
}
|
||||
|
||||
public void endElement(final String uri, String localName, final String qName) throws SAXException {
|
||||
// get rid of namespace prefixes
|
||||
if (localName.isEmpty()) {
|
||||
localName = qName.substring(qName.indexOf(':')+1);
|
||||
}
|
||||
if (this.isNewEntry && this.isSubtopic && localName.equals(DMOZ.ExternalPage.name())) {
|
||||
try {
|
||||
bookmarks.put(this.bmk);
|
||||
} catch (InterruptedException e) {
|
||||
e.printStackTrace();
|
||||
} finally {
|
||||
this.isSubtopic = false;
|
||||
this.isNewEntry = false;
|
||||
}
|
||||
} else if(localName.equals(DMOZ.topic.name())) {
|
||||
int d = 0;
|
||||
for(int i=0; i<this.buffer.length(); i++) {
|
||||
if (this.buffer.charAt(i) == '/') {
|
||||
d++;
|
||||
if (d > depth) {
|
||||
this.buffer.setLength(i);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (this.buffer.substring(targetFolder.length()+1).startsWith(sourceFolder)) {
|
||||
this.isSubtopic = true;
|
||||
this.bmk.put(this.tag, YMarkUtil.cleanFoldersString(buffer));
|
||||
} else {
|
||||
this.isSubtopic = false;
|
||||
this.isNewEntry = false;
|
||||
}
|
||||
} else if (this.tag != null) {
|
||||
this.bmk.put(this.tag, buffer.toString());
|
||||
}
|
||||
this.tag = null;
|
||||
this.buffer.setLength(0);
|
||||
}
|
||||
|
||||
public void characters(final char ch[], final int start, final int length) throws SAXException {
|
||||
// no processing here, as the SAX Parser characters method could be called more than once per tag!
|
||||
if(this.tag != null) {
|
||||
buffer.append(ch, start, length);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -0,0 +1,156 @@
|
||||
// YMarkImporter.java
|
||||
// (C) 2012 by Stefan Foerster (apfelmaennchen), sof@gmx.de, Norderstedt, Germany
|
||||
// first published 2012 on http://yacy.net
|
||||
//
|
||||
// This is a part of YaCy, a peer-to-peer based web search engine
|
||||
//
|
||||
// LICENSE
|
||||
//
|
||||
// This program is free software; you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation; either version 2 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with this program; if not, write to the Free Software
|
||||
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
|
||||
package de.anomic.data.ymark;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.concurrent.ArrayBlockingQueue;
|
||||
|
||||
import net.yacy.kelondro.logging.Log;
|
||||
import net.yacy.search.Switchboard;
|
||||
|
||||
public abstract class YMarkImporter implements Runnable {
|
||||
// Statics
|
||||
public final static String XML_NAMESPACE_PREFIXES = "http://xml.org/sax/features/namespace-prefixes";
|
||||
public final static String XML_NAMESPACES = "http://xml.org/sax/features/namespaces";
|
||||
public final static String XML_VALIDATION = "http://xml.org/sax/features/validation";
|
||||
|
||||
protected String importer;
|
||||
protected ArrayBlockingQueue<YMarkEntry> bookmarks;
|
||||
protected final MonitoredReader bmk_file;
|
||||
protected final String targetFolder;
|
||||
protected final String sourceFolder;
|
||||
|
||||
public YMarkImporter(final MonitoredReader bmk_file, final int queueSize, final String sourceFolder, final String targetFolder) {
|
||||
this.bookmarks = new ArrayBlockingQueue<YMarkEntry>(queueSize);
|
||||
this.bmk_file = bmk_file;
|
||||
this.sourceFolder = YMarkUtil.cleanFoldersString(sourceFolder);
|
||||
this.targetFolder = YMarkUtil.cleanFoldersString(targetFolder);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
try {
|
||||
parse();
|
||||
} catch (Exception e) {
|
||||
Log.logException(e);
|
||||
} finally {
|
||||
try {
|
||||
Log.logInfo(YMarkTables.BOOKMARKS_LOG, this.importer+" Importer inserted poison pill in queue");
|
||||
this.bookmarks.put(YMarkEntry.POISON);
|
||||
} catch (InterruptedException e1) {
|
||||
Log.logException(e1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public YMarkEntry take() {
|
||||
try {
|
||||
return this.bookmarks.take();
|
||||
} catch (InterruptedException e) {
|
||||
Log.logException(e);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
public void setImporter(final String importer) {
|
||||
this.importer = importer;
|
||||
}
|
||||
|
||||
public long getProgress() {
|
||||
return this.bmk_file.getProgress();
|
||||
}
|
||||
|
||||
public long maxProgress() {
|
||||
return this.bmk_file.maxProgress();
|
||||
}
|
||||
|
||||
public abstract void parse() throws Exception;
|
||||
|
||||
public Consumer getConsumer(final Switchboard sb, final String bmk_user, final ArrayBlockingQueue<String> autoTaggingQueue,
|
||||
final boolean autotag, final boolean empty, final String indexing, final boolean medialink) {
|
||||
return new Consumer(sb, bmk_user, autoTaggingQueue, autotag, empty, indexing, medialink);
|
||||
}
|
||||
|
||||
public class Consumer implements Runnable {
|
||||
private final Switchboard sb;
|
||||
private final String bmk_user;
|
||||
private final ArrayBlockingQueue<String> autoTaggingQueue;
|
||||
private final String indexing;
|
||||
|
||||
private final boolean autotag;
|
||||
private final boolean empty;
|
||||
private final boolean medialink;
|
||||
|
||||
public Consumer(final Switchboard sb, final String bmk_user, final ArrayBlockingQueue<String> autoTaggingQueue,
|
||||
final boolean autotag, final boolean empty, final String indexing, final boolean medialink) {
|
||||
this.sb = sb;
|
||||
this.bmk_user = bmk_user;
|
||||
this.autoTaggingQueue = autoTaggingQueue;
|
||||
this.autotag = autotag;
|
||||
this.empty = empty;
|
||||
this.indexing = indexing;
|
||||
this.medialink = medialink;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
YMarkEntry bmk;
|
||||
while ((bmk = take()) != YMarkEntry.POISON) {
|
||||
try {
|
||||
final String url = bmk.get(YMarkEntry.BOOKMARK.URL.key());
|
||||
// other protocols could cause problems
|
||||
if(url != null && url.startsWith("http")) {
|
||||
this.sb.tables.bookmarks.addBookmark(this.bmk_user, bmk, true, true);
|
||||
if(this.autotag) {
|
||||
if(!this.empty) {
|
||||
this.autoTaggingQueue.put(url);
|
||||
} else if(!bmk.containsKey(YMarkEntry.BOOKMARK.TAGS.key()) || bmk.get(YMarkEntry.BOOKMARK.TAGS.key()).equals(YMarkEntry.BOOKMARK.TAGS.deflt())) {
|
||||
this.autoTaggingQueue.put(url);
|
||||
}
|
||||
}
|
||||
// fill crawler
|
||||
if (this.indexing.equals("single")) {
|
||||
bmk.crawl(YMarkCrawlStart.CRAWLSTART.SINGLE, this.medialink, this.sb);
|
||||
} else if (this.indexing.equals("onelink")) {
|
||||
bmk.crawl(YMarkCrawlStart.CRAWLSTART.ONE_LINK, this.medialink, this.sb);
|
||||
} else if (this.indexing.equals("fulldomain")) {
|
||||
bmk.crawl(YMarkCrawlStart.CRAWLSTART.FULL_DOMAIN, this.medialink, this.sb);
|
||||
}
|
||||
}
|
||||
} catch (final IOException e) {
|
||||
Log.logException(e);
|
||||
} catch (final InterruptedException e) {
|
||||
Log.logException(e);
|
||||
}
|
||||
}
|
||||
if(this.autotag) {
|
||||
try {
|
||||
this.autoTaggingQueue.put(YMarkAutoTagger.POISON);
|
||||
Log.logInfo(YMarkTables.BOOKMARKS_LOG, YMarkImporter.this.importer+" inserted poison pill into autoTagging queue");
|
||||
} catch (final InterruptedException e) {
|
||||
Log.logException(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,161 @@
|
||||
package de.anomic.data.ymark;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.UnsupportedEncodingException;
|
||||
import java.util.HashMap;
|
||||
import java.util.Iterator;
|
||||
import java.util.Map;
|
||||
|
||||
import net.yacy.cora.document.UTF8;
|
||||
import net.yacy.cora.lod.vocabulary.AnnoteaA;
|
||||
import net.yacy.cora.lod.vocabulary.AnnoteaB;
|
||||
import net.yacy.cora.lod.vocabulary.DCElements;
|
||||
import net.yacy.cora.lod.vocabulary.Rdf;
|
||||
import net.yacy.kelondro.blob.Tables;
|
||||
|
||||
import com.hp.hpl.jena.rdf.model.Bag;
|
||||
import com.hp.hpl.jena.rdf.model.Model;
|
||||
import com.hp.hpl.jena.rdf.model.ModelFactory;
|
||||
import com.hp.hpl.jena.rdf.model.Property;
|
||||
import com.hp.hpl.jena.rdf.model.Resource;
|
||||
|
||||
public class YMarkRDF {
|
||||
|
||||
public final Model model;
|
||||
|
||||
public final static String USER = "USER";
|
||||
public final static String TYPE = "TYPE";
|
||||
public final static String SUBTOPIC = "SUBTOPIC";
|
||||
|
||||
private final Map<String, Property> property;
|
||||
|
||||
public final static String BOOKMARK = "/YMarks.rdf?id=";
|
||||
private final StringBuilder resourceURI;
|
||||
private final int len;
|
||||
|
||||
public YMarkRDF(final String peerURI) {
|
||||
this.model = ModelFactory.createDefaultModel();
|
||||
this.property = new HashMap<String, Property>();
|
||||
|
||||
this.len = peerURI.length()+BOOKMARK.length();
|
||||
this.resourceURI = new StringBuilder(len+20);
|
||||
this.resourceURI.append(peerURI);
|
||||
this.resourceURI.append(BOOKMARK);
|
||||
|
||||
model.setNsPrefix(Rdf.PREFIX, Rdf.IDENTIFIER);
|
||||
model.setNsPrefix(DCElements.PREFIX, DCElements.IDENTIFIER);
|
||||
model.setNsPrefix(AnnoteaA.PREFIX, AnnoteaA.NAMESPACE);
|
||||
model.setNsPrefix(AnnoteaB.PREFIX, AnnoteaB.NAMESPACE);
|
||||
|
||||
this.property.put(YMarkEntry.BOOKMARK.URL.key(), this.model.createProperty(AnnoteaB.recalls.getNamespace(), AnnoteaB.recalls.name()));
|
||||
this.property.put(YMarkEntry.BOOKMARK.FOLDERS.key(), this.model.createProperty(AnnoteaB.hasTopic.getNamespace(), AnnoteaB.hasTopic.name()));
|
||||
this.property.put(YMarkEntry.BOOKMARK.TITLE.key(), this.model.createProperty(DCElements.title.getNamespace(), DCElements.title.name()));
|
||||
this.property.put(YMarkEntry.BOOKMARK.DESC.key(), this.model.createProperty(DCElements.description.getNamespace(), DCElements.description.name()));
|
||||
this.property.put(YMarkEntry.BOOKMARK.DATE_ADDED.key(), this.model.createProperty(AnnoteaA.created.getNamespace(), AnnoteaA.created.name()));
|
||||
this.property.put(YMarkEntry.BOOKMARK.DATE_MODIFIED.key(), this.model.createProperty(DCElements.date.getNamespace(), DCElements.date.name()));
|
||||
this.property.put(YMarkEntry.BOOKMARK.TAGS.key(), this.model.createProperty(DCElements.subject.getNamespace(), DCElements.subject.name()));
|
||||
|
||||
this.property.put(USER, this.model.createProperty(DCElements.creator.getNamespace(), DCElements.creator.name()));
|
||||
this.property.put(TYPE, this.model.createProperty(Rdf.type.getNamespace(), Rdf.type.name()));
|
||||
this.property.put(SUBTOPIC, this.model.createProperty(AnnoteaB.subTopicOf.getNamespace(), AnnoteaB.subTopicOf.name()));
|
||||
}
|
||||
|
||||
/**
|
||||
* @param format {RDF/XML, RDF/XML-ABBREV, N-TRIPLE, N3, N3-PP, N3-PLAIN, N3-TRIPLE, TURTLE}
|
||||
* @return RDF
|
||||
*/
|
||||
public String getRDF(final String format) {
|
||||
ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
||||
this.model.write(baos, format);
|
||||
try {
|
||||
return baos.toString("UTF-8");
|
||||
} catch (UnsupportedEncodingException e) {
|
||||
return new String();
|
||||
}
|
||||
}
|
||||
|
||||
public void addTopic(final String bmk_user, final String folder) {
|
||||
this.resourceURI.append(bmk_user);
|
||||
this.resourceURI.append(":f:");
|
||||
this.resourceURI.append(UTF8.String(YMarkUtil.getKeyId(folder)));
|
||||
final Resource topic = this.model.createResource(this.resourceURI.toString());
|
||||
this.resourceURI.setLength(this.len);
|
||||
|
||||
topic.addProperty(this.property.get(YMarkEntry.BOOKMARK.DATE_MODIFIED.key()), YMarkUtil.EMPTY_STRING);
|
||||
topic.addProperty(this.property.get(YMarkEntry.BOOKMARK.DATE_ADDED.key()), YMarkUtil.EMPTY_STRING);
|
||||
topic.addProperty(this.property.get(USER), bmk_user);
|
||||
topic.addProperty(this.property.get(YMarkEntry.BOOKMARK.DESC.key()), YMarkUtil.EMPTY_STRING);
|
||||
final int i = folder.lastIndexOf(YMarkUtil.FOLDERS_SEPARATOR);
|
||||
if(i>0)
|
||||
topic.addProperty(this.property.get(SUBTOPIC), folder.substring(0, i));
|
||||
topic.addProperty(this.property.get(YMarkEntry.BOOKMARK.TITLE.key()), folder);
|
||||
topic.addProperty(this.property.get(TYPE), AnnoteaB.Topic.getPredicate());
|
||||
}
|
||||
|
||||
public void addBookmark (final String bmk_user, final Tables.Row bmk_row) {
|
||||
if(bmk_row == null || bmk_row.get(YMarkEntry.BOOKMARK.PUBLIC.key(), YMarkEntry.BOOKMARK.PUBLIC.deflt()).equals("false"))
|
||||
return;
|
||||
// create an annotea bookmark resource
|
||||
this.resourceURI.append(bmk_user);
|
||||
this.resourceURI.append(":b:");
|
||||
this.resourceURI.append(UTF8.String(bmk_row.getPK()));
|
||||
final Resource bmk = this.model.createResource(this.resourceURI.toString());
|
||||
this.resourceURI.setLength(this.len);
|
||||
|
||||
// add properties
|
||||
bmk.addProperty(this.property.get(USER), bmk_user);
|
||||
for (final YMarkEntry.BOOKMARK b : YMarkEntry.BOOKMARK.values()) {
|
||||
switch(b) {
|
||||
case FOLDERS:
|
||||
final String[] folders = bmk_row.get(b.key(), b.deflt()).split(b.seperator());
|
||||
if(folders.length > 1) {
|
||||
Bag topics = this.model.createBag();
|
||||
for(String folder : folders) {
|
||||
topics.add(folder);
|
||||
this.addTopic(bmk_user, folder);
|
||||
}
|
||||
bmk.addProperty(this.property.get(b.key()), topics);
|
||||
} else {
|
||||
bmk.addProperty(this.property.get(b.key()), folders[0]);
|
||||
this.addTopic(bmk_user, folders[0]);
|
||||
}
|
||||
break;
|
||||
case TAGS:
|
||||
final String[] tags = bmk_row.get(b.key(), b.deflt()).split(b.seperator());
|
||||
if(tags.length > 1) {
|
||||
Bag subjects = this.model.createBag();
|
||||
for(String tag : tags) {
|
||||
subjects.add(tag);
|
||||
}
|
||||
bmk.addProperty(this.property.get(b.key()), subjects);
|
||||
} else {
|
||||
bmk.addProperty(this.property.get(b.key()), tags[0]);
|
||||
}
|
||||
break;
|
||||
|
||||
case DATE_ADDED:
|
||||
case DATE_MODIFIED:
|
||||
final YMarkDate date = new YMarkDate(bmk_row.get(b.key()));
|
||||
bmk.addProperty(this.property.get(b.key()), date.toISO8601());
|
||||
break;
|
||||
// these cases are inserted for better readable RDF output
|
||||
case DESC:
|
||||
case URL:
|
||||
case TITLE:
|
||||
bmk.addProperty(this.property.get(b.key()), bmk_row.get(b.key(), b.deflt()));
|
||||
break;
|
||||
default:
|
||||
if(this.property.containsKey(b.key())) {
|
||||
bmk.addProperty(this.property.get(b.key()), bmk_row.get(b.key(), b.deflt()));
|
||||
}
|
||||
}
|
||||
}
|
||||
bmk.addProperty(this.property.get(TYPE), AnnoteaB.Bookmark.getPredicate());
|
||||
}
|
||||
|
||||
public void addBookmarks(final String bmk_user, final Iterator<Tables.Row> riter) {
|
||||
while(riter.hasNext()) {
|
||||
this.addBookmark(bmk_user, riter.next());
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,92 @@
|
||||
/**
|
||||
* AnnoteaA
|
||||
* Copyright 2011 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany
|
||||
* First released 16.12.2011 at http://yacy.net
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
* License as published by the Free Software Foundation; either
|
||||
* version 2.1 of the License, or (at your option) any later version.
|
||||
*
|
||||
* This library is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
* Lesser General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public License
|
||||
* along with this program in the file lgpl21.txt
|
||||
* If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
|
||||
package net.yacy.cora.lod.vocabulary;
|
||||
|
||||
import java.util.Set;
|
||||
|
||||
import net.yacy.cora.lod.Literal;
|
||||
import net.yacy.cora.lod.Vocabulary;
|
||||
|
||||
|
||||
/**
|
||||
* Annotea [Annotea] is a W3C Semantic Web Advanced Development project that
|
||||
* provides a framework for rich communication about Web pages through shared RDF metadata.
|
||||
*
|
||||
* The Annotea Annotation schema [AnnotationNS] defines properties for identifying
|
||||
* the document being annotated, a specific context within that document to which
|
||||
* the body of the annotation refers, the author of the annotation, and more.
|
||||
*
|
||||
* http://www.w3.org/2003/07/Annotea/BookmarkSchema-20030707
|
||||
*/
|
||||
public enum AnnoteaA implements Vocabulary {
|
||||
|
||||
Annotation, // The target type of a annotation resource.
|
||||
|
||||
annotates, // Relates an Annotation to the resource to which the Annotation applies. The inverse relation is 'hasAnnotation'
|
||||
|
||||
author, // The name of the person or organization most responsible for creating the Annotation. Sub property of dc:creator
|
||||
|
||||
body, // Relates the resource representing the 'content' of an Annotation to the Annotation resourceSub property of related
|
||||
|
||||
context, // The context within the resource named in 'annotates' to which the Annotation most directly applies
|
||||
|
||||
created, // The date and time on which the Annotation was created. yyyy-mm-ddThh:mm:ssZ format recommended.Sub property of dc:date
|
||||
|
||||
modified, // The date and time on which the Annotation was modified. yyyy-mm-ddThh:mm:ssZ format recommended.Sub property of dc:date
|
||||
|
||||
related; // A relationship between an annotation and additional resources that is less specific than 'body'.
|
||||
// The 'related' property is expected to be subclassed by more specific relationships
|
||||
|
||||
public final static String NAMESPACE = "http://www.w3.org/2000/10/annotation-ns#";
|
||||
public final static String PREFIX = "a";
|
||||
|
||||
private final String predicate;
|
||||
|
||||
private AnnoteaA() {
|
||||
this.predicate = NAMESPACE + this.name();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getNamespace() {
|
||||
return NAMESPACE;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getNamespacePrefix() {
|
||||
return PREFIX;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Set<Literal> getLiterals() {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getPredicate() {
|
||||
return this.predicate;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getURIref() {
|
||||
return PREFIX + ':' + this.name();
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,97 @@
|
||||
/**
|
||||
* AnnoteaB
|
||||
* Copyright 2011 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany
|
||||
* First released 16.12.2011 at http://yacy.net
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
* License as published by the Free Software Foundation; either
|
||||
* version 2.1 of the License, or (at your option) any later version.
|
||||
*
|
||||
* This library is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
* Lesser General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public License
|
||||
* along with this program in the file lgpl21.txt
|
||||
* If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
|
||||
package net.yacy.cora.lod.vocabulary;
|
||||
|
||||
import java.util.Set;
|
||||
|
||||
import net.yacy.cora.lod.Literal;
|
||||
import net.yacy.cora.lod.Vocabulary;
|
||||
|
||||
|
||||
/**
|
||||
* Annotea [Annotea] is a W3C Semantic Web Advanced Development project that
|
||||
* provides a framework for rich communication about Web pages through shared RDF metadata.
|
||||
*
|
||||
* The Annotea Bookmark schema [BookmarkNS] provides the basic concepts found in common browser bookmark implementations.
|
||||
* These basic concepts are also captured in the XML Bookmark Exchange Language [XBEL].
|
||||
* The use of RDF in Annotea permits bookmarks to express additional semantics.
|
||||
* XBEL can be easily mapped into this schema.
|
||||
*
|
||||
* http://www.w3.org/2003/07/Annotea/BookmarkSchema-20030707
|
||||
*/
|
||||
public enum AnnoteaB implements Vocabulary {
|
||||
|
||||
Bookmark, // The class to which all bookmarks belong
|
||||
|
||||
Shortcut, // Specifies a behavior; when the object of type 'Shortcut' is activated, the client follows the 'recalls' property
|
||||
// and activates the object at the end of that 'recalls' property. The target object may be another Bookmark or may be a Topic.
|
||||
|
||||
Topic, //
|
||||
|
||||
bookmarks, // This corresponds to XBEL:href an object of type Bookmark is expected to have a 'recalls' relationship to the document being bookmarked.
|
||||
// The 'bookmarks' property is an older name for the 'recalls' relationship.
|
||||
|
||||
hasTopic, // relates a bookmark to a topic. A bookmark must have at least one hasTopic property. The typical user operation of following a bookmark link
|
||||
// will use the value of the b:recalls property. This property corresponds to XBEL:href property.An instance of
|
||||
leadsTo, // connects a Shortcut to the bookmark or topic that is being included by reference in some other topic
|
||||
|
||||
recalls, // Relates a bookmark with the resource that has been bookmarked. This corresponds to XBEL:href;
|
||||
// an object of type Bookmark is expected to have a 'recalls' relationship to the document being bookmarked
|
||||
|
||||
subTopicOf; // Describes a relationship between Topics. When a topic T is a sub-topic of a topic U then all bookmarks that have topic T are also considered to have topic U.
|
||||
// A topic may be a sub-topic of one or more topics; trivially, every topic is a sub-topic of itself.
|
||||
// More formally; for all B, T, and U: b b:hasTopic T, T b:subTopicOf U implies B b:hasTopic U.
|
||||
|
||||
public final static String NAMESPACE = "http://www.w3.org/2002/01/bookmark#";
|
||||
public final static String PREFIX = "b";
|
||||
|
||||
private final String predicate;
|
||||
|
||||
private AnnoteaB() {
|
||||
this.predicate = NAMESPACE + this.name();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getNamespace() {
|
||||
return NAMESPACE;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getNamespacePrefix() {
|
||||
return PREFIX;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Set<Literal> getLiterals() {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getPredicate() {
|
||||
return this.predicate;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getURIref() {
|
||||
return PREFIX + ':' + this.name();
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,76 @@
|
||||
/**
|
||||
* DublinCore
|
||||
* Copyright 2011 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany
|
||||
* First released 16.12.2011 at http://yacy.net
|
||||
*
|
||||
* $LastChangedDate: 2011-04-14 00:04:23 +0200 (Do, 14 Apr 2011) $
|
||||
* $LastChangedRevision: 7653 $
|
||||
* $LastChangedBy: orbiter $
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
* License as published by the Free Software Foundation; either
|
||||
* version 2.1 of the License, or (at your option) any later version.
|
||||
*
|
||||
* This library is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
* Lesser General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public License
|
||||
* along with this program in the file lgpl21.txt
|
||||
* If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
|
||||
package net.yacy.cora.lod.vocabulary;
|
||||
|
||||
import java.util.Set;
|
||||
|
||||
import net.yacy.cora.lod.Literal;
|
||||
import net.yacy.cora.lod.Vocabulary;
|
||||
|
||||
public enum DCElements implements Vocabulary {
|
||||
|
||||
|
||||
creator,
|
||||
date,
|
||||
description,
|
||||
subject,
|
||||
title;
|
||||
|
||||
public final static String IDENTIFIER = "http://purl.org/dc/elements/1.1/";
|
||||
public final static String PREFIX = "dc";
|
||||
|
||||
private final String predicate, uriref;
|
||||
|
||||
private DCElements() {
|
||||
this.predicate = IDENTIFIER + this.name().toLowerCase();
|
||||
this.uriref = PREFIX + ':' + this.name().toLowerCase();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getNamespace() {
|
||||
return IDENTIFIER;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getNamespacePrefix() {
|
||||
return PREFIX;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Set<Literal> getLiterals() {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getPredicate() {
|
||||
return this.predicate;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getURIref() {
|
||||
return this.uriref;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,104 @@
|
||||
/**
|
||||
* DMOZ
|
||||
* Copyright 2011 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany
|
||||
* First released 16.12.2011 at http://yacy.net
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
* License as published by the Free Software Foundation; either
|
||||
* version 2.1 of the License, or (at your option) any later version.
|
||||
*
|
||||
* This library is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
* Lesser General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public License
|
||||
* along with this program in the file lgpl21.txt
|
||||
* If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
|
||||
package net.yacy.cora.lod.vocabulary;
|
||||
|
||||
import java.util.Set;
|
||||
|
||||
import net.yacy.cora.lod.Literal;
|
||||
import net.yacy.cora.lod.Vocabulary;
|
||||
|
||||
/**
|
||||
* The Open Directory Project is the largest, most comprehensive human-edited directory of the Web.
|
||||
* It is constructed and maintained by a vast, global community of volunteer editors.
|
||||
*
|
||||
* RDF dumps of the Open Directory database are available for download at http://www.dmoz.org/rdf.html *
|
||||
* An overview of the vocabulary can be found at http://rdf.dmoz.org/rdf/tags.html
|
||||
*/
|
||||
public enum DMOZ implements Vocabulary {
|
||||
|
||||
// Content
|
||||
ExternalPage,
|
||||
atom,
|
||||
link,
|
||||
link1,
|
||||
mediadate,
|
||||
pdf,
|
||||
pdf1,
|
||||
priority,
|
||||
rss,
|
||||
rss1,
|
||||
topic,
|
||||
type,
|
||||
|
||||
// Structure
|
||||
Alias,
|
||||
Target,
|
||||
Topic,
|
||||
altlang,
|
||||
altlang1,
|
||||
catid,
|
||||
editor,
|
||||
lastUpdate,
|
||||
letterbar,
|
||||
narrow,
|
||||
narrow1,
|
||||
narrow2,
|
||||
newsgroup,
|
||||
related,
|
||||
symbolic,
|
||||
symbolic1,
|
||||
symbolic2;
|
||||
|
||||
public final static String NAMESPACE = "http://dmoz.org/rdf/";
|
||||
public final static String PREFIX = "dmoz";
|
||||
|
||||
private final String predicate;
|
||||
|
||||
private DMOZ() {
|
||||
this.predicate = NAMESPACE + this.name().toLowerCase();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getNamespace() {
|
||||
return NAMESPACE;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getNamespacePrefix() {
|
||||
return PREFIX;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Set<Literal> getLiterals() {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getPredicate() {
|
||||
return this.predicate;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getURIref() {
|
||||
return PREFIX + ':' + this.name();
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,205 @@
|
||||
// TablesColumnBLOBIndex.java
|
||||
// (C) 2012 by Stefan Foerster, sof@gmx.de, Norderstedt, Germany
|
||||
// first published 2012 on http://yacy.net
|
||||
//
|
||||
// This is a part of YaCy, a peer-to-peer based web search engine
|
||||
//
|
||||
// LICENSE
|
||||
//
|
||||
// This program is free software; you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation; either version 2 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with this program; if not, write to the Free Software
|
||||
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
|
||||
package net.yacy.kelondro.blob;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.Iterator;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.TreeSet;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
|
||||
import net.yacy.cora.util.SpaceExceededException;
|
||||
import net.yacy.kelondro.logging.Log;
|
||||
import net.yacy.kelondro.util.ByteBuffer;
|
||||
import de.anomic.data.ymark.YMarkUtil;
|
||||
|
||||
public class TablesColumnBLOBIndex extends TablesColumnIndex{
|
||||
|
||||
// Map<ColumnName, Map<ColumnValue, T<PrimaryKey>>>
|
||||
private final BEncodedHeap index;
|
||||
private final static byte SEPERATOR = (byte) ',';
|
||||
|
||||
public TablesColumnBLOBIndex(final BEncodedHeap bheap) {
|
||||
super(TablesColumnIndex.INDEXTYPE.BLOB);
|
||||
this.index = bheap;
|
||||
}
|
||||
|
||||
public static Collection<byte[]> byteToCollection(final byte[] b) {
|
||||
final Collection<byte[]> PKset = ByteBuffer.split(b, SEPERATOR);
|
||||
return PKset;
|
||||
}
|
||||
|
||||
public static byte[] CollectionToByte(final Collection<byte[]> bc) {
|
||||
final ByteBuffer buf = new ByteBuffer(15 * bc.size());
|
||||
final Iterator<byte[]> iter = bc.iterator();
|
||||
while(iter.hasNext()) {
|
||||
buf.append(iter.next());
|
||||
buf.append(SEPERATOR);
|
||||
}
|
||||
return buf.getBytes();
|
||||
}
|
||||
|
||||
public void deleteIndex(final String columnName) {
|
||||
final byte[] column = YMarkUtil.getKeyId(columnName);
|
||||
try {
|
||||
this.index.remove(column);
|
||||
} catch (IOException e) {
|
||||
Log.logException(e);
|
||||
} catch (SpaceExceededException e) {
|
||||
Log.logException(e);
|
||||
}
|
||||
}
|
||||
|
||||
protected void insertPK(final String columnName, final String columnValue, final byte[] pk) {
|
||||
Map<String, byte[]> valueIdxMap;
|
||||
Collection<byte[]> PKset;
|
||||
final byte[] column = YMarkUtil.getKeyId(columnName);
|
||||
try {
|
||||
valueIdxMap = this.index.get(column);
|
||||
if(valueIdxMap != null) {
|
||||
if(valueIdxMap.containsKey(columnValue)) {
|
||||
PKset = byteToCollection(valueIdxMap.get(columnValue));
|
||||
if(!ByteBuffer.contains(PKset, pk)) {
|
||||
PKset.add(pk);
|
||||
}
|
||||
} else {
|
||||
PKset = new ArrayList<byte[]>(1);
|
||||
PKset.add(pk);
|
||||
valueIdxMap.put(columnValue, CollectionToByte(PKset));
|
||||
}
|
||||
} else {
|
||||
PKset = new ArrayList<byte[]>(1);
|
||||
PKset.add(pk);
|
||||
valueIdxMap = new ConcurrentHashMap<String, byte[]>();
|
||||
}
|
||||
valueIdxMap.put(columnValue, CollectionToByte(PKset));
|
||||
this.index.insert(column, valueIdxMap);
|
||||
return;
|
||||
} catch (IOException e) {
|
||||
Log.logException(e);
|
||||
} catch (SpaceExceededException e) {
|
||||
Log.logException(e);
|
||||
}
|
||||
}
|
||||
|
||||
protected void removePK(final byte[] pk) {
|
||||
final Iterator<Map.Entry<byte[], Map<String, byte[]>>> niter = this.index.iterator();
|
||||
while (niter.hasNext()) {
|
||||
final Map.Entry<byte[], Map<String,byte[]>> entry = niter.next();
|
||||
final Iterator<Map.Entry<String, byte[]>> viter = entry.getValue().entrySet().iterator();
|
||||
while(viter.hasNext()) {
|
||||
final Map.Entry<String, byte[]> columnValue = viter.next();
|
||||
final Collection<byte[]> PKset = byteToCollection(columnValue.getValue());
|
||||
ByteBuffer.remove(PKset, pk);
|
||||
if(PKset.isEmpty()) {
|
||||
viter.remove();
|
||||
} else {
|
||||
columnValue.setValue(CollectionToByte(PKset));
|
||||
}
|
||||
}
|
||||
try {
|
||||
this.index.insert(entry.getKey(), entry.getValue());
|
||||
} catch (SpaceExceededException e) {
|
||||
Log.logException(e);
|
||||
} catch (IOException e) {
|
||||
Log.logException(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void clear() {
|
||||
this.index.clear();
|
||||
}
|
||||
|
||||
public Collection<String> columns() {
|
||||
return this.index.columns();
|
||||
}
|
||||
|
||||
public Set<String> keySet(final String columnName) {
|
||||
final byte[] column = YMarkUtil.getKeyId(columnName);
|
||||
// a TreeSet is used to get sorted set of keys (e.g. folders)
|
||||
if(this.index.containsKey(column)) {
|
||||
try {
|
||||
return new TreeSet<String>(this.index.get(column).keySet());
|
||||
} catch (IOException e) {
|
||||
Log.logException(e);
|
||||
} catch (SpaceExceededException e) {
|
||||
Log.logException(e);
|
||||
}
|
||||
}
|
||||
return new TreeSet<String>();
|
||||
}
|
||||
|
||||
public boolean containsKey(final String columnName, final String key) {
|
||||
final byte[] column = YMarkUtil.getKeyId(columnName);
|
||||
if(this.index.containsKey(column)) {
|
||||
try {
|
||||
return this.index.get(column).containsKey(key);
|
||||
} catch (IOException e) {
|
||||
Log.logException(e);
|
||||
} catch (SpaceExceededException e) {
|
||||
Log.logException(e);
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
public boolean hasIndex(final String columnName) {
|
||||
final byte[] column = YMarkUtil.getKeyId(columnName);
|
||||
return this.index.containsKey(column);
|
||||
}
|
||||
|
||||
public Collection<byte[]> get(final String columnName, final String key) {
|
||||
final byte[] column = YMarkUtil.getKeyId(columnName);
|
||||
// deserialize
|
||||
try {
|
||||
return byteToCollection(this.index.get(column).get(key));
|
||||
} catch (IOException e) {
|
||||
Log.logException(e);
|
||||
} catch (SpaceExceededException e) {
|
||||
Log.logException(e);
|
||||
}
|
||||
return new ArrayList<byte[]>();
|
||||
}
|
||||
|
||||
public int size(final String columnName) {
|
||||
final byte[] column = YMarkUtil.getKeyId(columnName);
|
||||
if(this.index.containsKey(column)) {
|
||||
try {
|
||||
return this.index.get(column).size();
|
||||
} catch (IOException e) {
|
||||
Log.logException(e);
|
||||
} catch (SpaceExceededException e) {
|
||||
Log.logException(e);
|
||||
}
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
public int size() {
|
||||
return this.index.size();
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,176 @@
|
||||
// TablesColumnIndex.java
|
||||
// (C) 2012 by Stefan Foerster, sof@gmx.de, Norderstedt, Germany
|
||||
// first published 2012 on http://yacy.net
|
||||
//
|
||||
// This is a part of YaCy, a peer-to-peer based web search engine
|
||||
//
|
||||
// LICENSE
|
||||
//
|
||||
// This program is free software; you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation; either version 2 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with this program; if not, write to the Free Software
|
||||
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
|
||||
package net.yacy.kelondro.blob;
|
||||
|
||||
import java.util.Collection;
|
||||
import java.util.Comparator;
|
||||
import java.util.Iterator;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import net.yacy.cora.document.UTF8;
|
||||
import net.yacy.kelondro.order.NaturalOrder;
|
||||
|
||||
|
||||
/**
|
||||
* a mapping from a column name to maps with the value of the columns to the primary keys where the entry exist in the table
|
||||
*/
|
||||
public abstract class TablesColumnIndex {
|
||||
|
||||
public static enum INDEXTYPE {RAM, BLOB}
|
||||
private INDEXTYPE type;
|
||||
// Map<ColumnName, Map<ColumnValue, T<PrimaryKey>>>
|
||||
// private final Map<String, Map<String, TreeSet<byte[]>>> index;
|
||||
|
||||
protected final static Comparator<byte[]> NATURALORDER = new NaturalOrder(true);
|
||||
|
||||
protected abstract void insertPK(final String columnName, final String columnValue, final byte[] pk);
|
||||
protected abstract void removePK(final byte[] pk);
|
||||
protected abstract void clear();
|
||||
|
||||
public abstract Set<String> keySet(final String columnName);
|
||||
public abstract boolean containsKey(final String columnName, final String key);
|
||||
public abstract boolean hasIndex(final String columnName);
|
||||
public abstract Collection<byte[]> get(final String columnName, final String key);
|
||||
public abstract int size(final String columnName);
|
||||
public abstract int size();
|
||||
public abstract Collection<String> columns();
|
||||
public abstract void deleteIndex(final String columnName);
|
||||
|
||||
public TablesColumnIndex(INDEXTYPE type) {
|
||||
this.type = type;
|
||||
}
|
||||
|
||||
public INDEXTYPE getType() {
|
||||
return this.type;
|
||||
}
|
||||
|
||||
/**
|
||||
* create an index for a given table and column
|
||||
* @param columnName - name of the column you want to build an index for
|
||||
* @param valueIsArray - indicates whether the column value consist of an array (e.g. comma separated tags)
|
||||
* @param separator - a string value used to split column values into an array
|
||||
* @param table - an iterator over table rows which should be added to the index
|
||||
*/
|
||||
public synchronized void buildIndex(final String columnName, final String separator, final Iterator<Tables.Row> table) {
|
||||
this.deleteIndex(columnName);
|
||||
// loop through all rows of the table
|
||||
while (table.hasNext()) {
|
||||
this.add(columnName, separator, table.next());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* create an index for a given table and column
|
||||
* @param columnName - name of the column you want to build an index for
|
||||
* @param table - an iterator over table rows which should be added to the index
|
||||
*/
|
||||
public synchronized void buildIndex(final String columnName, final Iterator<Tables.Row> table) {
|
||||
this.buildIndex(columnName, "", table);
|
||||
}
|
||||
|
||||
/**
|
||||
* create an index for a given table and given columns
|
||||
* @param columns - a map of column names and booleans for 'valueIsArray' you want to build an index for
|
||||
* @param separator - a string value used to split column values into an array
|
||||
* @param table - an iterator over table rows which should be added to the index
|
||||
*/
|
||||
public synchronized void buildIndex(final Map<String,String> columns, final Iterator<Tables.Row> table) {
|
||||
this.clear();
|
||||
// loop through all rows of the table
|
||||
while (table.hasNext()) {
|
||||
this.add(columns, table.next());
|
||||
}
|
||||
}
|
||||
|
||||
private void insertPK(final String columnName, final String[] columnValues, final byte[] pk) {
|
||||
for (String columnValue : columnValues) {
|
||||
this.insertPK(columnName, columnValue, pk);
|
||||
}
|
||||
}
|
||||
|
||||
public void delete(final byte[] pk) {
|
||||
this.removePK(pk);
|
||||
}
|
||||
|
||||
public void delete(final Tables.Row row) {
|
||||
this.removePK(row.getPK());
|
||||
}
|
||||
|
||||
public void update(final String columnName, final String separator, final Tables.Row row) {
|
||||
this.removePK(row.getPK());
|
||||
this.add(columnName, separator, row);
|
||||
}
|
||||
|
||||
public void update(final Map<String,String> columns, final Tables.Row row) {
|
||||
this.removePK(row.getPK());
|
||||
this.add(columns, row);
|
||||
}
|
||||
|
||||
public void add(final String columnName, final String separator, final Map<String,String> map, final byte[] pk) {
|
||||
if(separator.isEmpty())
|
||||
this.insertPK(columnName, map.get(columnName), pk);
|
||||
else
|
||||
this.insertPK(columnName, map.get(columnName).split(separator), pk);
|
||||
}
|
||||
|
||||
public void add(final String columnName, final String separator, final Tables.Data row, final byte[] pk) {
|
||||
if(separator.isEmpty())
|
||||
this.insertPK(columnName, UTF8.String(row.get(columnName)), pk);
|
||||
else
|
||||
this.insertPK(columnName, UTF8.String(row.get(columnName)).split(separator), pk);
|
||||
}
|
||||
|
||||
public void add(final String columnName, final String separator, final Tables.Row row) {
|
||||
if(separator.isEmpty())
|
||||
this.insertPK(columnName, UTF8.String(row.get(columnName)), row.getPK());
|
||||
else
|
||||
this.insertPK(columnName, UTF8.String(row.get(columnName)).split(separator), row.getPK());
|
||||
}
|
||||
|
||||
public void add(final Map<String,String> columns, final Map<String,String> map, final byte[] pk) {
|
||||
final Iterator<String> iter = columns.keySet().iterator();
|
||||
while (iter.hasNext()) {
|
||||
final String columnName = iter.next();
|
||||
if(columns.get(columnName).isEmpty())
|
||||
this.insertPK(columnName, map.get(columnName), pk);
|
||||
else
|
||||
this.insertPK(columnName, map.get(columnName).split(columns.get(columnName)), pk);
|
||||
}
|
||||
}
|
||||
|
||||
public void add(final Map<String,String> columns, final Tables.Data row, final byte[] pk) {
|
||||
final Iterator<String> iter = columns.keySet().iterator();
|
||||
while (iter.hasNext()) {
|
||||
final String columnName = iter.next();
|
||||
if(columns.get(columnName).isEmpty())
|
||||
this.insertPK(columnName, UTF8.String(row.get(columnName)), pk);
|
||||
else
|
||||
this.insertPK(columnName, UTF8.String(row.get(columnName)).split(columns.get(columnName)), pk);
|
||||
}
|
||||
}
|
||||
|
||||
public void add(final Map<String,String> columns, final Tables.Row row) {
|
||||
this.add(columns, row, row.getPK());
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,124 @@
|
||||
// TablesColumnRAMIndex.java
|
||||
// (C) 2012 by Stefan Foerster, sof@gmx.de, Norderstedt, Germany
|
||||
// first published 2012 on http://yacy.net
|
||||
//
|
||||
// This is a part of YaCy, a peer-to-peer based web search engine
|
||||
//
|
||||
// LICENSE
|
||||
//
|
||||
// This program is free software; you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation; either version 2 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with this program; if not, write to the Free Software
|
||||
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
|
||||
package net.yacy.kelondro.blob;
|
||||
|
||||
import java.util.Collection;
|
||||
import java.util.Comparator;
|
||||
import java.util.Iterator;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.TreeSet;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
|
||||
import net.yacy.kelondro.order.NaturalOrder;
|
||||
|
||||
public class TablesColumnRAMIndex extends TablesColumnIndex{
|
||||
|
||||
// Map<ColumnName, Map<ColumnValue, T<PrimaryKey>>>
|
||||
private final Map<String, Map<String, TreeSet<byte[]>>> index;
|
||||
|
||||
private final static Comparator<byte[]> NATURALORDER = new NaturalOrder(true);
|
||||
|
||||
public TablesColumnRAMIndex() {
|
||||
super(TablesColumnIndex.INDEXTYPE.RAM);
|
||||
this.index = new ConcurrentHashMap<String, Map<String, TreeSet<byte[]>>>();
|
||||
}
|
||||
|
||||
public void deleteIndex(final String columnName) {
|
||||
this.index.remove(columnName);
|
||||
}
|
||||
|
||||
protected void insertPK(final String columnName, final String columnValue, final byte[] pk) {
|
||||
Map<String, TreeSet<byte[]>> valueIdxMap;
|
||||
TreeSet<byte[]> PKset;
|
||||
if(this.index.containsKey(columnName)) {
|
||||
valueIdxMap = this.index.get(columnName);
|
||||
}
|
||||
else {
|
||||
valueIdxMap = new ConcurrentHashMap<String, TreeSet<byte[]>>();
|
||||
this.index.put(columnName, valueIdxMap);
|
||||
}
|
||||
if(valueIdxMap.containsKey(columnValue)) {
|
||||
PKset = valueIdxMap.get(columnValue);
|
||||
}
|
||||
else {
|
||||
PKset = new TreeSet<byte[]>(NATURALORDER);
|
||||
valueIdxMap.put(columnValue, PKset);
|
||||
}
|
||||
PKset.add(pk);
|
||||
}
|
||||
|
||||
protected synchronized void removePK(final byte[] pk) {
|
||||
for(Map.Entry<String, Map<String, TreeSet<byte[]>>> columnName : this.index.entrySet()) {
|
||||
final Iterator<Map.Entry<String, TreeSet<byte[]>>> viter = columnName.getValue().entrySet().iterator();
|
||||
while(viter.hasNext()) {
|
||||
final Map.Entry<String, TreeSet<byte[]>> columnValue = viter.next();
|
||||
columnValue.getValue().remove(pk);
|
||||
if(columnValue.getValue().isEmpty())
|
||||
viter.remove();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void clear() {
|
||||
this.index.clear();
|
||||
}
|
||||
|
||||
public Collection<String> columns() {
|
||||
return this.index.keySet();
|
||||
}
|
||||
|
||||
public Set<String> keySet(final String columnName) {
|
||||
// a TreeSet is used to get sorted set of keys (e.g. folders)
|
||||
if(this.index.containsKey(columnName)) {
|
||||
return new TreeSet<String>(this.index.get(columnName).keySet());
|
||||
}
|
||||
return new TreeSet<String>();
|
||||
}
|
||||
|
||||
public boolean containsKey(final String columnName, final String columnValue) {
|
||||
if(this.index.containsKey(columnName)) {
|
||||
return this.index.get(columnName).containsKey(columnValue);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
public boolean hasIndex(final String columnName) {
|
||||
return this.index.containsKey(columnName);
|
||||
}
|
||||
|
||||
public Collection<byte[]> get(final String columnName, final String key) {
|
||||
return this.index.get(columnName).get(key);
|
||||
}
|
||||
|
||||
public int size(final String columnName) {
|
||||
if(this.index.containsKey(columnName)) {
|
||||
return this.index.get(columnName).size();
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
public int size() {
|
||||
return this.index.size();
|
||||
}
|
||||
}
|
||||
Loading…
Reference in new issue