boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

reduced footprint of BookmarksDB which can take quite a lot of memory if

Browse Source 
the number of bookmarks is high (i.e. > 2000 URLs)
pull/1/head
Michael Peter Christen 11 years ago
parent 81bb50118e
commit c152d996e6
1 changed files with 54 additions and 96 deletions
Show Stats Download Patch File Download Diff File

150
source/net/yacy/data/BookmarksDB.java
Unescape View File

@ -41,7 +41,10 @@ import net.yacy.cora.document.encoding.UTF8;
import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.document.id.DigestURL;
import net.yacy.cora.order.NaturalOrder; import net.yacy.cora.order.NaturalOrder;
import net.yacy.cora.util.ConcurrentLog; import net.yacy.cora.util.ConcurrentLog;
import net.yacy.cora.util.SpaceExceededException;
import net.yacy.kelondro.blob.MapHeap; import net.yacy.kelondro.blob.MapHeap;
import net.yacy.kelondro.data.meta.URIMetadataRow;
import net.yacy.kelondro.index.RowHandleSet;
public class BookmarksDB { public class BookmarksDB {
@ -147,11 +150,6 @@ public class BookmarksDB {
ConcurrentLog.logException(e); ConcurrentLog.logException(e);
} }
} }
public String addBookmark(final Bookmark bookmark){
saveBookmark(bookmark);
return bookmark.getUrlHash();
}
public Bookmark getBookmark(final String urlHash) throws IOException { public Bookmark getBookmark(final String urlHash) throws IOException {
try { try {
@ -214,18 +212,13 @@ public class BookmarksDB {
final TreeSet<String> set=new TreeSet<String>(new bookmarkComparator(true)); final TreeSet<String> set=new TreeSet<String>(new bookmarkComparator(true));
final String tagHash=BookmarkHelper.tagHash(tagName); final String tagHash=BookmarkHelper.tagHash(tagName);
final Tag tag=getTag(tagHash); final Tag tag=getTag(tagHash);
Set<String> hashes=new HashSet<String>(); RowHandleSet hashes = tag == null ? new RowHandleSet(URIMetadataRow.rowdef.primaryKeyLength, URIMetadataRow.rowdef.objectOrder, 10) : tag.getUrlHashes();
if (tag != null) {
hashes=getTag(tagHash).getUrlHashes();
}
if (priv) { if (priv) {
set.addAll(hashes); for (byte[] hash: hashes) set.add(ASCII.String(hash));
} else { } else {
final Iterator<String> it=hashes.iterator(); for (byte[] hash: hashes) {
Bookmark bm;
while(it.hasNext()){
try { try {
bm = getBookmark(it.next()); Bookmark bm = getBookmark(ASCII.String(hash));
if (bm != null && bm.getPublic()) { if (bm != null && bm.getPublic()) {
set.add(bm.getUrlHash()); set.add(bm.getUrlHash());
} }
@ -249,7 +242,7 @@ public class BookmarksDB {
* retrieve an object of type Tag from the the tagCache, if object is not cached return loadTag(hash) * retrieve an object of type Tag from the the tagCache, if object is not cached return loadTag(hash)
* @param hash an object of type String, containing a tagHash * @param hash an object of type String, containing a tagHash
*/ */
public Tag getTag(final String hash){ private Tag getTag(final String hash){
return this.tags.get(hash); //null if it does not exists return this.tags.get(hash); //null if it does not exists
} }
@ -257,7 +250,7 @@ public class BookmarksDB {
* store a Tag in tagsTable or remove an empty tag * store a Tag in tagsTable or remove an empty tag
* @param tag an object of type Tag to be stored/removed * @param tag an object of type Tag to be stored/removed
*/ */
public void putTag(final Tag tag){ private void putTag(final Tag tag){
if (tag == null) return; if (tag == null) return;
if (tag.isEmpty()) { if (tag.isEmpty()) {
this.tags.remove(tag.getTagHash()); this.tags.remove(tag.getTagHash());
@ -266,7 +259,7 @@ public class BookmarksDB {
} }
} }
public void removeTag(final String hash) { private void removeTag(final String hash) {
this.tags.remove(hash); this.tags.remove(hash);
} }
@ -301,7 +294,7 @@ public class BookmarksDB {
return set.iterator(); return set.iterator();
} }
public Iterator<Tag> getTagIterator(final String tagName, final boolean priv, final int comp) { private Iterator<Tag> getTagIterator(final String tagName, final boolean priv, final int comp) {
final TreeSet<Tag> set=new TreeSet<Tag>((comp == SORT_SIZE) ? tagSizeComparator : tagComparator); final TreeSet<Tag> set=new TreeSet<Tag>((comp == SORT_SIZE) ? tagSizeComparator : tagComparator);
Iterator<String> it=null; Iterator<String> it=null;
final Iterator<String> bit=getBookmarksIterator(tagName, priv); final Iterator<String> bit=getBookmarksIterator(tagName, priv);
@ -347,14 +340,14 @@ public class BookmarksDB {
final Tag oldTag=getTag(BookmarkHelper.tagHash(oldName)); final Tag oldTag=getTag(BookmarkHelper.tagHash(oldName));
if (oldTag != null) { if (oldTag != null) {
final Set<String> urlHashes = oldTag.getUrlHashes(); // preserve urlHashes of oldTag final RowHandleSet urlHashes = oldTag.getUrlHashes(); // preserve urlHashes of oldTag
removeTag(BookmarkHelper.tagHash(oldName)); // remove oldHash from TagsDB removeTag(BookmarkHelper.tagHash(oldName)); // remove oldHash from TagsDB
Bookmark bookmark; Bookmark bookmark;
Set<String> tagSet = new TreeSet<String>(String.CASE_INSENSITIVE_ORDER); Set<String> tagSet = new TreeSet<String>(String.CASE_INSENSITIVE_ORDER);
for (final String urlHash : urlHashes) { // looping through all bookmarks which were tagged with oldName for (final byte[] urlHash : urlHashes) { // looping through all bookmarks which were tagged with oldName
try { try {
bookmark = getBookmark(urlHash); bookmark = getBookmark(ASCII.String(urlHash));
tagSet = bookmark.getTags(); tagSet = bookmark.getTags();
tagSet.remove(oldName); tagSet.remove(oldName);
bookmark.setTags(tagSet, true); // might not be needed, but doesn't hurt bookmark.setTags(tagSet, true); // might not be needed, but doesn't hurt
@ -371,9 +364,9 @@ public class BookmarksDB {
public void addTag(final String selectTag, final String newTag) { public void addTag(final String selectTag, final String newTag) {
Bookmark bookmark; Bookmark bookmark;
for (final String urlHash : getTag(BookmarkHelper.tagHash(selectTag)).getUrlHashes()) { // looping through all bookmarks which were tagged with selectTag for (final byte[] urlHash : getTag(BookmarkHelper.tagHash(selectTag)).getUrlHashes()) { // looping through all bookmarks which were tagged with selectTag
try { try {
bookmark = getBookmark(urlHash); bookmark = getBookmark(ASCII.String(urlHash));
bookmark.addTag(newTag); bookmark.addTag(newTag);
saveBookmark(bookmark); saveBookmark(bookmark);
} catch (final IOException e) { } catch (final IOException e) {
@ -389,51 +382,24 @@ public class BookmarksDB {
* Subclass of bookmarksDB, which provides the Tag object-type * Subclass of bookmarksDB, which provides the Tag object-type
*/ */
public class Tag { public class Tag {
public static final String URL_HASHES = "urlHashes";
public static final String TAG_NAME = "tagName";
private final String tagHash; private final String tagHash;
private final Map<String, String> mem; private final String tagName;
private Set<String> urlHashes; private RowHandleSet urlHashes;
public Tag(final String hash, final Map<String, String> map){
this.tagHash = hash;
this.mem = map;
if (this.mem.containsKey(URL_HASHES)) {
this.urlHashes = ListManager.string2set(this.mem.get(URL_HASHES));
} else {
this.urlHashes = new HashSet<String>();
}
}
public Tag(final String name, final HashSet<String> entries){ private Tag(final String name) {
this.tagHash = BookmarkHelper.tagHash(name); this.tagHash = BookmarkHelper.tagHash(name);
this.mem = new HashMap<String, String>(); this.tagName = name;
//mem.put(URL_HASHES, listManager.arraylist2string(entries)); this.urlHashes = new RowHandleSet(URIMetadataRow.rowdef.primaryKeyLength, URIMetadataRow.rowdef.objectOrder, 10);
this.urlHashes = entries;
this.mem.put(TAG_NAME, name);
}
public Tag(final String name){
this(name, new HashSet<String>());
}
public Map<String, String> getMap(){
this.mem.put(URL_HASHES, ListManager.collection2string(this.urlHashes));
return this.mem;
} }
/** /**
* get the lowercase Tagname * get the lowercase Tagname
*/ */
public String getTagName(){ public String getTagName(){
/*if(this.mem.containsKey(TAG_NAME)){
return (String) this.mem.get(TAG_NAME);
}
return "";*/
return getFriendlyName().toLowerCase(); return getFriendlyName().toLowerCase();
} }
public String getTagHash(){ private String getTagHash(){
return this.tagHash; return this.tagHash;
} }
@ -441,37 +407,33 @@ public class BookmarksDB {
* @return the tag name, with all uppercase chars * @return the tag name, with all uppercase chars
*/ */
public String getFriendlyName(){ public String getFriendlyName(){
/*if(this.mem.containsKey(TAG_FRIENDLY_NAME)){ return this.tagName;
return (String) this.mem.get(TAG_FRIENDLY_NAME);
}
return getTagName();*/
if(this.mem.containsKey(TAG_NAME)){
return this.mem.get(TAG_NAME);
}
return "notagname";
} }
public Set<String> getUrlHashes(){ private RowHandleSet getUrlHashes(){
return this.urlHashes; return this.urlHashes;
} }
public boolean hasPublicItems(){ private boolean hasPublicItems(){
return getBookmarksIterator(getTagName(), false).hasNext(); return getBookmarksIterator(getTagName(), false).hasNext();
} }
public void addUrl(final String urlHash){ private void addUrl(final String urlHash){
this.urlHashes.add(urlHash); try {
this.urlHashes.put(ASCII.getBytes(urlHash));
} catch (SpaceExceededException e) {
}
} }
public void delete(final String urlHash){ private void delete(final String urlHash){
this.urlHashes.remove(urlHash); this.urlHashes.remove(ASCII.getBytes(urlHash));
} }
public int size(){ public int size(){
return this.urlHashes.size(); return this.urlHashes.size();
} }
public boolean isEmpty() { private boolean isEmpty() {
return this.urlHashes.isEmpty(); return this.urlHashes.isEmpty();
} }
} }
@ -481,27 +443,19 @@ public class BookmarksDB {
*/ */
public class Bookmark { public class Bookmark {
public static final String BOOKMARK_URL = "bookmarkUrl"; private static final String BOOKMARK_URL = "bookmarkUrl";
public static final String BOOKMARK_TITLE = "bookmarkTitle"; public static final String BOOKMARK_TITLE = "bookmarkTitle";
public static final String BOOKMARK_DESCRIPTION = "bookmarkDesc"; public static final String BOOKMARK_DESCRIPTION = "bookmarkDesc";
public static final String BOOKMARK_TAGS = "bookmarkTags"; private static final String BOOKMARK_TAGS = "bookmarkTags";
public static final String BOOKMARK_PUBLIC = "bookmarkPublic"; private static final String BOOKMARK_PUBLIC = "bookmarkPublic";
public static final String BOOKMARK_TIMESTAMP = "bookmarkTimestamp"; private static final String BOOKMARK_TIMESTAMP = "bookmarkTimestamp";
public static final String BOOKMARK_OWNER = "bookmarkOwner"; private static final String BOOKMARK_OWNER = "bookmarkOwner";
public static final String BOOKMARK_IS_FEED = "bookmarkIsFeed"; private static final String BOOKMARK_IS_FEED = "bookmarkIsFeed";
private final String urlHash; private final String urlHash;
private Set<String> tagNames; private Set<String> tagNames;
private long timestamp; private long timestamp;
private final Map<String, String> entry; private final Map<String, String> entry;
public Bookmark(final String urlHash, final Map<String, String> map) {
this.entry = map;
this.urlHash = urlHash;
this.tagNames = new TreeSet<String>(String.CASE_INSENSITIVE_ORDER);
if (map.containsKey(BOOKMARK_TAGS)) this.tagNames.addAll(ListManager.string2set(map.get(BOOKMARK_TAGS)));
loadTimestamp();
}
public Bookmark(final DigestURL url) { public Bookmark(final DigestURL url) {
this.entry = new HashMap<String, String>(); this.entry = new HashMap<String, String>();
this.urlHash = ASCII.String(url.hash()); this.urlHash = ASCII.String(url.hash());
@ -529,11 +483,15 @@ public class BookmarksDB {
this(new DigestURL((url.indexOf("://") < 0) ? "http://" + url : url)); this(new DigestURL((url.indexOf("://") < 0) ? "http://" + url : url));
} }
public Bookmark(final Map<String, String> map) throws MalformedURLException { private Bookmark(final Map<String, String> map) throws MalformedURLException {
this(ASCII.String((new DigestURL(map.get(BOOKMARK_URL))).hash()), map); this.entry = map;
this.urlHash = ASCII.String((new DigestURL(map.get(BOOKMARK_URL))).hash());
this.tagNames = new TreeSet<String>(String.CASE_INSENSITIVE_ORDER);
if (map.containsKey(BOOKMARK_TAGS)) this.tagNames.addAll(ListManager.string2set(map.get(BOOKMARK_TAGS)));
loadTimestamp();
} }
Map<String, String> toMap() { private Map<String, String> toMap() {
this.entry.put(BOOKMARK_TAGS, ListManager.collection2string(this.tagNames)); this.entry.put(BOOKMARK_TAGS, ListManager.collection2string(this.tagNames));
this.entry.put(BOOKMARK_TIMESTAMP, String.valueOf(this.timestamp)); this.entry.put(BOOKMARK_TIMESTAMP, String.valueOf(this.timestamp));
return this.entry; return this.entry;
@ -688,11 +646,11 @@ public class BookmarksDB {
/** /**
* Subclass of bookmarksDB, which provides the bookmarkIterator object-type * Subclass of bookmarksDB, which provides the bookmarkIterator object-type
*/ */
public class bookmarkIterator implements Iterator<Bookmark> { private class bookmarkIterator implements Iterator<Bookmark> {
Iterator<byte[]> bookmarkIter; Iterator<byte[]> bookmarkIter;
public bookmarkIterator(final boolean up) throws IOException { private bookmarkIterator(final boolean up) throws IOException {
//flushBookmarkCache(); //XXX: this will cost performance //flushBookmarkCache(); //XXX: this will cost performance
this.bookmarkIter = BookmarksDB.this.bookmarks.keys(up, false); this.bookmarkIter = BookmarksDB.this.bookmarks.keys(up, false);
//this.nextEntry = null; //this.nextEntry = null;
@ -722,14 +680,14 @@ public class BookmarksDB {
/** /**
* Comparator to sort objects of type Bookmark according to their timestamps * Comparator to sort objects of type Bookmark according to their timestamps
*/ */
public class bookmarkComparator implements Comparator<String> { private class bookmarkComparator implements Comparator<String> {
private final boolean newestFirst; private final boolean newestFirst;
/** /**
* @param newestFirst newest first, or oldest first? * @param newestFirst newest first, or oldest first?
*/ */
public bookmarkComparator(final boolean newestFirst){ private bookmarkComparator(final boolean newestFirst){
this.newestFirst = newestFirst; this.newestFirst = newestFirst;
} }
@ -752,13 +710,13 @@ public class BookmarksDB {
} }
} }
public static final TagComparator tagComparator = new TagComparator(); private static final TagComparator tagComparator = new TagComparator();
public static final TagSizeComparator tagSizeComparator = new TagSizeComparator(); private static final TagSizeComparator tagSizeComparator = new TagSizeComparator();
/** /**
* Comparator to sort objects of type Tag according to their names * Comparator to sort objects of type Tag according to their names
*/ */
public static class TagComparator implements Comparator<Tag>, Serializable { private static class TagComparator implements Comparator<Tag>, Serializable {
/** /**
* generated serial * generated serial
@ -772,7 +730,7 @@ public class BookmarksDB {
} }
public static class TagSizeComparator implements Comparator<Tag>, Serializable { private static class TagSizeComparator implements Comparator<Tag>, Serializable {
/** /**
* generated serial * generated serial

Write Preview
Loading…
Cancel
Save