diff --git a/source/de/anomic/data/bookmarksDB.java b/source/de/anomic/data/bookmarksDB.java index 308fb1201..fa9d25ddd 100644 --- a/source/de/anomic/data/bookmarksDB.java +++ b/source/de/anomic/data/bookmarksDB.java @@ -293,6 +293,7 @@ public class bookmarksDB { } return set.iterator(); } + public Iterator getBookmarksIterator(String tagName, boolean priv){ TreeSet set=new TreeSet(new bookmarkComparator(true)); String tagHash=tagHash(tagName); @@ -324,11 +325,13 @@ public class bookmarksDB { public int tagsSize(){ // TODO: shouldn't this be public int tagSize() return tagSize(false); } + public int tagSize(boolean flushed){ if(flushed) flushTagCache(); return tagsTable.size(); } + /** * load/retrieve an object of type Tag from the tagsTable (also save it in tagCache) * @param hash an object of type String, containing a tagHash @@ -343,6 +346,7 @@ public class bookmarksDB { } return ret; } + /** * retrieve an object of type Tag from the the tagCache, if object is not cached return loadTag(hash) * @param hash an object of type String, containing a tagHash @@ -370,25 +374,28 @@ public class bookmarksDB { * save a Tag in tagCache; see also flushTagCache(), addTag(), loadTag() * @param tag an object of type Tag to be saved in tagCache */ - public void saveTag(Tag tag){ + public void saveTag(Tag tag) { if(tag!=null){ tagCache.put(tag.getTagHash(), tag); } } - public void flushTagCache(){ + + public void flushTagCache() { Iterator it=tagCache.keySet().iterator(); while(it.hasNext()){ storeTag(tagCache.get(it.next())); } tagCache=new HashMap(); } - public String addTag(Tag tag){ // TODO: is addTag() really needed - check storeTag() and saveTag() + + public String addTag(Tag tag) { // TODO: is addTag() really needed - check storeTag() and saveTag() //tagsTable.set(tag.getTagName(), tag.getMap()); //tagCache.put(tag.getTagHash(), tag); saveTag(tag); return tag.getTagName(); } - public void removeTag(String hash){ + + public void removeTag(String hash) { try { if(tagCache.containsKey(hash)){ tagCache.remove(hash); @@ -396,22 +403,25 @@ public class bookmarksDB { tagsTable.remove(hash); } catch (IOException e) {} } - public Iterator tagIterator(boolean up){ + + public Iterator tagIterator(boolean up) { try { return new tagIterator(up); } catch (IOException e) { - return new HashSet().iterator(); + return new HashSet().iterator(); } } - public Iterator getTagIterator(boolean priv){ + + public Iterator getTagIterator(boolean priv) { return getTagIterator(priv,1); } - public Iterator getTagIterator(boolean priv, int c){ - Comparator comp; + + public Iterator getTagIterator(boolean priv, int c) { + Comparator comp; if (c == SORT_SIZE) comp = new tagSizeComparator(); else comp = new tagComparator(); TreeSet set=new TreeSet(comp); - Iterator it=tagIterator(true); + Iterator it = tagIterator(true); Tag tag; while(it.hasNext()){ tag=(Tag) it.next(); @@ -421,6 +431,7 @@ public class bookmarksDB { } return set.iterator(); } + public Iterator getTagIterator(boolean priv, int comp, int max){ if (max==SHOW_ALL) return getTagIterator(priv, comp); @@ -433,11 +444,13 @@ public class bookmarksDB { } return set.iterator(); } + public Iterator getTagIterator(String tagName, boolean priv){ return getTagIterator(tagName, priv, 1); } + public Iterator getTagIterator(String tagName, boolean priv, int comp){ - Comparator c; + Comparator c; if (comp == SORT_SIZE) c = new tagSizeComparator(); else c = new tagComparator(); TreeSet set=new TreeSet(c); @@ -459,6 +472,7 @@ public class bookmarksDB { } return set.iterator(); } + public Iterator getTagIterator(String tagName, boolean priv, int comp, int max){ if (max==SHOW_ALL) return getTagIterator(priv, comp); @@ -475,7 +489,7 @@ public class bookmarksDB { // rebuilds the tagsDB from the bookmarksDB public void rebuildTags(){ serverLog.logInfo("BOOKMARKS", "rebuilding tags.db from bookmarks.db..."); - Iterator it=bookmarkIterator(true); + Iterator it = bookmarkIterator(true); Bookmark bookmark; Tag tag; String[] tags; @@ -509,7 +523,7 @@ public class bookmarksDB { // rebuilds the datesDB from the bookmarksDB public void rebuildDates(){ serverLog.logInfo("BOOKMARKS", "rebuilding dates.db from bookmarks.db..."); - Iterator it=bookmarkIterator(true); + Iterator it=bookmarkIterator(true); Bookmark bookmark; String date; bookmarksDate bmDate; @@ -533,12 +547,12 @@ public class bookmarksDB { public boolean renameTag(String oldName, String newName){ String oldHash=tagHash(oldName); - String newHash=tagHash(newName); + //String newHash=tagHash(newName); Tag tag=getTag(oldHash); // set tag to oldHash if (tag != null) { - Set urlHashes = tag.getUrlHashes(); // preserve urlHashes of tag + Set urlHashes = tag.getUrlHashes(); // preserve urlHashes of tag removeTag(oldHash); - Iterator it = urlHashes.iterator(); + Iterator it = urlHashes.iterator(); Bookmark bookmark; Set tags = new HashSet(); String tagsString; @@ -557,10 +571,6 @@ public class bookmarksDB { } return false; } - - public void addBookmark(String url, String title, ArrayList tags){ - - } // -------------------------------------- // bookmarksDB's Import/Export functions @@ -568,7 +578,7 @@ public class bookmarksDB { public int importFromBookmarks(yacyURL baseURL, String input, String tag, boolean importPublic){ try { - // convert string to inputstream + // convert string to input stream ByteArrayInputStream byteIn = new ByteArrayInputStream(input.getBytes("UTF-8")); InputStreamReader reader = new InputStreamReader(byteIn,"UTF-8"); @@ -578,12 +588,12 @@ public class bookmarksDB { return 0; } } + public int importFromBookmarks(yacyURL baseURL, InputStreamReader input, String tag, boolean importPublic){ int importCount = 0; - HashMap links=new HashMap(); - Iterator it; + Map links = new HashMap(); String title; yacyURL url; Bookmark bm; @@ -595,12 +605,11 @@ public class bookmarksDB { Writer writer= new htmlFilterWriter(null,null,scraper, null, false); serverFileUtils.copy(input,writer); writer.close(); - links = (HashMap) scraper.getAnchors(); + links = scraper.getAnchors(); } catch (IOException e) {} - it=links.keySet().iterator(); - while(it.hasNext()){ - // url=(String) it.next(); - url= (yacyURL) it.next(); + Iterator it = links.keySet().iterator(); + while (it.hasNext()) { + url= it.next(); title=(String) links.get(url); serverLog.logInfo("BOOKMARKS", "links.get(url)"); if(title.equals("")){//cannot be displayed @@ -619,9 +628,10 @@ public class bookmarksDB { return importCount; } + public int importFromXML(String input, boolean importPublic){ try { - // convert string to inputstream + // convert string to input stream ByteArrayInputStream byteIn = new ByteArrayInputStream(input.getBytes("UTF-8")); // import stream @@ -630,6 +640,7 @@ public class bookmarksDB { return 0; } } + public int importFromXML(InputStream input, boolean importPublic){ DocumentBuilderFactory factory=DocumentBuilderFactory.newInstance(); factory.setValidating(false); @@ -646,6 +657,7 @@ public class bookmarksDB { return 0; } + public int parseXMLimport(Node doc, boolean importPublic){ int importCount = 0; if(doc.getNodeName()=="post"){ @@ -870,7 +882,7 @@ public class bookmarksDB { /** * Subclass of bookmarksDB, which provides the Bookmark object-type */ - public class Bookmark extends kelondroObjectsMapEntry{ + public class Bookmark extends kelondroObjectsMapEntry { public static final String BOOKMARK_URL="bookmarkUrl"; public static final String BOOKMARK_TITLE="bookmarkTitle"; public static final String BOOKMARK_DESCRIPTION="bookmarkDesc"; @@ -882,7 +894,8 @@ public class bookmarksDB { private String urlHash; private Set tags; private long timestamp; - public Bookmark(String urlHash, HashMap map){ + + public Bookmark(String urlHash, HashMap map) { super(map); this.urlHash=urlHash; if(map.containsKey(BOOKMARK_TAGS)) @@ -891,6 +904,7 @@ public class bookmarksDB { tags=new HashSet(); loadTimestamp(); } + public Bookmark(String url){ super(); if(!url.toLowerCase().startsWith("http://") && !url.toLowerCase().startsWith("https://")){ @@ -916,14 +930,16 @@ public class bookmarksDB { removeBookmark(this.urlHash); //prevent empty tags } - public Bookmark(String urlHash, yacyURL url){ + + public Bookmark(String urlHash, yacyURL url) { super(); this.urlHash=urlHash; entry.put(BOOKMARK_URL, url.toNormalform(false, true)); tags=new HashSet(); timestamp=System.currentTimeMillis(); } - public Bookmark(String urlHash, String url){ + + public Bookmark(String urlHash, String url) { super(); this.urlHash=urlHash; entry.put(BOOKMARK_URL, url); @@ -935,25 +951,30 @@ public class bookmarksDB { this((new yacyURL((String)map.map().get(BOOKMARK_URL), null)).hash(), map.map()); } - private Map toMap(){ + private Map toMap() { entry.put(BOOKMARK_TAGS, listManager.collection2string(tags)); entry.put(BOOKMARK_TIMESTAMP, String.valueOf(this.timestamp)); return entry; } - private void loadTimestamp(){ + + private void loadTimestamp() { if(entry.containsKey(BOOKMARK_TIMESTAMP)) this.timestamp=Long.parseLong(entry.get(BOOKMARK_TIMESTAMP)); } - public String getUrlHash(){ + + public String getUrlHash() { return urlHash; } - public String getUrl(){ + + public String getUrl() { return entry.get(BOOKMARK_URL); } - public Set getTags(){ + + public Set getTags() { return tags; } - public String getTagsString(){ + + public String getTagsString() { String s[] = listManager.collection2string(getTags()).split(","); String tagsString=""; for (int i=0; i tags2){ setTags(tags2, true); } + /** * set the Tags of the bookmark * @param tags ArrayList with the tagnames @@ -1060,6 +1094,7 @@ public class bookmarksDB { public long getTimeStamp(){ return timestamp; } + public void setTimeStamp(long ts){ this.timestamp=ts; } @@ -1070,11 +1105,13 @@ public class bookmarksDB { public class tagIterator implements Iterator { kelondroCloneableIterator tagIter; bookmarksDB.Tag nextEntry; + public tagIterator(boolean up) throws IOException { flushTagCache(); //XXX: This costs performace :-(( this.tagIter = bookmarksDB.this.tagsTable.keys(up, false); this.nextEntry = null; } + public boolean hasNext() { try { return this.tagIter.hasNext(); @@ -1083,6 +1120,7 @@ public class bookmarksDB { return false; } } + public Tag next() { try { return getTag(this.tagIter.next()); @@ -1091,6 +1129,7 @@ public class bookmarksDB { return null; } } + public void remove() { if (this.nextEntry != null) { try { @@ -1102,17 +1141,19 @@ public class bookmarksDB { } } } + /** * Subclass of bookmarksDB, which provides the bookmarkIterator object-type */ public class bookmarkIterator implements Iterator { - Iterator bookmarkIter; + Iterator bookmarkIter; bookmarksDB.Bookmark nextEntry; public bookmarkIterator(boolean up) throws IOException { //flushBookmarkCache(); //XXX: this will cost performance this.bookmarkIter = bookmarksDB.this.bookmarksTable.keys(up, false); this.nextEntry = null; } + public boolean hasNext() { try { return this.bookmarkIter.hasNext(); @@ -1121,6 +1162,7 @@ public class bookmarksDB { return false; } } + public Bookmark next() { try { return getBookmark((String) this.bookmarkIter.next()); @@ -1129,6 +1171,7 @@ public class bookmarksDB { return null; } } + public void remove() { if (this.nextEntry != null) { try { @@ -1140,21 +1183,24 @@ public class bookmarksDB { } } } + /** * Comparator to sort objects of type Bookmark according to their timestamps */ - public class bookmarkComparator implements Comparator { + public class bookmarkComparator implements Comparator { private boolean newestFirst; + /** * @param newestFirst newest first, or oldest first? */ public bookmarkComparator(boolean newestFirst){ this.newestFirst=newestFirst; } - public int compare(Object obj1, Object obj2){ - Bookmark bm1=getBookmark((String)obj1); - Bookmark bm2=getBookmark((String)obj2); + + public int compare(String obj1, String obj2) { + Bookmark bm1=getBookmark(obj1); + Bookmark bm2=getBookmark(obj2); if(bm1==null || bm2==null) return 0; //XXX: i think this should not happen? maybe this needs further tracing of the bug if(this.newestFirst){ @@ -1165,21 +1211,25 @@ public class bookmarksDB { return -1; } } + /** * Comparator to sort objects of type Tag according to their names */ - public class tagComparator implements Comparator{ - public int compare(Object obj1, Object obj2){ - return ((Tag)obj1).getTagName().compareTo(((Tag)obj2).getTagName()); + public class tagComparator implements Comparator { + + public int compare(Tag obj1, Tag obj2){ + return obj1.getTagName().compareTo(obj2.getTagName()); } + } - public class tagSizeComparator implements Comparator{ - public int compare(Object obj1, Object obj2){ - Tag t1 = (Tag)obj1; - Tag t2 = (Tag)obj2; - if (t1.size() { + + public int compare(Tag obj1, Tag obj2) { + if (obj1.size() < obj2.size()) return 1; + else if (obj1.getTagName().equals(obj2.getTagName())) return 0; else return -1; } + } } \ No newline at end of file diff --git a/source/de/anomic/kelondro/kelondroMergeIterator.java b/source/de/anomic/kelondro/kelondroMergeIterator.java index 138290e59..ae0ed6828 100644 --- a/source/de/anomic/kelondro/kelondroMergeIterator.java +++ b/source/de/anomic/kelondro/kelondroMergeIterator.java @@ -147,19 +147,18 @@ public class kelondroMergeIterator implements kelondroCloneableIterator { throw new java.lang.UnsupportedOperationException("merge does not support remove"); } - public static kelondroCloneableIterator cascade(Set> /*of*/ iterators, kelondroOrder c, Method merger, boolean up) { + public static kelondroCloneableIterator cascade(Set> iterators, kelondroOrder c, Method merger, boolean up) { // this extends the ability to combine two iterators - // to the abiliy of combining a set of iterators + // to the ability of combining a set of iterators if (iterators == null) return null; if (iterators.size() == 0) return null; - return cascade((Set>) iterators.iterator(), c, merger, up); + return cascade(iterators.iterator(), c, merger, up); } - @SuppressWarnings("unchecked") - private static kelondroCloneableIterator cascade(Iterator /*of*/ iiterators, kelondroOrder c, Method merger, boolean up) { + private static kelondroCloneableIterator cascade(Iterator> iiterators, kelondroOrder c, Method merger, boolean up) { if (iiterators == null) return null; if (!(iiterators.hasNext())) return null; - kelondroCloneableIterator one = (kelondroCloneableIterator) iiterators.next(); + kelondroCloneableIterator one = iiterators.next(); if (!(iiterators.hasNext())) return one; return new kelondroMergeIterator(one, cascade(iiterators, c, merger, up), c, merger, up); } diff --git a/source/de/anomic/plasma/plasmaRankingCRProcess.java b/source/de/anomic/plasma/plasmaRankingCRProcess.java index 61572cd9e..23c78e740 100644 --- a/source/de/anomic/plasma/plasmaRankingCRProcess.java +++ b/source/de/anomic/plasma/plasmaRankingCRProcess.java @@ -48,7 +48,6 @@ package de.anomic.plasma; import java.io.File; import java.io.IOException; import java.util.Iterator; -import java.util.Map; import de.anomic.kelondro.kelondroAttrSeq; import de.anomic.kelondro.kelondroBase64Order; @@ -355,12 +354,10 @@ public class plasmaRankingCRProcess { cr_UDate = cr_entry.getAttr("UDate", 0); // loop over all anchors - Iterator j = cr_entry.getSeqSet().iterator(); - Map.Entry entry; + Iterator j = cr_entry.getSeqSet().iterator(); while (j.hasNext()) { // get domain of anchors - entry = (Map.Entry) j.next(); - anchor = (String) entry.getKey(); + anchor = j.next(); if (anchor.length() == 6) anchorDom = anchor; else anchorDom = anchor.substring(6); // update domain-specific entry @@ -401,22 +398,22 @@ public class plasmaRankingCRProcess { int size = seq.size(); long start = System.currentTimeMillis(); long l; - final Iterator i = seq.keycollections(null, null, false); + final Iterator i = seq.keycollections(null, null, false); Object[] keycollection; String referee, refereeDom, anchor, anchorDom; kelondroRowSet cr_entry, rci_entry; while (i.hasNext()) { - keycollection = (Object[]) i.next(); + keycollection = i.next(); referee = new String((byte[]) keycollection[0]); if (referee.length() == 6) refereeDom = referee; else refereeDom = referee.substring(6); cr_entry = (kelondroRowSet) keycollection[1]; // loop over all anchors - Iterator j = cr_entry.rows(); + Iterator j = cr_entry.rows(); kelondroRow.Entry entry; while (j.hasNext()) { // get domain of anchors - entry = (kelondroRow.Entry) j.next(); + entry = j.next(); anchor = (String) entry.getColString(0, null); if (anchor.length() == 6) anchorDom = anchor; else anchorDom = anchor.substring(6); diff --git a/source/de/anomic/plasma/plasmaRankingRCIEvaluation.java b/source/de/anomic/plasma/plasmaRankingRCIEvaluation.java index 9b526fc9f..7cc2a1792 100644 --- a/source/de/anomic/plasma/plasmaRankingRCIEvaluation.java +++ b/source/de/anomic/plasma/plasmaRankingRCIEvaluation.java @@ -64,19 +64,19 @@ public class plasmaRankingRCIEvaluation { public static int[] rcieval(kelondroAttrSeq rci) { // collect information about which entry has how many references // the output is a reference-count:occurrences relation - HashMap counts = new HashMap(); - Iterator i = rci.keys(); + HashMap counts = new HashMap(); + Iterator i = rci.keys(); String key; kelondroAttrSeq.Entry entry; Integer count_key, count_count; int c, maxcount = 0; while (i.hasNext()) { - key = (String) i.next(); + key = i.next(); entry = rci.getEntry(key); c = entry.getSeqSet().size(); if (c > maxcount) maxcount = c; count_key = new Integer(c); - count_count = (Integer) counts.get(count_key); + count_count = counts.get(count_key); if (count_count == null) { count_count = new Integer(1); } else { @@ -86,7 +86,7 @@ public class plasmaRankingRCIEvaluation { } int[] ctable = new int[maxcount + 1]; for (int j = 0; j <= maxcount; j++) { - count_count = (Integer) counts.get(new Integer(j)); + count_count = counts.get(new Integer(j)); if (count_count == null) { ctable[j] = 0; } else { @@ -153,10 +153,11 @@ public class plasmaRankingRCIEvaluation { return partition.length - 1; } - public static TreeSet[] genRankingTable(kelondroAttrSeq rci, int[] partition) { - TreeSet[] ranked = new TreeSet[partition.length]; - for (int i = 0; i < partition.length; i++) ranked[i] = new TreeSet(kelondroBase64Order.enhancedComparator); - Iterator i = rci.keys(); + @SuppressWarnings("unchecked") + public static TreeSet[] genRankingTable(kelondroAttrSeq rci, int[] partition) { + TreeSet[] ranked = new TreeSet[partition.length]; + for (int i = 0; i < partition.length; i++) ranked[i] = new TreeSet(kelondroBase64Order.enhancedComparator); + Iterator i = rci.keys(); String key; kelondroAttrSeq.Entry entry; while (i.hasNext()) { @@ -167,13 +168,13 @@ public class plasmaRankingRCIEvaluation { return ranked; } - public static HashMap genReverseDomHash(File domlist) { - HashSet domset = serverFileUtils.loadList(domlist); - HashMap dommap = new HashMap(); - Iterator i = domset.iterator(); + public static HashMap genReverseDomHash(File domlist) { + HashSet domset = serverFileUtils.loadList(domlist); + HashMap dommap = new HashMap(); + Iterator i = domset.iterator(); String dom; while (i.hasNext()) { - dom = (String) i.next(); + dom = i.next(); if (dom.startsWith("www.")) dom = dom.substring(4); try { dommap.put((new yacyURL("http://" + dom, null)).hash().substring(6), dom); @@ -183,7 +184,7 @@ public class plasmaRankingRCIEvaluation { return dommap; } - public static void storeRankingTable(TreeSet[] ranking, File tablePath) throws IOException { + public static void storeRankingTable(TreeSet[] ranking, File tablePath) throws IOException { String filename; if (!(tablePath.exists())) tablePath.mkdirs(); for (int i = 0; i < ranking.length - 1; i++) { @@ -218,7 +219,7 @@ public class plasmaRankingRCIEvaluation { System.out.println("sum of all references: " + sum); // create ranking - TreeSet[] ranked = genRankingTable(rci, partition); + TreeSet[] ranked = genRankingTable(rci, partition); storeRankingTable(ranked, new File(root_path, "ranking/YBR")); long seconds = java.lang.Math.max(1, (System.currentTimeMillis() - start) / 1000); System.out.println("Finished YBR generation in " + seconds + " seconds."); @@ -230,7 +231,7 @@ public class plasmaRankingRCIEvaluation { plasmaSearchRankingProcess.loadYBR(new File(root_path, "ranking/YBR"), 16); // load domain list and generate hash index for domains - HashMap dommap = genReverseDomHash(new File(root_path, "domlist.txt")); + HashMap dommap = genReverseDomHash(new File(root_path, "domlist.txt")); // print out the table String hash, dom; @@ -238,7 +239,7 @@ public class plasmaRankingRCIEvaluation { System.out.print("YBR-" + i + ": "); for (int j = 0; j < plasmaSearchRankingProcess.ybrTables[i].size(); j++) { hash = new String(plasmaSearchRankingProcess.ybrTables[i].get(j)); - dom = (String) dommap.get(hash); + dom = dommap.get(hash); if (dom == null) System.out.print("[" + hash + "], "); else System.out.print(dom + ", "); } System.out.println();