diff --git a/source/net/yacy/cora/ranking/ClusteredScoreMap.java b/source/net/yacy/cora/ranking/ClusteredScoreMap.java index 3aa518c0a..5aac0f1a9 100644 --- a/source/net/yacy/cora/ranking/ClusteredScoreMap.java +++ b/source/net/yacy/cora/ranking/ClusteredScoreMap.java @@ -58,10 +58,12 @@ public final class ClusteredScoreMap extends AbstractScoreMap implements R this.encnt = 0; } + @Override public Iterator iterator() { return this.map.keySet().iterator(); } + @Override public synchronized void clear() { this.map.clear(); this.pam.clear(); @@ -73,6 +75,7 @@ public final class ClusteredScoreMap extends AbstractScoreMap implements R * shrink the cluster to a demanded size * @param maxsize */ + @Override public void shrinkToMaxSize(final int maxsize) { if (maxsize < 0) return; Long key; @@ -90,6 +93,7 @@ public final class ClusteredScoreMap extends AbstractScoreMap implements R * shrink the cluster in such a way that the smallest score is equal or greater than a given minScore * @param minScore */ + @Override public void shrinkToMinScore(final int minScore) { int score; Long key; @@ -121,7 +125,7 @@ public final class ClusteredScoreMap extends AbstractScoreMap implements R if (o instanceof Long) { final long l = ((Long) o).longValue(); if (l < Integer.MAX_VALUE) return (int) l; - o = ((Long) o).toString(); + return (int) (l & Integer.MAX_VALUE); } if (o instanceof Float) { final double d = 1000f * ((Float) o).floatValue(); @@ -148,13 +152,13 @@ public final class ClusteredScoreMap extends AbstractScoreMap implements R l = Long.parseLong(s); } // fix out-of-ranges - if (l > Integer.MAX_VALUE) return Integer.MAX_VALUE; //(int) (l & (Integer.MAX_VALUE)); + if (l > Integer.MAX_VALUE) return (int) (l & Integer.MAX_VALUE); if (l < 0) { System.out.println("string2score: negative score for input " + s); return 0; } return (int) l; - } catch (final Exception e) { + } catch (final Throwable e) { // try it lex int len = s.length(); if (len > 5) len = 5; @@ -188,26 +192,32 @@ public final class ClusteredScoreMap extends AbstractScoreMap implements R return this.gcount; } + @Override public synchronized int size() { return this.map.size(); } + @Override public boolean sizeSmaller(final int size) { return this.map.size() < size; } + @Override public synchronized boolean isEmpty() { return this.map.isEmpty(); } + @Override public synchronized void inc(final E obj) { inc(obj, 1); } + @Override public synchronized void dec(final E obj) { inc(obj, -1); } + @Override public void set(final E obj, final int newScore) { if (obj == null) return; synchronized (this) { @@ -242,6 +252,7 @@ public final class ClusteredScoreMap extends AbstractScoreMap implements R this.gcount += newScore; } + @Override public void inc(final E obj, final int incrementScore) { if (obj == null) return; synchronized (this) { @@ -277,10 +288,12 @@ public final class ClusteredScoreMap extends AbstractScoreMap implements R this.gcount += incrementScore; } + @Override public void dec(final E obj, final int incrementScore) { inc(obj, -incrementScore); } + @Override public int delete(final E obj) { // deletes entry and returns previous score if (obj == null) return 0; @@ -302,10 +315,12 @@ public final class ClusteredScoreMap extends AbstractScoreMap implements R return oldScore; } + @Override public synchronized boolean containsKey(final E obj) { return this.map.containsKey(obj); } + @Override public int get(final E obj) { if (obj == null) return 0; final Long cs; @@ -316,30 +331,36 @@ public final class ClusteredScoreMap extends AbstractScoreMap implements R return (int) ((cs.longValue() & 0xFFFFFFFF00000000L) >> 32); } + @Override public synchronized int getMaxScore() { if (this.map.isEmpty()) return -1; return (int) ((this.pam.lastKey().longValue() & 0xFFFFFFFF00000000L) >> 32); } + @Override public synchronized int getMinScore() { if (this.map.isEmpty()) return -1; return (int) ((this.pam.firstKey().longValue() & 0xFFFFFFFF00000000L) >> 32); } + @Override public synchronized E getMaxKey() { if (this.map.isEmpty()) return null; return this.pam.get(this.pam.lastKey()); } + @Override public synchronized E getMinKey() { if (this.map.isEmpty()) return null; return this.pam.get(this.pam.firstKey()); } + @Override public String toString() { return this.map + " / " + this.pam; } + @Override public synchronized Iterator keys(final boolean up) { if (up) return new simpleScoreIterator(); return new reverseScoreIterator(); @@ -354,10 +375,12 @@ public final class ClusteredScoreMap extends AbstractScoreMap implements R this.view = ClusteredScoreMap.this.pam; } + @Override public boolean hasNext() { return !this.view.isEmpty(); } + @Override public E next() { this.key = this.view.lastKey(); this.view = this.view.headMap(this.key); @@ -366,6 +389,7 @@ public final class ClusteredScoreMap extends AbstractScoreMap implements R return value; } + @Override public void remove() { final Object val = ClusteredScoreMap.this.pam.remove(this.key); if (val != null) ClusteredScoreMap.this.map.remove(val); @@ -382,16 +406,19 @@ public final class ClusteredScoreMap extends AbstractScoreMap implements R this.ii = ClusteredScoreMap.this.pam.entrySet().iterator(); } + @Override public boolean hasNext() { return this.ii.hasNext(); } + @Override public E next() { this.entry = this.ii.next(); //System.out.println("cluster simple iterator: score = " + ((((Long) entry.getKey()).longValue() & 0xFFFFFFFF00000000L) >> 32) + ", handle = " + (((Long) entry.getKey()).longValue() & 0xFFFFFFFFL) + ", value = " + entry.getValue()); return this.entry.getValue(); } + @Override public void remove() { this.ii.remove(); if (this.entry.getValue() != null) ClusteredScoreMap.this.map.remove(this.entry.getValue()); diff --git a/source/net/yacy/document/parser/html/ContentScraper.java b/source/net/yacy/document/parser/html/ContentScraper.java index caf7bdd18..595542d81 100644 --- a/source/net/yacy/document/parser/html/ContentScraper.java +++ b/source/net/yacy/document/parser/html/ContentScraper.java @@ -115,17 +115,17 @@ public class ContentScraper extends AbstractScraper implements Scraper { } // class variables: collectors for links - private Map anchors; + private final Map anchors; private final Map rss, css; private final Set script, frames, iframes; - private Map images; // urlhash/image relation + private final Map images; // urlhash/image relation private final Map metas; private String title; //private String headline; private List[] headlines; private final ClusteredScoreMap bold, italic; private final List li; - private CharBuffer content; + private final CharBuffer content; private final EventListenerList htmlFilterEventListeners; private float lon, lat; private MultiProtocolURI canonical; @@ -187,23 +187,7 @@ public class ContentScraper extends AbstractScraper implements Scraper { this.anchors.put(url, p0); } - /* - private void mergeAnchors(final MultiProtocolURI url, final String key, final String value) { - if (value == null) return; - if (value.length() == 0) return; - Properties p0 = this.anchors.get(url); - if (p0 == null) { - p0 = new Properties(); - p0.put(key, value); - this.anchors.put(url, p0); - return; - } - // merge properties - p0.put(key, value); - this.anchors.put(url, p0); - } - */ - + @Override public void scrapeText(final char[] newtext, final String insideTag) { // System.out.println("SCRAPE: " + UTF8.String(newtext)); int p, pl, q, s = 0; @@ -295,7 +279,10 @@ public class ContentScraper extends AbstractScraper implements Scraper { } catch (final MalformedURLException e) {} } // append string to content - if (b.length() != 0) this.content.append(b).append(32); + if (!b.isEmpty()) { + this.content.append(b); + this.content.appendSpace(); + } } private final static Pattern dpssp = Pattern.compile("://"); @@ -317,6 +304,7 @@ public class ContentScraper extends AbstractScraper implements Scraper { } } + @Override public void scrapeTag0(final String tagname, final Properties tagopts) { if (tagname.equalsIgnoreCase("img")) { final String src = tagopts.getProperty("src", EMPTY_STRING); @@ -407,6 +395,7 @@ public class ContentScraper extends AbstractScraper implements Scraper { fireScrapeTag0(tagname, tagopts); } + @Override public void scrapeTag1(final String tagname, final Properties tagopts, final char[] text) { // System.out.println("ScrapeTag1: tagname=" + tagname + ", opts=" + tagopts.toString() + ", text=" + UTF8.String(text)); if (tagname.equalsIgnoreCase("a") && text.length < 2048) { @@ -481,6 +470,7 @@ public class ContentScraper extends AbstractScraper implements Scraper { } + @Override public void scrapeComment(final char[] comment) { this.evaluationScores.match(Element.comment, comment); } @@ -837,11 +827,11 @@ public class ContentScraper extends AbstractScraper implements Scraper { public void close() { // free resources super.close(); - this.anchors = null; - this.images = null; + this.anchors.clear(); + this.images.clear(); this.title = null; this.headlines = null; - this.content = null; + this.content.clear(); this.root = null; } diff --git a/source/net/yacy/document/parser/html/ContentTransformer.java b/source/net/yacy/document/parser/html/ContentTransformer.java index 19a465d75..1858613e3 100644 --- a/source/net/yacy/document/parser/html/ContentTransformer.java +++ b/source/net/yacy/document/parser/html/ContentTransformer.java @@ -55,17 +55,18 @@ public class ContentTransformer extends AbstractTransformer implements Transform super(linkTags0, linkTags1); } + @Override public void init(final String initarg) { - if (bluelist == null) { + if (this.bluelist == null) { // here, the init arg is used to load a list of blue-listed words - bluelist = new ArrayList(); + this.bluelist = new ArrayList(); final File f = new File(initarg); if (f.canRead()) { try { final BufferedReader r = new BufferedReader(new FileReader(f)); String s; while ((s = r.readLine()) != null) { - if (s.length() > 0 && s.charAt(0) != '#') bluelist.add(s.toLowerCase()); + if (s.length() > 0 && s.charAt(0) != '#') this.bluelist.add(s.toLowerCase()); } r.close(); } catch (final IOException e) { @@ -75,8 +76,9 @@ public class ContentTransformer extends AbstractTransformer implements Transform } } + @Override public boolean isIdentityTransformer() { - return bluelist.isEmpty(); + return this.bluelist.isEmpty(); } private static char[] genBlueLetters(int length) { @@ -84,7 +86,7 @@ public class ContentTransformer extends AbstractTransformer implements Transform length = length / 2; if (length > 10) length = 7; while (length-- > 0) { - bb.append((int)'X'); + bb.append('X'); } bb.append(" "); final char[] result = bb.getChars(); @@ -97,16 +99,17 @@ public class ContentTransformer extends AbstractTransformer implements Transform } private boolean bluelistHit(final char[] text) { - if (text == null || bluelist == null) return false; + if (text == null || this.bluelist == null) return false; final String lc = new String(text).toLowerCase(); - for (int i = 0; i < bluelist.size(); i++) { - if (lc.indexOf(bluelist.get(i)) >= 0) return true; + for (int i = 0; i < this.bluelist.size(); i++) { + if (lc.indexOf(this.bluelist.get(i)) >= 0) return true; } return false; } + @Override public char[] transformText(final char[] text) { - if (bluelist != null) { + if (this.bluelist != null) { if (bluelistHit(text)) { // System.out.println("FILTERHIT: " + text); return genBlueLetters(text.length); diff --git a/source/net/yacy/document/parser/html/TransformerWriter.java b/source/net/yacy/document/parser/html/TransformerWriter.java index ccebeed9a..6670aa27d 100644 --- a/source/net/yacy/document/parser/html/TransformerWriter.java +++ b/source/net/yacy/document/parser/html/TransformerWriter.java @@ -82,7 +82,7 @@ public final class TransformerWriter extends Writer { final Transformer transformer, final boolean passbyIfBinarySuspect ) { - this(outStream, charSet, scraper, transformer, passbyIfBinarySuspect, 1024); + this(outStream, charSet, scraper, transformer, passbyIfBinarySuspect, 4096); } public TransformerWriter( @@ -115,9 +115,9 @@ public final class TransformerWriter extends Writer { public static char[] genTag0raw(final String tagname, final boolean opening, final char[] tagopts) { final CharBuffer bb = new CharBuffer(tagname.length() + tagopts.length + 3); - bb.append((int)'<'); + bb.append('<'); if (!opening) { - bb.append((int)'/'); + bb.append('/'); } bb.append(tagname); if (tagopts.length > 0) { @@ -125,7 +125,7 @@ public final class TransformerWriter extends Writer { bb.append(tagopts); // else bb.append((byte) 32).append(tagopts); } - bb.append((int)'>'); + bb.append('>'); final char[] result = bb.getChars(); try { bb.close(); @@ -137,15 +137,15 @@ public final class TransformerWriter extends Writer { public static char[] genTag1raw(final String tagname, final char[] tagopts, final char[] text) { final CharBuffer bb = new CharBuffer(2 * tagname.length() + tagopts.length + text.length + 5); - bb.append((int)'<').append(tagname); + bb.append('<').append(tagname); if (tagopts.length > 0) { // if (tagopts[0] == (byte) 32) bb.append(tagopts); // else bb.append((byte) 32).append(tagopts); } - bb.append((int)'>'); + bb.append('>'); bb.append(text); - bb.append((int)'<').append((int)'/').append(tagname).append((int)'>'); + bb.append('<').append('/').append(tagname).append('>'); final char[] result = bb.getChars(); try { bb.close(); @@ -158,12 +158,12 @@ public final class TransformerWriter extends Writer { public static char[] genTag0(final String tagname, final Properties tagopts, final char quotechar) { final char[] tagoptsx = (tagopts.isEmpty()) ? null : genOpts(tagopts, quotechar); final CharBuffer bb = new CharBuffer(tagname.length() + ((tagoptsx == null) ? 0 : (tagoptsx.length + 1)) + tagname.length() + 2); - bb.append((int)'<').append(tagname); + bb.append('<').append(tagname); if (tagoptsx != null) { - bb.append(32); + bb.appendSpace(); bb.append(tagoptsx); } - bb.append((int)'>'); + bb.append('>'); final char[] result = bb.getChars(); try { bb.close(); @@ -176,7 +176,7 @@ public final class TransformerWriter extends Writer { public static char[] genTag1(final String tagname, final Properties tagopts, final char[] text, final char quotechar) { final char[] gt0 = genTag0(tagname, tagopts, quotechar); final CharBuffer cb = new CharBuffer(gt0, gt0.length + text.length + tagname.length() + 3); - cb.append(text).append((int)'<').append((int)'/').append(tagname).append((int)'>'); + cb.append(text).append('<').append('/').append(tagname).append('>'); final char[] result = cb.getChars(); try { cb.close(); @@ -193,9 +193,9 @@ public final class TransformerWriter extends Writer { String key; while (e.hasMoreElements()) { key = (String) e.nextElement(); - bb.append(32).append(key).append((int)'=').append((int)quotechar); + bb.appendSpace().append(key).append('=').append(quotechar); bb.append(prop.getProperty(key)); - bb.append((int)quotechar); + bb.append(quotechar); } final char[] result; if (bb.length() > 0) @@ -530,12 +530,14 @@ public final class TransformerWriter extends Writer { write(b, 0, b.length); } + @Override public void write(final char b[], final int off, final int len) throws IOException { // System.out.println(UTF8.String(b, off, len)); if ((off | len | (b.length - (len + off)) | (off + len)) < 0) throw new IndexOutOfBoundsException(); for (int i = off ; i < (len - off) ; i++) this.write(b[i]); } + @Override public void flush() throws IOException { // we cannot flush the current string this.buffer to prevent that // the filter process is messed up @@ -544,6 +546,7 @@ public final class TransformerWriter extends Writer { // if you want to flush all, call close() at end of writing; } + @Override public void close() throws IOException { final char quotechar = (this.inSingleQuote) ? singlequote : doublequote; if (this.buffer != null) { diff --git a/source/net/yacy/document/parser/htmlParser.java b/source/net/yacy/document/parser/htmlParser.java index 0f9b839f0..117cd884e 100644 --- a/source/net/yacy/document/parser/htmlParser.java +++ b/source/net/yacy/document/parser/htmlParser.java @@ -83,6 +83,7 @@ public class htmlParser extends AbstractParser implements Parser { this.SUPPORTED_MIME_TYPES.add("text/csv"); } + @Override public Document[] parse( final MultiProtocolURI location, final String mimeType, @@ -196,7 +197,7 @@ public class htmlParser extends AbstractParser implements Parser { // parsing the content final ContentScraper scraper = new ContentScraper(location); - final TransformerWriter writer = new TransformerWriter(null,null,scraper,null,false, sourceStream.available()); + final TransformerWriter writer = new TransformerWriter(null,null,scraper,null,false, Math.max(4096, sourceStream.available())); try { FileUtils.copy(sourceStream, writer, c); } catch (final IOException e) { diff --git a/source/net/yacy/kelondro/blob/MapDataMining.java b/source/net/yacy/kelondro/blob/MapDataMining.java index 9a8bd56df..75b92dbd1 100644 --- a/source/net/yacy/kelondro/blob/MapDataMining.java +++ b/source/net/yacy/kelondro/blob/MapDataMining.java @@ -44,7 +44,6 @@ import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.order.Base64Order; import net.yacy.kelondro.order.ByteOrder; import net.yacy.kelondro.order.CloneableIterator; -import net.yacy.kelondro.util.LookAheadIterator; public class MapDataMining extends MapHeap { @@ -105,7 +104,7 @@ public class MapDataMining extends MapHeap { } // fill cluster and accumulator with values - if ((sortfields != null) || (longaccfields != null) || (floataccfields != null)) try { + if (sortfields != null || longaccfields != null || floataccfields != null) try { final CloneableIterator it = super.keys(true, false); byte[] mapnameb; String cell; @@ -122,37 +121,49 @@ public class MapDataMining extends MapHeap { } if (map == null) break; - if (sortfields != null && cluster != null) for (int i = 0; i < sortfields.length; i++) { - cell = map.get(sortfields[i]); - if (cell != null) cluster[i].set(UTF8.String(mapnameb), ClusteredScoreMap.object2score(cell)); + if (sortfields != null && cluster != null) { + for (int i = 0; i < sortfields.length; i++) { + cell = map.get(sortfields[i]); + if (cell != null) cluster[i].set(UTF8.String(mapnameb), ClusteredScoreMap.object2score(cell)); + } } - if (longaccfields != null && longaccumulator != null) for (int i = 0; i < longaccfields.length; i++) { - cell = map.get(longaccfields[i]); - valuel = 0; - if (cell != null) try { - valuel = Long.parseLong(cell); - longaccumulator[i] = Long.valueOf(longaccumulator[i].longValue() + valuel); - } catch (final NumberFormatException e) {} + if (longaccfields != null && longaccumulator != null) { + for (int i = 0; i < longaccfields.length; i++) { + cell = map.get(longaccfields[i]); + valuel = 0; + if (cell != null) try { + valuel = Long.parseLong(cell); + longaccumulator[i] = Long.valueOf(longaccumulator[i].longValue() + valuel); + } catch (final NumberFormatException e) {} + } } - if (floataccfields != null && floataccumulator != null) for (int i = 0; i < floataccfields.length; i++) { - cell = map.get(floataccfields[i]); - valued = 0f; - if (cell != null) try { - valued = Float.parseFloat(cell); - floataccumulator[i] = new Float(floataccumulator[i].floatValue() + valued); - } catch (final NumberFormatException e) {} + if (floataccfields != null && floataccumulator != null) { + for (int i = 0; i < floataccfields.length; i++) { + cell = map.get(floataccfields[i]); + valued = 0f; + if (cell != null) try { + valued = Float.parseFloat(cell); + floataccumulator[i] = new Float(floataccumulator[i].floatValue() + valued); + } catch (final NumberFormatException e) {} + } } } } catch (final IOException e) {} // fill cluster - if (sortfields != null && cluster != null) for (int i = 0; i < sortfields.length; i++) this.sortClusterMap.put(sortfields[i], cluster[i]); + if (sortfields != null && cluster != null) { + for (int i = 0; i < sortfields.length; i++) this.sortClusterMap.put(sortfields[i], cluster[i]); + } // fill acc map - if (longaccfields != null && longaccumulator != null) for (int i = 0; i < longaccfields.length; i++) this.accLong.put(longaccfields[i], longaccumulator[i]); - if (floataccfields != null && floataccumulator != null) for (int i = 0; i < floataccfields.length; i++) this.accFloat.put(floataccfields[i], floataccumulator[i]); + if (longaccfields != null && longaccumulator != null) { + for (int i = 0; i < longaccfields.length; i++) this.accLong.put(longaccfields[i], longaccumulator[i]); + } + if (floataccfields != null && floataccumulator != null) { + for (int i = 0; i < floataccfields.length; i++) this.accFloat.put(floataccfields[i], floataccumulator[i]); + } } @Override @@ -213,7 +224,7 @@ public class MapDataMining extends MapHeap { float valued; Long longaccumulator; Float floataccumulator; - if (this.longaccfields != null) + if (this.longaccfields != null) { for (final String longaccfield : this.longaccfields) { value = map.get(longaccfield); if (value != null) { @@ -228,7 +239,8 @@ public class MapDataMining extends MapHeap { } catch (final NumberFormatException e) {} } } - if (this.floataccfields != null) + } + if (this.floataccfields != null) { for (final String floataccfield : this.floataccfields) { value = map.get(floataccfield); if (value != null) { @@ -243,6 +255,7 @@ public class MapDataMining extends MapHeap { } catch (final NumberFormatException e) {} } } + } } private void updateSortCluster(final String key, final Map map) { @@ -283,23 +296,6 @@ public class MapDataMining extends MapHeap { super.delete(key); } -/* would be better but does not work (recursion) - @Override - public synchronized void delete(final byte[] key) throws IOException { - if (key == null) return; - - // update elementCount - Map map = super.remove(key); - if (map != null && (sortfields != null || longaccfields != null || floataccfields != null)) { - // update accumulators (subtract) - if ((longaccfields != null) || (floataccfields != null)) updateAcc(map, false); - - // remove from sortCluster - if (sortfields != null) deleteSortCluster(UTF8.String(key)); - } - } -*/ - private void deleteSortCluster(final String key) { if (key == null) return; ScoreMap cluster; @@ -327,34 +323,34 @@ public class MapDataMining extends MapHeap { this.s = s; } + @Override public boolean hasNext() { return this.s.hasNext(); } + @Override public byte[] next() { final String r = this.s.next(); if (r == null) return null; return UTF8.getBytes(r); } + @Override public void remove() { this.s.remove(); } } - public synchronized mapIterator maps(final boolean up, final String field) { - return new mapIterator(keys(up, field)); + @Override + public synchronized Iterator>> entries(final String whereKey, final String isValue) throws IOException { + return super.entries(whereKey, isValue); } - - public synchronized mapIterator maps(final boolean up, final boolean rotating) throws IOException { - return new mapIterator(keys(up, rotating)); + + public synchronized Iterator>> entries(final boolean up, final String field) { + return new MapIterator(keys(up, field), null, null); } - - public synchronized mapIterator maps(final boolean up, final boolean rotating, final byte[] firstKey, final byte[] secondKey) throws IOException { - return new mapIterator(keys(up, rotating, firstKey, secondKey)); - } - + public synchronized long getLongAcc(final String field) { final Long accumulator = this.accLong.get(field); if (accumulator == null) return -1; @@ -389,50 +385,33 @@ public class MapDataMining extends MapHeap { super.close(); } - public class mapIterator extends LookAheadIterator> implements Iterator> { - // enumerates Map-Type elements - // the key is also included in every map that is returned; it's key is 'key' +/* + public byte[] lookupBy( + final String whereKey, + final String isValue + ) { - private final Iterator keyIterator; - - private mapIterator(final Iterator keyIterator) { - this.keyIterator = keyIterator; - } - - public Map next0() { - if (this.keyIterator == null) return null; - byte[] nextKey; - Map map; - while (this.keyIterator.hasNext()) { - nextKey = this.keyIterator.next(); - try { - map = get(nextKey, false); - } catch (final IOException e) { - Log.logWarning("MapDataMining", e.getMessage()); - continue; - } catch (final RowSpaceExceededException e) { - Log.logException(e); - continue; - } - if (map == null) continue; // circumvention of a modified exception - map.put("key", UTF8.String(nextKey)); - return map; - } - return null; - } - } // class mapIterator + } +*/ public static void main(final String[] args) { try { - final MapDataMining db = new MapDataMining(new File("/tmp/MapDataMinig.test.db"), Word.commonHashLength, Base64Order.enhancedCoder, 1024 * 512, 500, new String[] {"X"}, new String[] {"X"}, new String[] {}, null); - final Map m1 = new HashMap(); m1.put("X", Long.toString(System.currentTimeMillis())); + File f = new File("/tmp/MapDataMinig.test.db"); + f.delete(); + final MapDataMining db = new MapDataMining(f, Word.commonHashLength, Base64Order.enhancedCoder, 1024 * 512, 500, new String[] {"X"}, new String[] {"X"}, new String[] {}, null); + final Map m1 = new HashMap(); + long t = System.currentTimeMillis(); + m1.put("X", Long.toString(t)); db.put("abcdefghijk1".getBytes(), m1); - final Map m2 = new HashMap(); m2.put("X", Long.toString(System.currentTimeMillis() - 1000)); + final Map m2 = new HashMap(); + m2.put("X", Long.toString(t - 1000)); db.put("abcdefghijk2".getBytes(), m2); - final Map m3 = new HashMap(); m3.put("X", Long.toString(System.currentTimeMillis() + 2000)); + final Map m3 = new HashMap(); + m3.put("X", Long.toString(t + 2000)); db.put("abcdefghijk3".getBytes(), m3); + // iterate the keys, sorted by field X in ascending order (must be: abcdefghijk2 - abcdefghijk1 - abcdefghijk3) final Iterator i1 = db.keys(true, "X"); byte[] k; while (i1.hasNext()) { @@ -440,11 +419,12 @@ public class MapDataMining extends MapHeap { System.out.println(new String(k)); } - final Iterator> i2 = db.maps(false, "X"); - Map e; + // iterate the maps, sorted by field X in descending order (must be: abcdefghijk3 - abcdefghijk1 - abcdefghijk2) + final Iterator>> i2 = db.entries(false, "X"); + Map.Entry> e; while (i2.hasNext()) { e = i2.next(); - System.out.println(e); + System.out.println(UTF8.String(e.getKey()) + ":" + e.getValue()); } System.exit(0); diff --git a/source/net/yacy/kelondro/blob/MapHeap.java b/source/net/yacy/kelondro/blob/MapHeap.java index bee7400f2..1d1611244 100644 --- a/source/net/yacy/kelondro/blob/MapHeap.java +++ b/source/net/yacy/kelondro/blob/MapHeap.java @@ -32,6 +32,7 @@ import java.io.ByteArrayInputStream; import java.io.File; import java.io.IOException; import java.io.InputStreamReader; +import java.util.AbstractMap; import java.util.Collection; import java.util.HashMap; import java.util.Iterator; @@ -51,8 +52,8 @@ import net.yacy.kelondro.order.CloneableIterator; import net.yacy.kelondro.order.NaturalOrder; import net.yacy.kelondro.order.RotateIterator; import net.yacy.kelondro.util.FileUtils; +import net.yacy.kelondro.util.LookAheadIterator; import net.yacy.kelondro.util.MemoryControl; -import net.yacy.kelondro.util.kelondroException; public class MapHeap implements Map> { @@ -85,6 +86,7 @@ public class MapHeap implements Map> { * clears the content of the database * @throws IOException */ + @Override public synchronized void clear() { try { this.blob.clear(); @@ -168,6 +170,7 @@ public class MapHeap implements Map> { } } + @Override public Map put(final byte[] key, final Map newMap) { Map v = null; try { @@ -200,6 +203,7 @@ public class MapHeap implements Map> { } } + @Override public Map remove(final Object key) { Map v = null; try { @@ -218,6 +222,7 @@ public class MapHeap implements Map> { * @throws IOException */ + @Override public boolean containsKey(final Object k) { if (!(k instanceof byte[])) return false; assert k != null; @@ -241,6 +246,7 @@ public class MapHeap implements Map> { return get(key, true); } + @Override public Map get(final Object key) { if (key == null) return null; try { @@ -375,6 +381,7 @@ public class MapHeap implements Map> { this.iterator = (rotating) ? new RotateIterator(i, secondKey, MapHeap.this.blob.size()) : i; } + @Override public byte[] next() { return removeFillchar(this.iterator.next()); } @@ -399,23 +406,29 @@ public class MapHeap implements Map> { } } + + public synchronized Iterator>> entries(final String whereKey, final String isValue) throws IOException { + return new MapIterator(this.blob.keys(true, null), whereKey, isValue); + } - public synchronized MapIterator entries(final boolean up, final boolean rotating) throws IOException { - return new MapIterator(keys(up, rotating)); + public synchronized Iterator>> entries(final boolean up, final boolean rotating) throws IOException { + return new MapIterator(keys(up, rotating), null, null); } - public synchronized MapIterator entries(final boolean up, final boolean rotating, final byte[] firstKey, final byte[] secondKey) throws IOException { - return new MapIterator(keys(up, rotating, firstKey, secondKey)); + public synchronized Iterator>> entries(final boolean up, final boolean rotating, final byte[] firstKey, final byte[] secondKey) throws IOException { + return new MapIterator(keys(up, rotating, firstKey, secondKey), null, null); } /** * ask for the number of entries * @return the number of entries in the table */ + @Override public synchronized int size() { return (this.blob == null) ? 0 : this.blob.size(); } + @Override public synchronized boolean isEmpty() { return (this.blob == null) ? true : this.blob.isEmpty(); } @@ -435,48 +448,52 @@ public class MapHeap implements Map> { public void finalize() { close(); } - - public class MapIterator implements Iterator> { + + public class MapIterator extends LookAheadIterator>> implements Iterator>> { // enumerates Map-Type elements // the key is also included in every map that is returned; it's key is 'key' - Iterator keyIterator; - boolean finish; + private final Iterator keyIterator; + private final String whereKey, isValue; - public MapIterator(final Iterator keyIterator) { + MapIterator(final Iterator keyIterator, final String whereKey, final String isValue) { this.keyIterator = keyIterator; - this.finish = false; - } - - public boolean hasNext() { - return (!(this.finish)) && (this.keyIterator.hasNext()); + this.whereKey = whereKey; + this.isValue = isValue; } - public Map next() { - byte[] nextKey = this.keyIterator.next(); - if (nextKey == null) { - this.finish = true; - return null; - } - nextKey = normalizeKey(nextKey); // the key must be normalized because the keyIterator may iterate over not-normalized keys - try { - final Map obj = get(nextKey, false); - if (obj == null) throw new kelondroException("no more elements available"); - return obj; - } catch (final IOException e) { - this.finish = true; - return null; - } catch (final RowSpaceExceededException e) { - this.finish = true; - return null; + @Override + public Map.Entry> next0() { + if (this.keyIterator == null) return null; + byte[] nextKey; + Map map; + while (this.keyIterator.hasNext()) { + nextKey = this.keyIterator.next(); + try { + map = get(nextKey, false); + } catch (final IOException e) { + Log.logWarning("MapDataMining", e.getMessage()); + continue; + } catch (final RowSpaceExceededException e) { + Log.logException(e); + continue; + } + if (map == null) continue; // circumvention of a modified exception + // check if the where case holds + if (this.whereKey != null && this.isValue != null) { + String v = map.get(this.whereKey); + if (v == null) continue; + if (!v.equals(this.isValue)) continue; + } + // produce entry + Map.Entry> entry = new AbstractMap.SimpleImmutableEntry>(nextKey, map); + return entry; } - } - - public void remove() { - throw new UnsupportedOperationException(); + return null; } } // class mapIterator + @Override public void putAll(final Map> map) { for (final Map.Entry> me: map.entrySet()) { try { @@ -489,6 +506,7 @@ public class MapHeap implements Map> { } } + @Override public Set keySet() { final TreeSet set = new TreeSet(this.blob.ordering()); try { @@ -498,16 +516,19 @@ public class MapHeap implements Map> { return set; } + @Override public Collection> values() { // this method shall not be used because it is not appropriate for this kind of data throw new UnsupportedOperationException(); } + @Override public Set>> entrySet() { // this method shall not be used because it is not appropriate for this kind of data throw new UnsupportedOperationException(); } + @Override public boolean containsValue(final Object value) { // this method shall not be used because it is not appropriate for this kind of data throw new UnsupportedOperationException(); diff --git a/source/net/yacy/kelondro/data/meta/DigestURI.java b/source/net/yacy/kelondro/data/meta/DigestURI.java index f80501e49..77f3d0fed 100644 --- a/source/net/yacy/kelondro/data/meta/DigestURI.java +++ b/source/net/yacy/kelondro/data/meta/DigestURI.java @@ -213,8 +213,10 @@ public class DigestURI extends MultiProtocolURI implements Serializable { int p = (this.host == null) ? -1 : this.host.lastIndexOf('.'); String dom = (p > 0) ? dom = this.host.substring(0, p) : ""; p = dom.lastIndexOf('.'); // locate subdomain - String subdom = ""; - if (p > 0) { + final String subdom; + if (p <= 0) { + subdom = ""; + } else { subdom = dom.substring(0, p); dom = dom.substring(p + 1); } diff --git a/source/net/yacy/kelondro/data/meta/URIMetadataRow.java b/source/net/yacy/kelondro/data/meta/URIMetadataRow.java index 4f1fcb564..d0f5e9286 100644 --- a/source/net/yacy/kelondro/data/meta/URIMetadataRow.java +++ b/source/net/yacy/kelondro/data/meta/URIMetadataRow.java @@ -189,12 +189,15 @@ public class URIMetadataRow implements URIMetadata { final float lat, final float lon) { final CharBuffer s = new CharBuffer(360); - s.append(url.toNormalform(false, true)).append(10); - s.append(dc_title).append(10); - s.append(dc_creator.length() > 80 ? dc_creator.substring(0, 80) : dc_creator).append(10); - s.append(dc_subject.length() > 120 ? dc_subject.substring(0, 120) : dc_subject).append(10); - s.append(dc_publisher).append(10); - if (lon == 0.0f && lat == 0.0f) s.append(10); else s.append(Float.toString(lat)).append(',').append(Float.toString(lon)).append(10); + s.append(url.toNormalform(false, true)).appendLF(); + s.append(dc_title).appendLF(); + if (dc_creator.length() > 80) s.append(dc_creator, 0, 80); else s.append(dc_creator); + s.appendLF(); + if (dc_subject.length() > 120) s.append(dc_subject, 0, 120); else s.append(dc_subject); + s.appendLF(); + if (dc_publisher.length() > 80) s.append(dc_publisher, 0, 80); else s.append(dc_publisher); + s.appendLF(); + if (lon == 0.0f && lat == 0.0f) s.appendLF(); else s.append(Float.toString(lat)).append(',').append(Float.toString(lon)).appendLF(); return UTF8.getBytes(s.toString()); } diff --git a/source/net/yacy/kelondro/io/CharBuffer.java b/source/net/yacy/kelondro/io/CharBuffer.java index 494d56006..70fe1b3f3 100644 --- a/source/net/yacy/kelondro/io/CharBuffer.java +++ b/source/net/yacy/kelondro/io/CharBuffer.java @@ -45,9 +45,9 @@ public final class CharBuffer extends Writer { public CharBuffer() { - buffer = new char[10]; - length = 0; - offset = 0; + this.buffer = new char[10]; + this.length = 0; + this.offset = 0; } public CharBuffer(final int initLength) { @@ -57,44 +57,44 @@ public final class CharBuffer extends Writer { } public CharBuffer(final char[] bb) { - buffer = bb; - length = bb.length; - offset = 0; + this.buffer = bb; + this.length = bb.length; + this.offset = 0; } public CharBuffer(final char[] bb, final int initLength) { this.buffer = new char[initLength]; - System.arraycopy(bb, 0, buffer, 0, bb.length); - length = bb.length; - offset = 0; + System.arraycopy(bb, 0, this.buffer, 0, bb.length); + this.length = bb.length; + this.offset = 0; } public CharBuffer(final char[] bb, final int of, final int le) { if (of * 2 > bb.length) { - buffer = new char[le]; - System.arraycopy(bb, of, buffer, 0, le); - length = le; - offset = 0; + this.buffer = new char[le]; + System.arraycopy(bb, of, this.buffer, 0, le); + this.length = le; + this.offset = 0; } else { - buffer = bb; - length = le; - offset = of; + this.buffer = bb; + this.length = le; + this.offset = of; } } public CharBuffer(final CharBuffer bb) { - buffer = bb.buffer; - length = bb.length; - offset = bb.offset; + this.buffer = bb.buffer; + this.length = bb.length; + this.offset = bb.offset; } public CharBuffer(final File f) throws IOException { // initially fill the buffer with the content of a file if (f.length() > Integer.MAX_VALUE) throw new IOException("file is too large for buffering"); - length = 0; - buffer = new char[(int) f.length()*2]; - offset = 0; + this.length = 0; + this.buffer = new char[(int) f.length()*2]; + this.offset = 0; FileReader fr = null; try { @@ -114,52 +114,66 @@ public final class CharBuffer extends Writer { public void clear() { this.buffer = new char[0]; - length = 0; - offset = 0; + this.length = 0; + this.offset = 0; } public int length() { - return length; + return this.length; } private void grow(int minSize) { - int newsize = buffer.length + 1024; + int newsize = this.buffer.length + 1024; if (newsize < minSize) newsize = minSize+1; char[] tmp = new char[newsize]; - System.arraycopy(buffer, offset, tmp, 0, length); - buffer = tmp; - offset = 0; + System.arraycopy(this.buffer, this.offset, tmp, 0, this.length); + this.buffer = tmp; + this.offset = 0; } + @Override public void write(final int b) { write((char)b); } public void write(final char b) { - if (offset + length + 1 > buffer.length) grow(offset + length + 1); - buffer[offset + length++] = b; + if (this.offset + this.length + 1 > this.buffer.length) grow(this.offset + this.length + 1); + this.buffer[this.offset + this.length++] = b; } + @Override public void write(final char[] bb) { write(bb, 0, bb.length); } + @Override public void write(final char[] bb, final int of, final int le) { - if (offset + length + le > buffer.length) grow(offset + length + le); - System.arraycopy(bb, of, buffer, offset + length, le); - length += le; + if (this.offset + this.length + le > this.buffer.length) grow(this.offset + this.length + le); + System.arraycopy(bb, of, this.buffer, this.offset + this.length, le); + this.length += le; + } + + private static final char SPACE = ' '; + private static final char CR = (char) 13; + private static final char LF = (char) 10; + + public CharBuffer appendSpace() { + write(SPACE); + return this; } -// do not use/implement the following method, a -// "overridden method is a bridge method" -// will occur -// public serverCharBuffer append(char b) { -// write(b); -// return this; -// } + public CharBuffer appendCR() { + write(CR); + return this; + } + + public CharBuffer appendLF() { + write(LF); + return this; + } public CharBuffer append(final int i) { - write((char) (i)); + write((char) i); return this; } @@ -173,6 +187,7 @@ public final class CharBuffer extends Writer { return this; } + @Override public CharBuffer append(final char c) { write(c); return this; @@ -180,39 +195,36 @@ public final class CharBuffer extends Writer { public CharBuffer append(final String s) { final char[] temp = new char[s.length()]; - s.getChars(0, temp.length, temp, 0); - return append(temp); + s.getChars(0, temp.length, temp, 0); + write(temp); + return this; } public CharBuffer append(final String s, final int off, final int len) { final char[] temp = new char[len]; - s.getChars(off, (off + len), temp, 0); - return append(temp); + s.getChars(off, (off + len), temp, 0); + write(temp); + return this; } public CharBuffer append(final CharBuffer bb) { - return append(bb.buffer, bb.offset, bb.length); + write(bb.buffer, bb.offset, bb.length); + return this; } - -// public serverCharBuffer append(Object o) { -// if (o instanceof String) return append((String) o); -// if (o instanceof char[]) return append((char[]) o); -// return null; -// } public char charAt(final int pos) { if (pos < 0) throw new IndexOutOfBoundsException(); - if (pos > length) throw new IndexOutOfBoundsException(); - return buffer[offset + pos]; + if (pos > this.length) throw new IndexOutOfBoundsException(); + return this.buffer[this.offset + pos]; } public void deleteCharAt(final int pos) { if (pos < 0) return; - if (pos >= length) return; - if (pos == length - 1) { - length--; + if (pos >= this.length) return; + if (pos == this.length - 1) { + this.length--; } else { - System.arraycopy(buffer, offset + pos + 1, buffer, offset + pos, length - pos - 1); + System.arraycopy(this.buffer, this.offset + pos + 1, this.buffer, this.offset + pos, this.length - pos - 1); } } @@ -225,20 +237,20 @@ public final class CharBuffer extends Writer { } public int indexOf(final char b, final int start) { - if (start >= length) return -1; - for (int i = start; i < length; i++) if (buffer[offset + i] == b) return i; + if (start >= this.length) return -1; + for (int i = start; i < this.length; i++) if (this.buffer[this.offset + i] == b) return i; return -1; } public int indexOf(final char[] bs, final int start) { - if (start + bs.length > length) return -1; - loop: for (int i = start; i <= length - bs.length; i++) { + if (start + bs.length > this.length) return -1; + loop: for (int i = start; i <= this.length - bs.length; i++) { // first test only first char - if (buffer[offset + i] != bs[0]) continue loop; + if (this.buffer[this.offset + i] != bs[0]) continue loop; // then test all remaining char for (int j = 1; j < bs.length; j++) { - if (buffer[offset + i + j] != bs[j]) continue loop; + if (this.buffer[this.offset + i + j] != bs[j]) continue loop; } // found hit @@ -278,14 +290,14 @@ public final class CharBuffer extends Writer { } public int lastIndexOf(final char b) { - for (int i = length - 1; i >= 0; i--) if (buffer[offset + i] == b) return i; + for (int i = this.length - 1; i >= 0; i--) if (this.buffer[this.offset + i] == b) return i; return -1; } public boolean startsWith(final char[] bs) { - if (length < bs.length) return false; + if (this.length < bs.length) return false; for (int i = 0; i < bs.length; i++) { - if (buffer[offset + i] != bs[i]) return false; + if (this.buffer[this.offset + i] != bs[i]) return false; } return true; } @@ -295,45 +307,45 @@ public final class CharBuffer extends Writer { } public char[] getChars(final int start) { - return getChars(start, length); + return getChars(start, this.length); } public char[] getChars(final int start, final int end) { // start is inclusive, end is exclusive - if (end > length) throw new IndexOutOfBoundsException("getBytes: end > length"); - if (start > length) throw new IndexOutOfBoundsException("getBytes: start > length"); + if (end > this.length) throw new IndexOutOfBoundsException("getBytes: end > length"); + if (start > this.length) throw new IndexOutOfBoundsException("getBytes: start > length"); final char[] tmp = new char[end - start]; - System.arraycopy(buffer, offset + start, tmp, 0, end - start); + System.arraycopy(this.buffer, this.offset + start, tmp, 0, end - start); return tmp; } public byte[] getBytes() { - return UTF8.getBytes(new String(buffer, offset, length)); + return UTF8.getBytes(new String(this.buffer, this.offset, this.length)); } public CharBuffer trim(final int start) { // the end value is outside (+1) of the wanted target array - if (start > length) throw new IndexOutOfBoundsException("trim: start > length"); - offset = offset + start; - length = length - start; + if (start > this.length) throw new IndexOutOfBoundsException("trim: start > length"); + this.offset = this.offset + start; + this.length = this.length - start; return this; } public CharBuffer trim(final int start, final int end) { // the end value is outside (+1) of the wanted target array - if (start > length) throw new IndexOutOfBoundsException("trim: start > length"); - if (end > length) throw new IndexOutOfBoundsException("trim: end > length"); + if (start > this.length) throw new IndexOutOfBoundsException("trim: start > length"); + if (end > this.length) throw new IndexOutOfBoundsException("trim: end > length"); if (start > end) throw new IndexOutOfBoundsException("trim: start > end"); - offset = offset + start; - length = end - start; + this.offset = this.offset + start; + this.length = end - start; return this; } public CharBuffer trim() { int l = 0; - while ((l < length) && (buffer[offset + l] <= ' ')) l++; - int r = length; - while ((r > 0) && (buffer[offset + r - 1] <= ' ')) r--; + while ((l < this.length) && (this.buffer[this.offset + l] <= ' ')) l++; + int r = this.length; + while ((r > 0) && (this.buffer[this.offset + r - 1] <= ' ')) r--; if (l > r) r = l; return trim(l, r); } @@ -342,12 +354,12 @@ public final class CharBuffer extends Writer { // returns true, if trim() would result in an empty serverByteBuffer if (includeNonLetterBytes) { char b; - for (int i = 0; i < length; i++) { - b = buffer[offset + i]; + for (int i = 0; i < this.length; i++) { + b = this.buffer[this.offset + i]; if (((b >= '0') && (b <= '9')) || ((b >= 'A') && (b <= 'Z')) || ((b >= 'a') && (b <= 'z'))) return false; } } else { - for (int i = 0; i < length; i++) if (buffer[offset + i] > 32) return false; + for (int i = 0; i < this.length; i++) if (this.buffer[this.offset + i] > 32) return false; } return true; } @@ -356,86 +368,87 @@ public final class CharBuffer extends Writer { // returns number of whitespace char at the beginning of text if (includeNonLetterBytes) { char b; - for (int i = 0; i < length; i++) { - b = buffer[offset + i]; + for (int i = 0; i < this.length; i++) { + b = this.buffer[this.offset + i]; if (((b >= '0') && (b <= '9')) || ((b >= 'A') && (b <= 'Z')) || ((b >= 'a') && (b <= 'z'))) return i; } } else { - for (int i = 0; i < length; i++) if (buffer[offset + i] > 32) return i; + for (int i = 0; i < this.length; i++) if (this.buffer[this.offset + i] > 32) return i; } - return length; + return this.length; } public int whitespaceEnd(final boolean includeNonLetterBytes) { // returns position of whitespace at the end of text if (includeNonLetterBytes) { char b; - for (int i = length - 1; i >= 0; i--) { - b = buffer[offset + i]; + for (int i = this.length - 1; i >= 0; i--) { + b = this.buffer[this.offset + i]; if (((b >= '0') && (b <= '9')) || ((b >= 'A') && (b <= 'Z')) || ((b >= 'a') && (b <= 'z'))) return i + 1; } } else { - for (int i = length - 1; i >= 0; i--) if (buffer[offset + i] > 32) return i + 1; + for (int i = this.length - 1; i >= 0; i--) if (this.buffer[this.offset + i] > 32) return i + 1; } return 0; } + @Override public String toString() { - return new String(buffer, offset, length); + return new String(this.buffer, this.offset, this.length); } public String toString(final int left, final int rightbound) { - return new String(buffer, offset + left, rightbound - left); + return new String(this.buffer, this.offset + left, rightbound - left); } public Properties propParser() { // extract a=b or a="b" - relations from the buffer - int pos = offset; + int pos = this.offset; int start; String key; final Properties p = new Properties(); // eat up spaces at beginning - while ((pos < length) && (buffer[pos] <= 32)) pos++; - while (pos < length) { + while ((pos < this.length) && (this.buffer[pos] <= 32)) pos++; + while (pos < this.length) { // pos is at start of next key start = pos; - while ((pos < length) && (buffer[pos] != equal)) pos++; - if (pos >= length) break; // this is the case if we found no equal - key = new String(buffer, start, pos - start).trim().toLowerCase(); + while ((pos < this.length) && (this.buffer[pos] != equal)) pos++; + if (pos >= this.length) break; // this is the case if we found no equal + key = new String(this.buffer, start, pos - start).trim().toLowerCase(); // we have a key pos++; // find start of value - while ((pos < length) && (buffer[pos] <= 32)) pos++; + while ((pos < this.length) && (this.buffer[pos] <= 32)) pos++; // doublequotes are obligatory. However, we want to be fuzzy if they // are ommittet - if (pos >= length) { + if (pos >= this.length) { // error case: input ended too early break; - } else if (buffer[pos] == doublequote) { + } else if (this.buffer[pos] == doublequote) { // search next doublequote pos++; start = pos; - while ((pos < length) && (buffer[pos] != doublequote)) pos++; - if (pos >= length) break; // this is the case if we found no parent doublequote - p.setProperty(key, new String(buffer, start, pos - start).trim()); + while ((pos < this.length) && (this.buffer[pos] != doublequote)) pos++; + if (pos >= this.length) break; // this is the case if we found no parent doublequote + p.setProperty(key, new String(this.buffer, start, pos - start).trim()); pos++; - } else if (buffer[pos] == singlequote) { + } else if (this.buffer[pos] == singlequote) { // search next singlequote pos++; start = pos; - while ((pos < length) && (buffer[pos] != singlequote)) pos++; - if (pos >= length) break; // this is the case if we found no parent singlequote - p.setProperty(key, new String(buffer, start, pos - start).trim()); + while ((pos < this.length) && (this.buffer[pos] != singlequote)) pos++; + if (pos >= this.length) break; // this is the case if we found no parent singlequote + p.setProperty(key, new String(this.buffer, start, pos - start).trim()); pos++; } else { // search next whitespace start = pos; - while ((pos < length) && (buffer[pos] > 32)) pos++; - p.setProperty(key, new String(buffer, start, pos - start).trim()); + while ((pos < this.length) && (this.buffer[pos] > 32)) pos++; + p.setProperty(key, new String(this.buffer, start, pos - start).trim()); } // pos should point now to a whitespace: eat up spaces - while ((pos < length) && (buffer[pos] <= 32)) pos++; + while ((pos < this.length) && (this.buffer[pos] <= 32)) pos++; // go on with next loop } return p; @@ -475,10 +488,12 @@ public final class CharBuffer extends Writer { return newbuf; } + @Override public void close() throws IOException { - buffer = null; // assist with garbage collection + this.buffer = null; // assist with garbage collection } + @Override public void flush() throws IOException { // TODO Auto-generated method stub } diff --git a/source/net/yacy/peers/SeedDB.java b/source/net/yacy/peers/SeedDB.java index 21e5ee02e..baaa8f35e 100644 --- a/source/net/yacy/peers/SeedDB.java +++ b/source/net/yacy/peers/SeedDB.java @@ -274,22 +274,27 @@ public final class SeedDB implements AlternativeDomainNames { this.myBotIDs.add(name + ".yacy"); } + @Override public String myAlternativeAddress() { return mySeed().getName() + ".yacy"; } + @Override public String myIP() { return mySeed().getIP(); } + @Override public int myPort() { return mySeed().getPort(); } + @Override public String myName() { return this.mySeed.getName(); } + @Override public String myID() { return this.mySeed.hash; } @@ -697,7 +702,7 @@ public final class SeedDB implements AlternativeDomainNames { final HandleSet badPeerHashes = new HandleSet(12, Base64Order.enhancedCoder, 0); if (lookupConnected) { - // enumerate the cache and simultanous insert values + // enumerate the cache and simultaneously insert values final Iterator e = seedsConnected(true, false, null, (float) 0.0); while (e.hasNext()) { seed = e.next(); @@ -1004,7 +1009,7 @@ public final class SeedDB implements AlternativeDomainNames { private class seedEnum implements Iterator { - private MapDataMining.mapIterator it; + private Iterator>> it; private Seed nextSeed; private final MapDataMining database; private float minVersion; @@ -1013,7 +1018,7 @@ public final class SeedDB implements AlternativeDomainNames { this.database = database; this.minVersion = minVersion; try { - this.it = (firstKey == null) ? database.maps(up, rot) : database.maps(up, rot, firstKey, secondKey); + this.it = (firstKey == null) ? database.entries(up, rot) : database.entries(up, rot, firstKey, secondKey); float version; while (true) { this.nextSeed = internalNext(); @@ -1039,7 +1044,7 @@ public final class SeedDB implements AlternativeDomainNames { private seedEnum(final boolean up, final String field, final MapDataMining database) { this.database = database; try { - this.it = database.maps(up, field); + this.it = database.entries(up, field); this.nextSeed = internalNext(); } catch (final kelondroException e) { Log.logException(e); @@ -1058,7 +1063,7 @@ public final class SeedDB implements AlternativeDomainNames { private Seed internalNext() { if (this.it == null || !(this.it.hasNext())) return null; try { - Map dna0; + Map.Entry> dna0; ConcurrentHashMap dna; while (this.it.hasNext()) { try { @@ -1069,15 +1074,14 @@ public final class SeedDB implements AlternativeDomainNames { } assert dna0 != null; if (dna0 == null) continue; - if (dna0 instanceof ConcurrentHashMap) { - dna = (ConcurrentHashMap) dna0; + if (dna0.getValue() instanceof ConcurrentHashMap) { + dna = (ConcurrentHashMap) dna0.getValue(); } else { dna = new ConcurrentHashMap(); - dna.putAll(dna0); + dna.putAll(dna0.getValue()); } - final String hash = dna.remove("key"); - //assert hash != null; - if (hash == null) continue; // bad seed + if (dna0.getKey() == null) continue; // bad seed + final String hash = UTF8.String(dna0.getKey()); return new Seed(hash, dna); } return null; diff --git a/source/net/yacy/yacy.java b/source/net/yacy/yacy.java index 4d6e2521f..90fef8654 100644 --- a/source/net/yacy/yacy.java +++ b/source/net/yacy/yacy.java @@ -47,6 +47,7 @@ import java.util.zip.ZipOutputStream; import net.yacy.cora.date.GenericFormatter; import net.yacy.cora.document.ASCII; +import net.yacy.cora.document.UTF8; import net.yacy.cora.protocol.ClientIdentification; import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.protocol.http.HTTPClient; @@ -881,15 +882,15 @@ public final class yacy { final File dbFile = new File(yacyDBPath,dbFileName); final MapDataMining db = new MapDataMining(dbFile, Word.commonHashLength, Base64Order.enhancedCoder, 1024 * 512, 500, SeedDB.sortFields, SeedDB.longaccFields, SeedDB.doubleaccFields, null); - MapDataMining.mapIterator it; - it = db.maps(true, false); + Iterator>> it; + it = db.entries(true, false); while (it.hasNext()) { - final Map dna = it.next(); - String peerHash = dna.get("key"); + final Map.Entry> dna = it.next(); + String peerHash = UTF8.String(dna.getKey()); if (peerHash.length() < Word.commonHashLength) { - final String peerName = dna.get("Name"); - final String peerIP = dna.get("IP"); - final String peerPort = dna.get("Port"); + final String peerName = dna.getValue().get("Name"); + final String peerIP = dna.getValue().get("IP"); + final String peerPort = dna.getValue().get("Port"); while (peerHash.length() < Word.commonHashLength) { peerHash = peerHash + "_"; } System.err.println("Invalid Peer-Hash found in '" + dbFileName + "': " + peerName + ":" + peerHash + ", http://" + peerIP + ":" + peerPort);