boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

performance hacks

Browse Source
pull/1/head
Michael Christen 13 years ago
parent 675d557e88
commit 1f4afb4dc0
12 changed files with 366 additions and 316 deletions
Show Stats Download Patch File Download Diff File

33
source/net/yacy/cora/ranking/ClusteredScoreMap.java
Unescape View File

@ -58,10 +58,12 @@ public final class ClusteredScoreMap<E> extends AbstractScoreMap<E> implements R
this.encnt = 0;
}
@Override
public Iterator<E> iterator() {
return this.map.keySet().iterator();
}
@Override
public synchronized void clear() {
this.map.clear();
this.pam.clear();
@ -73,6 +75,7 @@ public final class ClusteredScoreMap<E> extends AbstractScoreMap<E> implements R
* shrink the cluster to a demanded size
* @param maxsize
*/
@Override
public void shrinkToMaxSize(final int maxsize) {
if (maxsize < 0) return;
Long key;
@ -90,6 +93,7 @@ public final class ClusteredScoreMap<E> extends AbstractScoreMap<E> implements R
* shrink the cluster in such a way that the smallest score is equal or greater than a given minScore
* @param minScore
*/
@Override
public void shrinkToMinScore(final int minScore) {
int score;
Long key;
@ -121,7 +125,7 @@ public final class ClusteredScoreMap<E> extends AbstractScoreMap<E> implements R
if (o instanceof Long) {
final long l = ((Long) o).longValue();
if (l < Integer.MAX_VALUE) return (int) l;
o = ((Long) o).toString();
return (int) (l & Integer.MAX_VALUE);
}
if (o instanceof Float) {
final double d = 1000f * ((Float) o).floatValue();
@ -148,13 +152,13 @@ public final class ClusteredScoreMap<E> extends AbstractScoreMap<E> implements R
l = Long.parseLong(s);
}
// fix out-of-ranges
if (l > Integer.MAX_VALUE) return Integer.MAX_VALUE; //(int) (l & (Integer.MAX_VALUE));
if (l > Integer.MAX_VALUE) return (int) (l & Integer.MAX_VALUE);
if (l < 0) {
System.out.println("string2score: negative score for input " + s);
return 0;
}
return (int) l;
} catch (final Exception e) {
} catch (final Throwable e) {
// try it lex
int len = s.length();
if (len > 5) len = 5;
@ -188,26 +192,32 @@ public final class ClusteredScoreMap<E> extends AbstractScoreMap<E> implements R
return this.gcount;
}
@Override
public synchronized int size() {
return this.map.size();
}
@Override
public boolean sizeSmaller(final int size) {
return this.map.size() < size;
}
@Override
public synchronized boolean isEmpty() {
return this.map.isEmpty();
}
@Override
public synchronized void inc(final E obj) {
inc(obj, 1);
}
@Override
public synchronized void dec(final E obj) {
inc(obj, -1);
}
@Override
public void set(final E obj, final int newScore) {
if (obj == null) return;
synchronized (this) {
@ -242,6 +252,7 @@ public final class ClusteredScoreMap<E> extends AbstractScoreMap<E> implements R
this.gcount += newScore;
}
@Override
public void inc(final E obj, final int incrementScore) {
if (obj == null) return;
synchronized (this) {
@ -277,10 +288,12 @@ public final class ClusteredScoreMap<E> extends AbstractScoreMap<E> implements R
this.gcount += incrementScore;
}
@Override
public void dec(final E obj, final int incrementScore) {
inc(obj, -incrementScore);
}
@Override
public int delete(final E obj) {
// deletes entry and returns previous score
if (obj == null) return 0;
@ -302,10 +315,12 @@ public final class ClusteredScoreMap<E> extends AbstractScoreMap<E> implements R
return oldScore;
}
@Override
public synchronized boolean containsKey(final E obj) {
return this.map.containsKey(obj);
}
@Override
public int get(final E obj) {
if (obj == null) return 0;
final Long cs;
@ -316,30 +331,36 @@ public final class ClusteredScoreMap<E> extends AbstractScoreMap<E> implements R
return (int) ((cs.longValue() & 0xFFFFFFFF00000000L) >> 32);
}
@Override
public synchronized int getMaxScore() {
if (this.map.isEmpty()) return -1;
return (int) ((this.pam.lastKey().longValue() & 0xFFFFFFFF00000000L) >> 32);
}
@Override
public synchronized int getMinScore() {
if (this.map.isEmpty()) return -1;
return (int) ((this.pam.firstKey().longValue() & 0xFFFFFFFF00000000L) >> 32);
}
@Override
public synchronized E getMaxKey() {
if (this.map.isEmpty()) return null;
return this.pam.get(this.pam.lastKey());
}
@Override
public synchronized E getMinKey() {
if (this.map.isEmpty()) return null;
return this.pam.get(this.pam.firstKey());
}
@Override
public String toString() {
return this.map + " / " + this.pam;
}
@Override
public synchronized Iterator<E> keys(final boolean up) {
if (up) return new simpleScoreIterator<E>();
return new reverseScoreIterator<E>();
@ -354,10 +375,12 @@ public final class ClusteredScoreMap<E> extends AbstractScoreMap<E> implements R
this.view = ClusteredScoreMap.this.pam;
}
@Override
public boolean hasNext() {
return !this.view.isEmpty();
}
@Override
public E next() {
this.key = this.view.lastKey();
this.view = this.view.headMap(this.key);
@ -366,6 +389,7 @@ public final class ClusteredScoreMap<E> extends AbstractScoreMap<E> implements R
return value;
}
@Override
public void remove() {
final Object val = ClusteredScoreMap.this.pam.remove(this.key);
if (val != null) ClusteredScoreMap.this.map.remove(val);
@ -382,16 +406,19 @@ public final class ClusteredScoreMap<E> extends AbstractScoreMap<E> implements R
this.ii = ClusteredScoreMap.this.pam.entrySet().iterator();
}
@Override
public boolean hasNext() {
return this.ii.hasNext();
}
@Override
public E next() {
this.entry = this.ii.next();
//System.out.println("cluster simple iterator: score = " + ((((Long) entry.getKey()).longValue() & 0xFFFFFFFF00000000L) >> 32) + ", handle = " + (((Long) entry.getKey()).longValue() & 0xFFFFFFFFL) + ", value = " + entry.getValue());
return this.entry.getValue();
}
@Override
public void remove() {
this.ii.remove();
if (this.entry.getValue() != null) ClusteredScoreMap.this.map.remove(this.entry.getValue());

38
source/net/yacy/document/parser/html/ContentScraper.java
Unescape View File

@ -115,17 +115,17 @@ public class ContentScraper extends AbstractScraper implements Scraper {
}
// class variables: collectors for links
private Map<MultiProtocolURI, Properties> anchors;
private final Map<MultiProtocolURI, Properties> anchors;
private final Map<MultiProtocolURI, String> rss, css;
private final Set<MultiProtocolURI> script, frames, iframes;
private Map<MultiProtocolURI, ImageEntry> images; // urlhash/image relation
private final Map<MultiProtocolURI, ImageEntry> images; // urlhash/image relation
private final Map<String, String> metas;
private String title;
//private String headline;
private List<String>[] headlines;
private final ClusteredScoreMap<String> bold, italic;
private final List<String> li;
private CharBuffer content;
private final CharBuffer content;
private final EventListenerList htmlFilterEventListeners;
private float lon, lat;
private MultiProtocolURI canonical;
@ -187,23 +187,7 @@ public class ContentScraper extends AbstractScraper implements Scraper {
this.anchors.put(url, p0);
}
/*
private void mergeAnchors(final MultiProtocolURI url, final String key, final String value) {
if (value == null) return;
if (value.length() == 0) return;
Properties p0 = this.anchors.get(url);
if (p0 == null) {
p0 = new Properties();
p0.put(key, value);
this.anchors.put(url, p0);
return;
}
// merge properties
p0.put(key, value);
this.anchors.put(url, p0);
}
*/
@Override
public void scrapeText(final char[] newtext, final String insideTag) {
// System.out.println("SCRAPE: " + UTF8.String(newtext));
int p, pl, q, s = 0;
@ -295,7 +279,10 @@ public class ContentScraper extends AbstractScraper implements Scraper {
} catch (final MalformedURLException e) {}
}
// append string to content
if (b.length() != 0) this.content.append(b).append(32);
if (!b.isEmpty()) {
this.content.append(b);
this.content.appendSpace();
}
}
private final static Pattern dpssp = Pattern.compile("://");
@ -317,6 +304,7 @@ public class ContentScraper extends AbstractScraper implements Scraper {
}
}
@Override
public void scrapeTag0(final String tagname, final Properties tagopts) {
if (tagname.equalsIgnoreCase("img")) {
final String src = tagopts.getProperty("src", EMPTY_STRING);
@ -407,6 +395,7 @@ public class ContentScraper extends AbstractScraper implements Scraper {
fireScrapeTag0(tagname, tagopts);
}
@Override
public void scrapeTag1(final String tagname, final Properties tagopts, final char[] text) {
// System.out.println("ScrapeTag1: tagname=" + tagname + ", opts=" + tagopts.toString() + ", text=" + UTF8.String(text));
if (tagname.equalsIgnoreCase("a") && text.length < 2048) {
@ -481,6 +470,7 @@ public class ContentScraper extends AbstractScraper implements Scraper {
}
@Override
public void scrapeComment(final char[] comment) {
this.evaluationScores.match(Element.comment, comment);
}
@ -837,11 +827,11 @@ public class ContentScraper extends AbstractScraper implements Scraper {
public void close() {
// free resources
super.close();
this.anchors = null;
this.images = null;
this.anchors.clear();
this.images.clear();
this.title = null;
this.headlines = null;
this.content = null;
this.content.clear();
this.root = null;
}

21
source/net/yacy/document/parser/html/ContentTransformer.java
Unescape View File

@ -55,17 +55,18 @@ public class ContentTransformer extends AbstractTransformer implements Transform
super(linkTags0, linkTags1);
}
@Override
public void init(final String initarg) {
if (bluelist == null) {
if (this.bluelist == null) {
// here, the init arg is used to load a list of blue-listed words
bluelist = new ArrayList<String>();
this.bluelist = new ArrayList<String>();
final File f = new File(initarg);
if (f.canRead()) {
try {
final BufferedReader r = new BufferedReader(new FileReader(f));
String s;
while ((s = r.readLine()) != null) {
if (s.length() > 0 && s.charAt(0) != '#') bluelist.add(s.toLowerCase());
if (s.length() > 0 && s.charAt(0) != '#') this.bluelist.add(s.toLowerCase());
}
r.close();
} catch (final IOException e) {
@ -75,8 +76,9 @@ public class ContentTransformer extends AbstractTransformer implements Transform
}
}
@Override
public boolean isIdentityTransformer() {
return bluelist.isEmpty();
return this.bluelist.isEmpty();
}
private static char[] genBlueLetters(int length) {
@ -84,7 +86,7 @@ public class ContentTransformer extends AbstractTransformer implements Transform
length = length / 2;
if (length > 10) length = 7;
while (length-- > 0) {
bb.append((int)'X');
bb.append('X');
}
bb.append("</FONT> ");
final char[] result = bb.getChars();
@ -97,16 +99,17 @@ public class ContentTransformer extends AbstractTransformer implements Transform
}
private boolean bluelistHit(final char[] text) {
if (text == null || bluelist == null) return false;
if (text == null || this.bluelist == null) return false;
final String lc = new String(text).toLowerCase();
for (int i = 0; i < bluelist.size(); i++) {
if (lc.indexOf(bluelist.get(i)) >= 0) return true;
for (int i = 0; i < this.bluelist.size(); i++) {
if (lc.indexOf(this.bluelist.get(i)) >= 0) return true;
}
return false;
}
@Override
public char[] transformText(final char[] text) {
if (bluelist != null) {
if (this.bluelist != null) {
if (bluelistHit(text)) {
// System.out.println("FILTERHIT: " + text);
return genBlueLetters(text.length);

29
source/net/yacy/document/parser/html/TransformerWriter.java
Unescape View File

@ -82,7 +82,7 @@ public final class TransformerWriter extends Writer {
final Transformer transformer,
final boolean passbyIfBinarySuspect
) {
this(outStream, charSet, scraper, transformer, passbyIfBinarySuspect, 1024);
this(outStream, charSet, scraper, transformer, passbyIfBinarySuspect, 4096);
}
public TransformerWriter(
@ -115,9 +115,9 @@ public final class TransformerWriter extends Writer {
public static char[] genTag0raw(final String tagname, final boolean opening, final char[] tagopts) {
final CharBuffer bb = new CharBuffer(tagname.length() + tagopts.length + 3);
bb.append((int)'<');
bb.append('<');
if (!opening) {
bb.append((int)'/');
bb.append('/');
}
bb.append(tagname);
if (tagopts.length > 0) {
@ -125,7 +125,7 @@ public final class TransformerWriter extends Writer {
bb.append(tagopts);
// else bb.append((byte) 32).append(tagopts);
}
bb.append((int)'>');
bb.append('>');
final char[] result = bb.getChars();
try {
bb.close();
@ -137,15 +137,15 @@ public final class TransformerWriter extends Writer {
public static char[] genTag1raw(final String tagname, final char[] tagopts, final char[] text) {
final CharBuffer bb = new CharBuffer(2 * tagname.length() + tagopts.length + text.length + 5);
bb.append((int)'<').append(tagname);
bb.append('<').append(tagname);
if (tagopts.length > 0) {
// if (tagopts[0] == (byte) 32)
bb.append(tagopts);
// else bb.append((byte) 32).append(tagopts);
}
bb.append((int)'>');
bb.append('>');
bb.append(text);
bb.append((int)'<').append((int)'/').append(tagname).append((int)'>');
bb.append('<').append('/').append(tagname).append('>');
final char[] result = bb.getChars();
try {
bb.close();
@ -158,12 +158,12 @@ public final class TransformerWriter extends Writer {
public static char[] genTag0(final String tagname, final Properties tagopts, final char quotechar) {
final char[] tagoptsx = (tagopts.isEmpty()) ? null : genOpts(tagopts, quotechar);
final CharBuffer bb = new CharBuffer(tagname.length() + ((tagoptsx == null) ? 0 : (tagoptsx.length + 1)) + tagname.length() + 2);
bb.append((int)'<').append(tagname);
bb.append('<').append(tagname);
if (tagoptsx != null) {
bb.append(32);
bb.appendSpace();
bb.append(tagoptsx);
}
bb.append((int)'>');
bb.append('>');
final char[] result = bb.getChars();
try {
bb.close();
@ -176,7 +176,7 @@ public final class TransformerWriter extends Writer {
public static char[] genTag1(final String tagname, final Properties tagopts, final char[] text, final char quotechar) {
final char[] gt0 = genTag0(tagname, tagopts, quotechar);
final CharBuffer cb = new CharBuffer(gt0, gt0.length + text.length + tagname.length() + 3);
cb.append(text).append((int)'<').append((int)'/').append(tagname).append((int)'>');
cb.append(text).append('<').append('/').append(tagname).append('>');
final char[] result = cb.getChars();
try {
cb.close();
@ -193,9 +193,9 @@ public final class TransformerWriter extends Writer {
String key;
while (e.hasMoreElements()) {
key = (String) e.nextElement();
bb.append(32).append(key).append((int)'=').append((int)quotechar);
bb.appendSpace().append(key).append('=').append(quotechar);
bb.append(prop.getProperty(key));
bb.append((int)quotechar);
bb.append(quotechar);
}
final char[] result;
if (bb.length() > 0)
@ -530,12 +530,14 @@ public final class TransformerWriter extends Writer {
write(b, 0, b.length);
}
@Override
public void write(final char b[], final int off, final int len) throws IOException {
// System.out.println(UTF8.String(b, off, len));
if ((off | len | (b.length - (len + off)) | (off + len)) < 0) throw new IndexOutOfBoundsException();
for (int i = off ; i < (len - off) ; i++) this.write(b[i]);
}
@Override
public void flush() throws IOException {
// we cannot flush the current string this.buffer to prevent that
// the filter process is messed up
@ -544,6 +546,7 @@ public final class TransformerWriter extends Writer {
// if you want to flush all, call close() at end of writing;
}
@Override
public void close() throws IOException {
final char quotechar = (this.inSingleQuote) ? singlequote : doublequote;
if (this.buffer != null) {

3
source/net/yacy/document/parser/htmlParser.java
Unescape View File

@ -83,6 +83,7 @@ public class htmlParser extends AbstractParser implements Parser {
this.SUPPORTED_MIME_TYPES.add("text/csv");
}
@Override
public Document[] parse(
final MultiProtocolURI location,
final String mimeType,
@ -196,7 +197,7 @@ public class htmlParser extends AbstractParser implements Parser {
// parsing the content
final ContentScraper scraper = new ContentScraper(location);
final TransformerWriter writer = new TransformerWriter(null,null,scraper,null,false, sourceStream.available());
final TransformerWriter writer = new TransformerWriter(null,null,scraper,null,false, Math.max(4096, sourceStream.available()));
try {
FileUtils.copy(sourceStream, writer, c);
} catch (final IOException e) {

158
source/net/yacy/kelondro/blob/MapDataMining.java
Unescape View File

@ -44,7 +44,6 @@ import net.yacy.kelondro.logging.Log;
import net.yacy.kelondro.order.Base64Order;
import net.yacy.kelondro.order.ByteOrder;
import net.yacy.kelondro.order.CloneableIterator;
import net.yacy.kelondro.util.LookAheadIterator;
public class MapDataMining extends MapHeap {
@ -105,7 +104,7 @@ public class MapDataMining extends MapHeap {
}
// fill cluster and accumulator with values
if ((sortfields != null) || (longaccfields != null) || (floataccfields != null)) try {
if (sortfields != null || longaccfields != null || floataccfields != null) try {
final CloneableIterator<byte[]> it = super.keys(true, false);
byte[] mapnameb;
String cell;
@ -122,37 +121,49 @@ public class MapDataMining extends MapHeap {
}
if (map == null) break;
if (sortfields != null && cluster != null) for (int i = 0; i < sortfields.length; i++) {
cell = map.get(sortfields[i]);
if (cell != null) cluster[i].set(UTF8.String(mapnameb), ClusteredScoreMap.object2score(cell));
if (sortfields != null && cluster != null) {
for (int i = 0; i < sortfields.length; i++) {
cell = map.get(sortfields[i]);
if (cell != null) cluster[i].set(UTF8.String(mapnameb), ClusteredScoreMap.object2score(cell));
}
}
if (longaccfields != null && longaccumulator != null) for (int i = 0; i < longaccfields.length; i++) {
cell = map.get(longaccfields[i]);
valuel = 0;
if (cell != null) try {
valuel = Long.parseLong(cell);
longaccumulator[i] = Long.valueOf(longaccumulator[i].longValue() + valuel);
} catch (final NumberFormatException e) {}
if (longaccfields != null && longaccumulator != null) {
for (int i = 0; i < longaccfields.length; i++) {
cell = map.get(longaccfields[i]);
valuel = 0;
if (cell != null) try {
valuel = Long.parseLong(cell);
longaccumulator[i] = Long.valueOf(longaccumulator[i].longValue() + valuel);
} catch (final NumberFormatException e) {}
}
}
if (floataccfields != null && floataccumulator != null) for (int i = 0; i < floataccfields.length; i++) {
cell = map.get(floataccfields[i]);
valued = 0f;
if (cell != null) try {
valued = Float.parseFloat(cell);
floataccumulator[i] = new Float(floataccumulator[i].floatValue() + valued);
} catch (final NumberFormatException e) {}
if (floataccfields != null && floataccumulator != null) {
for (int i = 0; i < floataccfields.length; i++) {
cell = map.get(floataccfields[i]);
valued = 0f;
if (cell != null) try {
valued = Float.parseFloat(cell);
floataccumulator[i] = new Float(floataccumulator[i].floatValue() + valued);
} catch (final NumberFormatException e) {}
}
}
}
} catch (final IOException e) {}
// fill cluster
if (sortfields != null && cluster != null) for (int i = 0; i < sortfields.length; i++) this.sortClusterMap.put(sortfields[i], cluster[i]);
if (sortfields != null && cluster != null) {
for (int i = 0; i < sortfields.length; i++) this.sortClusterMap.put(sortfields[i], cluster[i]);
}
// fill acc map
if (longaccfields != null && longaccumulator != null) for (int i = 0; i < longaccfields.length; i++) this.accLong.put(longaccfields[i], longaccumulator[i]);
if (floataccfields != null && floataccumulator != null) for (int i = 0; i < floataccfields.length; i++) this.accFloat.put(floataccfields[i], floataccumulator[i]);
if (longaccfields != null && longaccumulator != null) {
for (int i = 0; i < longaccfields.length; i++) this.accLong.put(longaccfields[i], longaccumulator[i]);
}
if (floataccfields != null && floataccumulator != null) {
for (int i = 0; i < floataccfields.length; i++) this.accFloat.put(floataccfields[i], floataccumulator[i]);
}
}
@Override
@ -213,7 +224,7 @@ public class MapDataMining extends MapHeap {
float valued;
Long longaccumulator;
Float floataccumulator;
if (this.longaccfields != null)
if (this.longaccfields != null) {
for (final String longaccfield : this.longaccfields) {
value = map.get(longaccfield);
if (value != null) {
@ -228,7 +239,8 @@ public class MapDataMining extends MapHeap {
} catch (final NumberFormatException e) {}
}
}
if (this.floataccfields != null)
}
if (this.floataccfields != null) {
for (final String floataccfield : this.floataccfields) {
value = map.get(floataccfield);
if (value != null) {
@ -243,6 +255,7 @@ public class MapDataMining extends MapHeap {
} catch (final NumberFormatException e) {}
}
}
}
}
private void updateSortCluster(final String key, final Map<String, String> map) {
@ -283,23 +296,6 @@ public class MapDataMining extends MapHeap {
super.delete(key);
}
/* would be better but does not work (recursion)
@Override
public synchronized void delete(final byte[] key) throws IOException {
if (key == null) return;
// update elementCount
Map<String, String> map = super.remove(key);
if (map != null && (sortfields != null || longaccfields != null || floataccfields != null)) {
// update accumulators (subtract)
if ((longaccfields != null) || (floataccfields != null)) updateAcc(map, false);
// remove from sortCluster
if (sortfields != null) deleteSortCluster(UTF8.String(key));
}
}
*/
private void deleteSortCluster(final String key) {
if (key == null) return;
ScoreMap<String> cluster;
@ -327,34 +323,34 @@ public class MapDataMining extends MapHeap {
this.s = s;
}
@Override
public boolean hasNext() {
return this.s.hasNext();
}
@Override
public byte[] next() {
final String r = this.s.next();
if (r == null) return null;
return UTF8.getBytes(r);
}
@Override
public void remove() {
this.s.remove();
}
}
public synchronized mapIterator maps(final boolean up, final String field) {
return new mapIterator(keys(up, field));
@Override
public synchronized Iterator<Map.Entry<byte[], Map<String, String>>> entries(final String whereKey, final String isValue) throws IOException {
return super.entries(whereKey, isValue);
}
public synchronized mapIterator maps(final boolean up, final boolean rotating) throws IOException {
return new mapIterator(keys(up, rotating));
public synchronized Iterator<Map.Entry<byte[], Map<String, String>>> entries(final boolean up, final String field) {
return new MapIterator(keys(up, field), null, null);
}
public synchronized mapIterator maps(final boolean up, final boolean rotating, final byte[] firstKey, final byte[] secondKey) throws IOException {
return new mapIterator(keys(up, rotating, firstKey, secondKey));
}
public synchronized long getLongAcc(final String field) {
final Long accumulator = this.accLong.get(field);
if (accumulator == null) return -1;
@ -389,50 +385,33 @@ public class MapDataMining extends MapHeap {
super.close();
}
public class mapIterator extends LookAheadIterator<Map<String, String>> implements Iterator<Map<String, String>> {
// enumerates Map-Type elements
// the key is also included in every map that is returned; it's key is 'key'
/*
public byte[] lookupBy(
final String whereKey,
final String isValue
) {
private final Iterator<byte[]> keyIterator;
private mapIterator(final Iterator<byte[]> keyIterator) {
this.keyIterator = keyIterator;
}
public Map<String, String> next0() {
if (this.keyIterator == null) return null;
byte[] nextKey;
Map<String, String> map;
while (this.keyIterator.hasNext()) {
nextKey = this.keyIterator.next();
try {
map = get(nextKey, false);
} catch (final IOException e) {
Log.logWarning("MapDataMining", e.getMessage());
continue;
} catch (final RowSpaceExceededException e) {
Log.logException(e);
continue;
}
if (map == null) continue; // circumvention of a modified exception
map.put("key", UTF8.String(nextKey));
return map;
}
return null;
}
} // class mapIterator
}
*/
public static void main(final String[] args) {
try {
final MapDataMining db = new MapDataMining(new File("/tmp/MapDataMinig.test.db"), Word.commonHashLength, Base64Order.enhancedCoder, 1024 * 512, 500, new String[] {"X"}, new String[] {"X"}, new String[] {}, null);
final Map<String, String> m1 = new HashMap<String, String>(); m1.put("X", Long.toString(System.currentTimeMillis()));
File f = new File("/tmp/MapDataMinig.test.db");
f.delete();
final MapDataMining db = new MapDataMining(f, Word.commonHashLength, Base64Order.enhancedCoder, 1024 * 512, 500, new String[] {"X"}, new String[] {"X"}, new String[] {}, null);
final Map<String, String> m1 = new HashMap<String, String>();
long t = System.currentTimeMillis();
m1.put("X", Long.toString(t));
db.put("abcdefghijk1".getBytes(), m1);
final Map<String, String> m2 = new HashMap<String, String>(); m2.put("X", Long.toString(System.currentTimeMillis() - 1000));
final Map<String, String> m2 = new HashMap<String, String>();
m2.put("X", Long.toString(t - 1000));
db.put("abcdefghijk2".getBytes(), m2);
final Map<String, String> m3 = new HashMap<String, String>(); m3.put("X", Long.toString(System.currentTimeMillis() + 2000));
final Map<String, String> m3 = new HashMap<String, String>();
m3.put("X", Long.toString(t + 2000));
db.put("abcdefghijk3".getBytes(), m3);
// iterate the keys, sorted by field X in ascending order (must be: abcdefghijk2 - abcdefghijk1 - abcdefghijk3)
final Iterator<byte[]> i1 = db.keys(true, "X");
byte[] k;
while (i1.hasNext()) {
@ -440,11 +419,12 @@ public class MapDataMining extends MapHeap {
System.out.println(new String(k));
}
final Iterator<Map<String, String>> i2 = db.maps(false, "X");
Map<String, String> e;
// iterate the maps, sorted by field X in descending order (must be: abcdefghijk3 - abcdefghijk1 - abcdefghijk2)
final Iterator<Map.Entry<byte[], Map<String, String>>> i2 = db.entries(false, "X");
Map.Entry<byte[], Map<String, String>> e;
while (i2.hasNext()) {
e = i2.next();
System.out.println(e);
System.out.println(UTF8.String(e.getKey()) + ":" + e.getValue());
}
System.exit(0);

93
source/net/yacy/kelondro/blob/MapHeap.java
Unescape View File

@ -32,6 +32,7 @@ import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.AbstractMap;
import java.util.Collection;
import java.util.HashMap;
import java.util.Iterator;
@ -51,8 +52,8 @@ import net.yacy.kelondro.order.CloneableIterator;
import net.yacy.kelondro.order.NaturalOrder;
import net.yacy.kelondro.order.RotateIterator;
import net.yacy.kelondro.util.FileUtils;
import net.yacy.kelondro.util.LookAheadIterator;
import net.yacy.kelondro.util.MemoryControl;
import net.yacy.kelondro.util.kelondroException;
public class MapHeap implements Map<byte[], Map<String, String>> {
@ -85,6 +86,7 @@ public class MapHeap implements Map<byte[], Map<String, String>> {
* clears the content of the database
* @throws IOException
*/
@Override
public synchronized void clear() {
try {
this.blob.clear();
@ -168,6 +170,7 @@ public class MapHeap implements Map<byte[], Map<String, String>> {
}
}
@Override
public Map<String, String> put(final byte[] key, final Map<String, String> newMap) {
Map<String, String> v = null;
try {
@ -200,6 +203,7 @@ public class MapHeap implements Map<byte[], Map<String, String>> {
}
}
@Override
public Map<String, String> remove(final Object key) {
Map<String, String> v = null;
try {
@ -218,6 +222,7 @@ public class MapHeap implements Map<byte[], Map<String, String>> {
* @throws IOException
*/
@Override
public boolean containsKey(final Object k) {
if (!(k instanceof byte[])) return false;
assert k != null;
@ -241,6 +246,7 @@ public class MapHeap implements Map<byte[], Map<String, String>> {
return get(key, true);
}
@Override
public Map<String, String> get(final Object key) {
if (key == null) return null;
try {
@ -375,6 +381,7 @@ public class MapHeap implements Map<byte[], Map<String, String>> {
this.iterator = (rotating) ? new RotateIterator<byte[]>(i, secondKey, MapHeap.this.blob.size()) : i;
}
@Override
public byte[] next() {
return removeFillchar(this.iterator.next());
}
@ -399,23 +406,29 @@ public class MapHeap implements Map<byte[], Map<String, String>> {
}
}
public synchronized Iterator<Map.Entry<byte[], Map<String, String>>> entries(final String whereKey, final String isValue) throws IOException {
return new MapIterator(this.blob.keys(true, null), whereKey, isValue);
}
public synchronized MapIterator entries(final boolean up, final boolean rotating) throws IOException {
return new MapIterator(keys(up, rotating));
public synchronized Iterator<Map.Entry<byte[], Map<String, String>>> entries(final boolean up, final boolean rotating) throws IOException {
return new MapIterator(keys(up, rotating), null, null);
}
public synchronized MapIterator entries(final boolean up, final boolean rotating, final byte[] firstKey, final byte[] secondKey) throws IOException {
return new MapIterator(keys(up, rotating, firstKey, secondKey));
public synchronized Iterator<Map.Entry<byte[], Map<String, String>>> entries(final boolean up, final boolean rotating, final byte[] firstKey, final byte[] secondKey) throws IOException {
return new MapIterator(keys(up, rotating, firstKey, secondKey), null, null);
}
/**
* ask for the number of entries
* @return the number of entries in the table
*/
@Override
public synchronized int size() {
return (this.blob == null) ? 0 : this.blob.size();
}
@Override
public synchronized boolean isEmpty() {
return (this.blob == null) ? true : this.blob.isEmpty();
}
@ -435,48 +448,52 @@ public class MapHeap implements Map<byte[], Map<String, String>> {
public void finalize() {
close();
}
public class MapIterator implements Iterator<Map<String, String>> {
public class MapIterator extends LookAheadIterator<Map.Entry<byte[], Map<String, String>>> implements Iterator<Map.Entry<byte[], Map<String, String>>> {
// enumerates Map-Type elements
// the key is also included in every map that is returned; it's key is 'key'
Iterator<byte[]> keyIterator;
boolean finish;
private final Iterator<byte[]> keyIterator;
private final String whereKey, isValue;
public MapIterator(final Iterator<byte[]> keyIterator) {
MapIterator(final Iterator<byte[]> keyIterator, final String whereKey, final String isValue) {
this.keyIterator = keyIterator;
this.finish = false;
}
public boolean hasNext() {
return (!(this.finish)) && (this.keyIterator.hasNext());
this.whereKey = whereKey;
this.isValue = isValue;
}
public Map<String, String> next() {
byte[] nextKey = this.keyIterator.next();
if (nextKey == null) {
this.finish = true;
return null;
}
nextKey = normalizeKey(nextKey); // the key must be normalized because the keyIterator may iterate over not-normalized keys
try {
final Map<String, String> obj = get(nextKey, false);
if (obj == null) throw new kelondroException("no more elements available");
return obj;
} catch (final IOException e) {
this.finish = true;
return null;
} catch (final RowSpaceExceededException e) {
this.finish = true;
return null;
@Override
public Map.Entry<byte[], Map<String, String>> next0() {
if (this.keyIterator == null) return null;
byte[] nextKey;
Map<String, String> map;
while (this.keyIterator.hasNext()) {
nextKey = this.keyIterator.next();
try {
map = get(nextKey, false);
} catch (final IOException e) {
Log.logWarning("MapDataMining", e.getMessage());
continue;
} catch (final RowSpaceExceededException e) {
Log.logException(e);
continue;
}
if (map == null) continue; // circumvention of a modified exception
// check if the where case holds
if (this.whereKey != null && this.isValue != null) {
String v = map.get(this.whereKey);
if (v == null) continue;
if (!v.equals(this.isValue)) continue;
}
// produce entry
Map.Entry<byte[], Map<String, String>> entry = new AbstractMap.SimpleImmutableEntry<byte[], Map<String, String>>(nextKey, map);
return entry;
}
}
public void remove() {
throw new UnsupportedOperationException();
return null;
}
} // class mapIterator
@Override
public void putAll(final Map<? extends byte[], ? extends Map<String, String>> map) {
for (final Map.Entry<? extends byte[], ? extends Map<String, String>> me: map.entrySet()) {
try {
@ -489,6 +506,7 @@ public class MapHeap implements Map<byte[], Map<String, String>> {
}
}
@Override
public Set<byte[]> keySet() {
final TreeSet<byte[]> set = new TreeSet<byte[]>(this.blob.ordering());
try {
@ -498,16 +516,19 @@ public class MapHeap implements Map<byte[], Map<String, String>> {
return set;
}
@Override
public Collection<Map<String, String>> values() {
// this method shall not be used because it is not appropriate for this kind of data
throw new UnsupportedOperationException();
}
@Override
public Set<java.util.Map.Entry<byte[], Map<String, String>>> entrySet() {
// this method shall not be used because it is not appropriate for this kind of data
throw new UnsupportedOperationException();
}
@Override
public boolean containsValue(final Object value) {
// this method shall not be used because it is not appropriate for this kind of data
throw new UnsupportedOperationException();

6
source/net/yacy/kelondro/data/meta/DigestURI.java
Unescape View File

@ -213,8 +213,10 @@ public class DigestURI extends MultiProtocolURI implements Serializable {
int p = (this.host == null) ? -1 : this.host.lastIndexOf('.');
String dom = (p > 0) ? dom = this.host.substring(0, p) : "";
p = dom.lastIndexOf('.'); // locate subdomain
String subdom = "";
if (p > 0) {
final String subdom;
if (p <= 0) {
subdom = "";
} else {
subdom = dom.substring(0, p);
dom = dom.substring(p + 1);
}

15
source/net/yacy/kelondro/data/meta/URIMetadataRow.java
Unescape View File

@ -189,12 +189,15 @@ public class URIMetadataRow implements URIMetadata {
final float lat,
final float lon) {
final CharBuffer s = new CharBuffer(360);
s.append(url.toNormalform(false, true)).append(10);
s.append(dc_title).append(10);
s.append(dc_creator.length() > 80 ? dc_creator.substring(0, 80) : dc_creator).append(10);
s.append(dc_subject.length() > 120 ? dc_subject.substring(0, 120) : dc_subject).append(10);
s.append(dc_publisher).append(10);
if (lon == 0.0f && lat == 0.0f) s.append(10); else s.append(Float.toString(lat)).append(',').append(Float.toString(lon)).append(10);
s.append(url.toNormalform(false, true)).appendLF();
s.append(dc_title).appendLF();
if (dc_creator.length() > 80) s.append(dc_creator, 0, 80); else s.append(dc_creator);
s.appendLF();
if (dc_subject.length() > 120) s.append(dc_subject, 0, 120); else s.append(dc_subject);
s.appendLF();
if (dc_publisher.length() > 80) s.append(dc_publisher, 0, 80); else s.append(dc_publisher);
s.appendLF();
if (lon == 0.0f && lat == 0.0f) s.appendLF(); else s.append(Float.toString(lat)).append(',').append(Float.toString(lon)).appendLF();
return UTF8.getBytes(s.toString());
}

245
source/net/yacy/kelondro/io/CharBuffer.java
Unescape View File

@ -45,9 +45,9 @@ public final class CharBuffer extends Writer {
public CharBuffer() {
buffer = new char[10];
length = 0;
offset = 0;
this.buffer = new char[10];
this.length = 0;
this.offset = 0;
}
public CharBuffer(final int initLength) {
@ -57,44 +57,44 @@ public final class CharBuffer extends Writer {
}
public CharBuffer(final char[] bb) {
buffer = bb;
length = bb.length;
offset = 0;
this.buffer = bb;
this.length = bb.length;
this.offset = 0;
}
public CharBuffer(final char[] bb, final int initLength) {
this.buffer = new char[initLength];
System.arraycopy(bb, 0, buffer, 0, bb.length);
length = bb.length;
offset = 0;
System.arraycopy(bb, 0, this.buffer, 0, bb.length);
this.length = bb.length;
this.offset = 0;
}
public CharBuffer(final char[] bb, final int of, final int le) {
if (of * 2 > bb.length) {
buffer = new char[le];
System.arraycopy(bb, of, buffer, 0, le);
length = le;
offset = 0;
this.buffer = new char[le];
System.arraycopy(bb, of, this.buffer, 0, le);
this.length = le;
this.offset = 0;
} else {
buffer = bb;
length = le;
offset = of;
this.buffer = bb;
this.length = le;
this.offset = of;
}
}
public CharBuffer(final CharBuffer bb) {
buffer = bb.buffer;
length = bb.length;
offset = bb.offset;
this.buffer = bb.buffer;
this.length = bb.length;
this.offset = bb.offset;
}
public CharBuffer(final File f) throws IOException {
// initially fill the buffer with the content of a file
if (f.length() > Integer.MAX_VALUE) throw new IOException("file is too large for buffering");
length = 0;
buffer = new char[(int) f.length()*2];
offset = 0;
this.length = 0;
this.buffer = new char[(int) f.length()*2];
this.offset = 0;
FileReader fr = null;
try {
@ -114,52 +114,66 @@ public final class CharBuffer extends Writer {
public void clear() {
this.buffer = new char[0];
length = 0;
offset = 0;
this.length = 0;
this.offset = 0;
}
public int length() {
return length;
return this.length;
}
private void grow(int minSize) {
int newsize = buffer.length + 1024;
int newsize = this.buffer.length + 1024;
if (newsize < minSize) newsize = minSize+1;
char[] tmp = new char[newsize];
System.arraycopy(buffer, offset, tmp, 0, length);
buffer = tmp;
offset = 0;
System.arraycopy(this.buffer, this.offset, tmp, 0, this.length);
this.buffer = tmp;
this.offset = 0;
}
@Override
public void write(final int b) {
write((char)b);
}
public void write(final char b) {
if (offset + length + 1 > buffer.length) grow(offset + length + 1);
buffer[offset + length++] = b;
if (this.offset + this.length + 1 > this.buffer.length) grow(this.offset + this.length + 1);
this.buffer[this.offset + this.length++] = b;
}
@Override
public void write(final char[] bb) {
write(bb, 0, bb.length);
}
@Override
public void write(final char[] bb, final int of, final int le) {
if (offset + length + le > buffer.length) grow(offset + length + le);
System.arraycopy(bb, of, buffer, offset + length, le);
length += le;
if (this.offset + this.length + le > this.buffer.length) grow(this.offset + this.length + le);
System.arraycopy(bb, of, this.buffer, this.offset + this.length, le);
this.length += le;
}
private static final char SPACE = ' ';
private static final char CR = (char) 13;
private static final char LF = (char) 10;
public CharBuffer appendSpace() {
write(SPACE);
return this;
}
// do not use/implement the following method, a
// "overridden method is a bridge method"
// will occur
// public serverCharBuffer append(char b) {
// write(b);
// return this;
// }
public CharBuffer appendCR() {
write(CR);
return this;
}
public CharBuffer appendLF() {
write(LF);
return this;
}
public CharBuffer append(final int i) {
write((char) (i));
write((char) i);
return this;
}
@ -173,6 +187,7 @@ public final class CharBuffer extends Writer {
return this;
}
@Override
public CharBuffer append(final char c) {
write(c);
return this;
@ -180,39 +195,36 @@ public final class CharBuffer extends Writer {
public CharBuffer append(final String s) {
final char[] temp = new char[s.length()];
s.getChars(0, temp.length, temp, 0);
return append(temp);
s.getChars(0, temp.length, temp, 0);
write(temp);
return this;
}
public CharBuffer append(final String s, final int off, final int len) {
final char[] temp = new char[len];
s.getChars(off, (off + len), temp, 0);
return append(temp);
s.getChars(off, (off + len), temp, 0);
write(temp);
return this;
}
public CharBuffer append(final CharBuffer bb) {
return append(bb.buffer, bb.offset, bb.length);
write(bb.buffer, bb.offset, bb.length);
return this;
}
// public serverCharBuffer append(Object o) {
// if (o instanceof String) return append((String) o);
// if (o instanceof char[]) return append((char[]) o);
// return null;
// }
public char charAt(final int pos) {
if (pos < 0) throw new IndexOutOfBoundsException();
if (pos > length) throw new IndexOutOfBoundsException();
return buffer[offset + pos];
if (pos > this.length) throw new IndexOutOfBoundsException();
return this.buffer[this.offset + pos];
}
public void deleteCharAt(final int pos) {
if (pos < 0) return;
if (pos >= length) return;
if (pos == length - 1) {
length--;
if (pos >= this.length) return;
if (pos == this.length - 1) {
this.length--;
} else {
System.arraycopy(buffer, offset + pos + 1, buffer, offset + pos, length - pos - 1);
System.arraycopy(this.buffer, this.offset + pos + 1, this.buffer, this.offset + pos, this.length - pos - 1);
}
}
@ -225,20 +237,20 @@ public final class CharBuffer extends Writer {
}
public int indexOf(final char b, final int start) {
if (start >= length) return -1;
for (int i = start; i < length; i++) if (buffer[offset + i] == b) return i;
if (start >= this.length) return -1;
for (int i = start; i < this.length; i++) if (this.buffer[this.offset + i] == b) return i;
return -1;
}
public int indexOf(final char[] bs, final int start) {
if (start + bs.length > length) return -1;
loop: for (int i = start; i <= length - bs.length; i++) {
if (start + bs.length > this.length) return -1;
loop: for (int i = start; i <= this.length - bs.length; i++) {
// first test only first char
if (buffer[offset + i] != bs[0]) continue loop;
if (this.buffer[this.offset + i] != bs[0]) continue loop;
// then test all remaining char
for (int j = 1; j < bs.length; j++) {
if (buffer[offset + i + j] != bs[j]) continue loop;
if (this.buffer[this.offset + i + j] != bs[j]) continue loop;
}
// found hit
@ -278,14 +290,14 @@ public final class CharBuffer extends Writer {
}
public int lastIndexOf(final char b) {
for (int i = length - 1; i >= 0; i--) if (buffer[offset + i] == b) return i;
for (int i = this.length - 1; i >= 0; i--) if (this.buffer[this.offset + i] == b) return i;
return -1;
}
public boolean startsWith(final char[] bs) {
if (length < bs.length) return false;
if (this.length < bs.length) return false;
for (int i = 0; i < bs.length; i++) {
if (buffer[offset + i] != bs[i]) return false;
if (this.buffer[this.offset + i] != bs[i]) return false;
}
return true;
}
@ -295,45 +307,45 @@ public final class CharBuffer extends Writer {
}
public char[] getChars(final int start) {
return getChars(start, length);
return getChars(start, this.length);
}
public char[] getChars(final int start, final int end) {
// start is inclusive, end is exclusive
if (end > length) throw new IndexOutOfBoundsException("getBytes: end > length");
if (start > length) throw new IndexOutOfBoundsException("getBytes: start > length");
if (end > this.length) throw new IndexOutOfBoundsException("getBytes: end > length");
if (start > this.length) throw new IndexOutOfBoundsException("getBytes: start > length");
final char[] tmp = new char[end - start];
System.arraycopy(buffer, offset + start, tmp, 0, end - start);
System.arraycopy(this.buffer, this.offset + start, tmp, 0, end - start);
return tmp;
}
public byte[] getBytes() {
return UTF8.getBytes(new String(buffer, offset, length));
return UTF8.getBytes(new String(this.buffer, this.offset, this.length));
}
public CharBuffer trim(final int start) {
// the end value is outside (+1) of the wanted target array
if (start > length) throw new IndexOutOfBoundsException("trim: start > length");
offset = offset + start;
length = length - start;
if (start > this.length) throw new IndexOutOfBoundsException("trim: start > length");
this.offset = this.offset + start;
this.length = this.length - start;
return this;
}
public CharBuffer trim(final int start, final int end) {
// the end value is outside (+1) of the wanted target array
if (start > length) throw new IndexOutOfBoundsException("trim: start > length");
if (end > length) throw new IndexOutOfBoundsException("trim: end > length");
if (start > this.length) throw new IndexOutOfBoundsException("trim: start > length");
if (end > this.length) throw new IndexOutOfBoundsException("trim: end > length");
if (start > end) throw new IndexOutOfBoundsException("trim: start > end");
offset = offset + start;
length = end - start;
this.offset = this.offset + start;
this.length = end - start;
return this;
}
public CharBuffer trim() {
int l = 0;
while ((l < length) && (buffer[offset + l] <= ' ')) l++;
int r = length;
while ((r > 0) && (buffer[offset + r - 1] <= ' ')) r--;
while ((l < this.length) && (this.buffer[this.offset + l] <= ' ')) l++;
int r = this.length;
while ((r > 0) && (this.buffer[this.offset + r - 1] <= ' ')) r--;
if (l > r) r = l;
return trim(l, r);
}
@ -342,12 +354,12 @@ public final class CharBuffer extends Writer {
// returns true, if trim() would result in an empty serverByteBuffer
if (includeNonLetterBytes) {
char b;
for (int i = 0; i < length; i++) {
b = buffer[offset + i];
for (int i = 0; i < this.length; i++) {
b = this.buffer[this.offset + i];
if (((b >= '0') && (b <= '9')) || ((b >= 'A') && (b <= 'Z')) || ((b >= 'a') && (b <= 'z'))) return false;
}
} else {
for (int i = 0; i < length; i++) if (buffer[offset + i] > 32) return false;
for (int i = 0; i < this.length; i++) if (this.buffer[this.offset + i] > 32) return false;
}
return true;
}
@ -356,86 +368,87 @@ public final class CharBuffer extends Writer {
// returns number of whitespace char at the beginning of text
if (includeNonLetterBytes) {
char b;
for (int i = 0; i < length; i++) {
b = buffer[offset + i];
for (int i = 0; i < this.length; i++) {
b = this.buffer[this.offset + i];
if (((b >= '0') && (b <= '9')) || ((b >= 'A') && (b <= 'Z')) || ((b >= 'a') && (b <= 'z'))) return i;
}
} else {
for (int i = 0; i < length; i++) if (buffer[offset + i] > 32) return i;
for (int i = 0; i < this.length; i++) if (this.buffer[this.offset + i] > 32) return i;
}
return length;
return this.length;
}
public int whitespaceEnd(final boolean includeNonLetterBytes) {
// returns position of whitespace at the end of text
if (includeNonLetterBytes) {
char b;
for (int i = length - 1; i >= 0; i--) {
b = buffer[offset + i];
for (int i = this.length - 1; i >= 0; i--) {
b = this.buffer[this.offset + i];
if (((b >= '0') && (b <= '9')) || ((b >= 'A') && (b <= 'Z')) || ((b >= 'a') && (b <= 'z'))) return i + 1;
}
} else {
for (int i = length - 1; i >= 0; i--) if (buffer[offset + i] > 32) return i + 1;
for (int i = this.length - 1; i >= 0; i--) if (this.buffer[this.offset + i] > 32) return i + 1;
}
return 0;
}
@Override
public String toString() {
return new String(buffer, offset, length);
return new String(this.buffer, this.offset, this.length);
}
public String toString(final int left, final int rightbound) {
return new String(buffer, offset + left, rightbound - left);
return new String(this.buffer, this.offset + left, rightbound - left);
}
public Properties propParser() {
// extract a=b or a="b" - relations from the buffer
int pos = offset;
int pos = this.offset;
int start;
String key;
final Properties p = new Properties();
// eat up spaces at beginning
while ((pos < length) && (buffer[pos] <= 32)) pos++;
while (pos < length) {
while ((pos < this.length) && (this.buffer[pos] <= 32)) pos++;
while (pos < this.length) {
// pos is at start of next key
start = pos;
while ((pos < length) && (buffer[pos] != equal)) pos++;
if (pos >= length) break; // this is the case if we found no equal
key = new String(buffer, start, pos - start).trim().toLowerCase();
while ((pos < this.length) && (this.buffer[pos] != equal)) pos++;
if (pos >= this.length) break; // this is the case if we found no equal
key = new String(this.buffer, start, pos - start).trim().toLowerCase();
// we have a key
pos++;
// find start of value
while ((pos < length) && (buffer[pos] <= 32)) pos++;
while ((pos < this.length) && (this.buffer[pos] <= 32)) pos++;
// doublequotes are obligatory. However, we want to be fuzzy if they
// are ommittet
if (pos >= length) {
if (pos >= this.length) {
// error case: input ended too early
break;
} else if (buffer[pos] == doublequote) {
} else if (this.buffer[pos] == doublequote) {
// search next doublequote
pos++;
start = pos;
while ((pos < length) && (buffer[pos] != doublequote)) pos++;
if (pos >= length) break; // this is the case if we found no parent doublequote
p.setProperty(key, new String(buffer, start, pos - start).trim());
while ((pos < this.length) && (this.buffer[pos] != doublequote)) pos++;
if (pos >= this.length) break; // this is the case if we found no parent doublequote
p.setProperty(key, new String(this.buffer, start, pos - start).trim());
pos++;
} else if (buffer[pos] == singlequote) {
} else if (this.buffer[pos] == singlequote) {
// search next singlequote
pos++;
start = pos;
while ((pos < length) && (buffer[pos] != singlequote)) pos++;
if (pos >= length) break; // this is the case if we found no parent singlequote
p.setProperty(key, new String(buffer, start, pos - start).trim());
while ((pos < this.length) && (this.buffer[pos] != singlequote)) pos++;
if (pos >= this.length) break; // this is the case if we found no parent singlequote
p.setProperty(key, new String(this.buffer, start, pos - start).trim());
pos++;
} else {
// search next whitespace
start = pos;
while ((pos < length) && (buffer[pos] > 32)) pos++;
p.setProperty(key, new String(buffer, start, pos - start).trim());
while ((pos < this.length) && (this.buffer[pos] > 32)) pos++;
p.setProperty(key, new String(this.buffer, start, pos - start).trim());
}
// pos should point now to a whitespace: eat up spaces
while ((pos < length) && (buffer[pos] <= 32)) pos++;
while ((pos < this.length) && (this.buffer[pos] <= 32)) pos++;
// go on with next loop
}
return p;
@ -475,10 +488,12 @@ public final class CharBuffer extends Writer {
return newbuf;
}
@Override
public void close() throws IOException {
buffer = null; // assist with garbage collection
this.buffer = null; // assist with garbage collection
}
@Override
public void flush() throws IOException {
// TODO Auto-generated method stub
}

26
source/net/yacy/peers/SeedDB.java
Unescape View File

@ -274,22 +274,27 @@ public final class SeedDB implements AlternativeDomainNames {
this.myBotIDs.add(name + ".yacy");
}
@Override
public String myAlternativeAddress() {
return mySeed().getName() + ".yacy";
}
@Override
public String myIP() {
return mySeed().getIP();
}
@Override
public int myPort() {
return mySeed().getPort();
}
@Override
public String myName() {
return this.mySeed.getName();
}
@Override
public String myID() {
return this.mySeed.hash;
}
@ -697,7 +702,7 @@ public final class SeedDB implements AlternativeDomainNames {
final HandleSet badPeerHashes = new HandleSet(12, Base64Order.enhancedCoder, 0);
if (lookupConnected) {
// enumerate the cache and simultanous insert values
// enumerate the cache and simultaneously insert values
final Iterator<Seed> e = seedsConnected(true, false, null, (float) 0.0);
while (e.hasNext()) {
seed = e.next();
@ -1004,7 +1009,7 @@ public final class SeedDB implements AlternativeDomainNames {
private class seedEnum implements Iterator<Seed> {
private MapDataMining.mapIterator it;
private Iterator<Map.Entry<byte[], Map<String, String>>> it;
private Seed nextSeed;
private final MapDataMining database;
private float minVersion;
@ -1013,7 +1018,7 @@ public final class SeedDB implements AlternativeDomainNames {
this.database = database;
this.minVersion = minVersion;
try {
this.it = (firstKey == null) ? database.maps(up, rot) : database.maps(up, rot, firstKey, secondKey);
this.it = (firstKey == null) ? database.entries(up, rot) : database.entries(up, rot, firstKey, secondKey);
float version;
while (true) {
this.nextSeed = internalNext();
@ -1039,7 +1044,7 @@ public final class SeedDB implements AlternativeDomainNames {
private seedEnum(final boolean up, final String field, final MapDataMining database) {
this.database = database;
try {
this.it = database.maps(up, field);
this.it = database.entries(up, field);
this.nextSeed = internalNext();
} catch (final kelondroException e) {
Log.logException(e);
@ -1058,7 +1063,7 @@ public final class SeedDB implements AlternativeDomainNames {
private Seed internalNext() {
if (this.it == null || !(this.it.hasNext())) return null;
try {
Map<String, String> dna0;
Map.Entry<byte[], Map<String, String>> dna0;
ConcurrentHashMap<String, String> dna;
while (this.it.hasNext()) {
try {
@ -1069,15 +1074,14 @@ public final class SeedDB implements AlternativeDomainNames {
}
assert dna0 != null;
if (dna0 == null) continue;
if (dna0 instanceof ConcurrentHashMap) {
dna = (ConcurrentHashMap<String, String>) dna0;
if (dna0.getValue() instanceof ConcurrentHashMap) {
dna = (ConcurrentHashMap<String, String>) dna0.getValue();
} else {
dna = new ConcurrentHashMap<String, String>();
dna.putAll(dna0);
dna.putAll(dna0.getValue());
}
final String hash = dna.remove("key");
//assert hash != null;
if (hash == null) continue; // bad seed
if (dna0.getKey() == null) continue; // bad seed
final String hash = UTF8.String(dna0.getKey());
return new Seed(hash, dna);
}
return null;

15
source/net/yacy/yacy.java
Unescape View File

@ -47,6 +47,7 @@ import java.util.zip.ZipOutputStream;
import net.yacy.cora.date.GenericFormatter;
import net.yacy.cora.document.ASCII;
import net.yacy.cora.document.UTF8;
import net.yacy.cora.protocol.ClientIdentification;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.cora.protocol.http.HTTPClient;
@ -881,15 +882,15 @@ public final class yacy {
final File dbFile = new File(yacyDBPath,dbFileName);
final MapDataMining db = new MapDataMining(dbFile, Word.commonHashLength, Base64Order.enhancedCoder, 1024 * 512, 500, SeedDB.sortFields, SeedDB.longaccFields, SeedDB.doubleaccFields, null);
MapDataMining.mapIterator it;
it = db.maps(true, false);
Iterator<Map.Entry<byte[], Map<String, String>>> it;
it = db.entries(true, false);
while (it.hasNext()) {
final Map<String, String> dna = it.next();
String peerHash = dna.get("key");
final Map.Entry<byte[], Map<String, String>> dna = it.next();
String peerHash = UTF8.String(dna.getKey());
if (peerHash.length() < Word.commonHashLength) {
final String peerName = dna.get("Name");
final String peerIP = dna.get("IP");
final String peerPort = dna.get("Port");
final String peerName = dna.getValue().get("Name");
final String peerIP = dna.getValue().get("IP");
final String peerPort = dna.getValue().get("Port");
while (peerHash.length() < Word.commonHashLength) { peerHash = peerHash + "_"; }
System.err.println("Invalid Peer-Hash found in '" + dbFileName + "': " + peerName + ":" + peerHash + ", http://" + peerIP + ":" + peerPort);

Write Preview
Loading…
Cancel
Save