From 89e13fa34ee4322f2b2911d8a93a8c6142b6338e Mon Sep 17 00:00:00 2001 From: Michael Peter Christen Date: Wed, 21 May 2014 15:31:47 +0200 Subject: [PATCH 1/3] fixed bug in test function --- htroot/IndexControlRWIs_p.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/htroot/IndexControlRWIs_p.java b/htroot/IndexControlRWIs_p.java index 0c8d08147..8cf254a53 100644 --- a/htroot/IndexControlRWIs_p.java +++ b/htroot/IndexControlRWIs_p.java @@ -283,7 +283,7 @@ public class IndexControlRWIs_p { Reference iEntry; while (urlIter.hasNext()) { iEntry = urlIter.next(); - if (segment.fulltext().getLoadTime(ASCII.String(iEntry.urlhash())) >= 0) { + if (segment.fulltext().getLoadTime(ASCII.String(iEntry.urlhash())) < 0) { try { unknownURLEntries.put(iEntry.urlhash()); } catch (final SpaceExceededException e) { From 6634b5b7374ccb27b24ec980b85973df342b9a97 Mon Sep 17 00:00:00 2001 From: Michael Peter Christen Date: Wed, 21 May 2014 18:20:16 +0200 Subject: [PATCH 2/3] debug code for index distribution testing --- htroot/IndexControlRWIs_p.java | 3 ++- .../kelondro/rwi/ReferenceContainerCache.java | 15 +++++++++++++++ source/net/yacy/peers/Protocol.java | 18 +++++++++--------- source/net/yacy/search/index/Fulltext.java | 2 +- 4 files changed, 27 insertions(+), 11 deletions(-) diff --git a/htroot/IndexControlRWIs_p.java b/htroot/IndexControlRWIs_p.java index 8cf254a53..8afd32597 100644 --- a/htroot/IndexControlRWIs_p.java +++ b/htroot/IndexControlRWIs_p.java @@ -283,7 +283,8 @@ public class IndexControlRWIs_p { Reference iEntry; while (urlIter.hasNext()) { iEntry = urlIter.next(); - if (segment.fulltext().getLoadTime(ASCII.String(iEntry.urlhash())) < 0) { + long loadTime = segment.fulltext().getLoadTime(ASCII.String(iEntry.urlhash())); + if (loadTime < 0) { try { unknownURLEntries.put(iEntry.urlhash()); } catch (final SpaceExceededException e) { diff --git a/source/net/yacy/kelondro/rwi/ReferenceContainerCache.java b/source/net/yacy/kelondro/rwi/ReferenceContainerCache.java index b94784d72..671ce1bac 100644 --- a/source/net/yacy/kelondro/rwi/ReferenceContainerCache.java +++ b/source/net/yacy/kelondro/rwi/ReferenceContainerCache.java @@ -36,6 +36,7 @@ import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import net.yacy.cora.document.encoding.ASCII; import net.yacy.cora.order.ByteOrder; import net.yacy.cora.order.CloneableIterator; import net.yacy.cora.sorting.Rating; @@ -66,6 +67,7 @@ public final class ReferenceContainerCache exte private final ContainerOrder containerOrder; private ConcurrentHashMap> cache; + /** * open an existing heap file in undefined mode * after this a initialization should be made to use the heap: @@ -630,4 +632,17 @@ public final class ReferenceContainerCache exte } } + /** + * return a string with the key as a list + */ + @Override + public String toString() { + StringBuilder sb = new StringBuilder(cache.size() * 13 + 2); + sb.append('['); + for (ByteArray k: cache.keySet()) { + sb.append(ASCII.String(k.asBytes())).append(','); + } + if (sb.length() == 1) sb.append(']'); else sb.setCharAt(sb.length() - 1, ']'); + return sb.toString(); + } } diff --git a/source/net/yacy/peers/Protocol.java b/source/net/yacy/peers/Protocol.java index 832fd73bf..7790ad0bc 100644 --- a/source/net/yacy/peers/Protocol.java +++ b/source/net/yacy/peers/Protocol.java @@ -1401,14 +1401,6 @@ public final class Protocol { if ( uhss == null ) { return "no unknownURL tag in response"; } - EventChannel - .channels(EventChannel.DHTSEND) - .addMessage( - new RSSMessage( - "Sent " + indexes.size() + " RWIs to " + targetSeed.getName(), - "", - targetSeed.hash)); - uhss = uhss.trim(); if ( uhss.isEmpty() || uhss.equals(",") ) { return null; @@ -1419,6 +1411,14 @@ public final class Protocol { return null; } // all url's known + EventChannel + .channels(EventChannel.DHTSEND) + .addMessage( + new RSSMessage( + "Sent " + indexes.size() + " RWIs " + indexes.toString() + " to " + targetSeed.getName() + "/[" + targetSeed.hash + "], " + uhs.length + " URLs there unknown", + "", + targetSeed.hash)); + in = transferURL(targetSeed, uhs, urlRefs, segment, gzipBody, timeout); if ( in == null ) { @@ -1435,7 +1435,7 @@ public final class Protocol { } EventChannel.channels(EventChannel.DHTSEND).addMessage( new RSSMessage( - "Sent " + uhs.length + " URLs to peer " + targetSeed.getName(), + "Sent " + uhs.length + " URLs to peer " + targetSeed.getName()+ "/[" + targetSeed.hash + "]", "", targetSeed.hash)); diff --git a/source/net/yacy/search/index/Fulltext.java b/source/net/yacy/search/index/Fulltext.java index e1407183a..cd05bdabb 100644 --- a/source/net/yacy/search/index/Fulltext.java +++ b/source/net/yacy/search/index/Fulltext.java @@ -494,7 +494,7 @@ public final class Fulltext { if (urlHash == null) return -1l; try { SolrConnector.Metadata md = this.getDefaultConnector().getMetadata(urlHash); - if (md == null) return -1; + if (md == null) return -1l; return md.date; } catch (final Throwable e) { ConcurrentLog.logException(e); From 3dc5fb00506af45ced6b21fad4a9e1c29a291718 Mon Sep 17 00:00:00 2001 From: Michael Peter Christen Date: Wed, 21 May 2014 18:37:52 +0200 Subject: [PATCH 3/3] fix for operator precedence bug (cast binds stronger than bitwise AND) in peer hash hashing. This should not change anything if java casts long to int by masking with 0xFFFFFFFFL but you never know. The important thing is, that the hashCode() should not return numbers that have the same order as the hash code order because hashing of seeds is used to remove the order in some places. --- source/net/yacy/peers/Seed.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/net/yacy/peers/Seed.java b/source/net/yacy/peers/Seed.java index 44f43b4f7..5581dd65b 100644 --- a/source/net/yacy/peers/Seed.java +++ b/source/net/yacy/peers/Seed.java @@ -1204,7 +1204,7 @@ public class Seed implements Cloneable, Comparable, Comparator @Override public int hashCode() { - return (int) (Base64Order.enhancedCoder.cardinal(this.hash) & (Integer.MAX_VALUE)); + return (int) ((Base64Order.enhancedCoder.cardinal(this.hash) & (Integer.MAX_VALUE))); } @Override