diff --git a/htroot/News.html b/htroot/News.html index 1c2b83507..320717d3d 100644 --- a/htroot/News.html +++ b/htroot/News.html @@ -43,6 +43,10 @@ A change in the personal profile will create a news entry. You can see recently made changes of profile entries on the Network page, where that profile change is visualized with a '*' beside the 'P' (profile) - selector. +
  • + Publishing of added or modified translation for the user interface. Other peers may include it in their local translation list. + To publish a translation, use the integrated translation editor to add a translation and publish it afterwards. +

    • More news services will follow. diff --git a/htroot/TransNews_p.html b/htroot/TransNews_p.html index d5b755c02..7e69deedb 100644 --- a/htroot/TransNews_p.html +++ b/htroot/TransNews_p.html @@ -6,7 +6,7 @@ #%env/templates/header.template%# - + #%env/templates/submenuComputation.template%#

    Translation News for Language #[currentlang]#

    diff --git a/htroot/TransNews_p.java b/htroot/TransNews_p.java index 5a2ab53aa..98c432afd 100644 --- a/htroot/TransNews_p.java +++ b/htroot/TransNews_p.java @@ -42,7 +42,6 @@ import net.yacy.server.serverObjects; import net.yacy.server.serverSwitch; import net.yacy.utils.crypt; import net.yacy.utils.translation.TranslationManager; -import net.yacy.utils.translation.TranslatorXliff; public class TransNews_p { @@ -90,32 +89,13 @@ public class TransNews_p { continue; } if (NewsPool.CATEGORY_TRANSLATION_ADD.equals(rtmp.category())) { - //String tmplng = rtmp.attribute("language", null); + String tmplng = rtmp.attribute("language", null); String tmpfile = rtmp.attribute("file", null); String tmpsource = rtmp.attribute("source", null); - String tmptarget = rtmp.attribute("target", null); + //String tmptarget = rtmp.attribute("target", null); - if (sb.peers.mySeed().hash.equals(rtmp.originator())) { - /* - if (tmplng != null && tmplng.equals(currentlang)) { - sendit = false; - break; - }*/ - if (tmpfile != null && tmpfile.equals(file)) { - sendit = false; - break; - } - if (tmpsource != null && tmpsource.equals(sourcetxt)) { - sendit = false; - break; - } - if (tmptarget != null && tmptarget.equals(targettxt)) { - sendit = false; - break; - } - } // if news with file and source exist (maybe from other peer) - skip sending another msg (to avoid confusion) - if ((tmpfile != null && tmpfile.equals(file)) + if ((tmplng != null && tmplng.equals(currentlang)) && (tmpfile != null && tmpfile.equals(file)) && (tmpsource != null && tmpsource.equals(sourcetxt))) { sendit = false; break; @@ -189,8 +169,8 @@ public class TransNews_p { final HashMap positiveHashes = new HashMap(); // a mapping from an url hash to Integer (count of votes) accumulateVotes(sb, negativeHashes, positiveHashes, NewsPool.INCOMING_DB); final ScoreMap ranking = new ConcurrentScoreMap(); // score cluster for url hashes - final HashMap Translation = new HashMap(); // a mapping from an url hash to a kelondroRow.Entry with display properties - accumulateTranslations(sb, Translation, ranking, negativeHashes, positiveHashes, NewsPool.INCOMING_DB); + final HashMap translation = new HashMap(); // a mapping from an url hash to a kelondroRow.Entry with display properties + accumulateTranslations(sb, translation, ranking, negativeHashes, positiveHashes, NewsPool.INCOMING_DB); // read out translation-news array and create property entries final Iterator k = ranking.keys(false); @@ -207,7 +187,7 @@ public class TransNews_p { continue; } - row = Translation.get(refid); + row = translation.get(refid); if (row == null) { continue; } @@ -220,6 +200,7 @@ public class TransNews_p { continue; } + if (!lang.equals(currentlang)) continue; String existingtarget = null; //transMgr.getTranslation(filename, source); Map tmpMap = localTrans.get(filename); diff --git a/htroot/env/templates/submenuComputation.template b/htroot/env/templates/submenuComputation.template index b6062bcf8..5a112d2ce 100644 --- a/htroot/env/templates/submenuComputation.template +++ b/htroot/env/templates/submenuComputation.template @@ -33,6 +33,7 @@ diff --git a/htroot/yacy/search.java b/htroot/yacy/search.java index 0c831c5dd..c208e4282 100644 --- a/htroot/yacy/search.java +++ b/htroot/yacy/search.java @@ -380,7 +380,9 @@ public final class search { // prepare reference hints final long timer = System.currentTimeMillis(); - final ScoreMap topicNavigator = sb.index.connectedRWI() ? theSearch.getTopics(5, 100) : new ConcurrentScoreMap(); + //final ScoreMap topicNavigator = sb.index.connectedRWI() ? theSearch.getTopics(5, 100) : new ConcurrentScoreMap(); + final ScoreMap topicNavigator = theSearch.getTopics(); // as there is currently no index interaction in getTopics(), we can use it by default + final StringBuilder refstr = new StringBuilder(6000); final Iterator navigatorIterator = topicNavigator.keys(false); int i = 0; diff --git a/locales/fr.lng b/locales/fr.lng index 912f628f0..9d81c49c6 100644 --- a/locales/fr.lng +++ b/locales/fr.lng @@ -1212,8 +1212,14 @@ Category==Categorie Received==Reçu Distributed==Distribué Attributes==Attributs -"#(page)#::Process Selected News::Delete Selected News::Abort Publication of Selected News::Delete Selected News#(/page)#"==#(page)#::Traiter les nouvelles sélectionnées::Supprimer les nouvelles sélectionnées::Annuler la publication des nouvelles sélectionnées::Supprimer les nouvelles sélectionées#(/page)# -"#(page)#::Process All News::Delete All News::Abort Publication of All News::Delete All News#(/page)#"==#(page)#::Traiter toutes les nouvelles::Supprimer toutes les nouvelles::Annuler la publication de toutes les nouvelles::Supprimer toutes les nouvelles#(/page)# +Process Selected News==Traiter les nouvelles sélectionnées +Delete Selected News==Supprimer les nouvelles sélectionnées +Abort Publication of Selected News==Annuler la publication des nouvelles sélectionnées +Delete Selected News==Supprimer les nouvelles sélectionées +Process All News==Traiter toutes les nouvelles +Delete All News==Supprimer toutes les nouvelles +Abort Publication of All News==Annuler la publication de toutes les nouvelles +Delete All News==Supprimer toutes les nouvelles #----------------------------- #File: Performance_p.html diff --git a/locales/master.lng.xlf b/locales/master.lng.xlf index 47d038e9a..e31b52804 100644 --- a/locales/master.lng.xlf +++ b/locales/master.lng.xlf @@ -5966,6 +5966,21 @@ profile entries on the Network page, where that profile change is visualized with a '*' beside the 'P' (profile) - selector. + + Publishing of added or modified translation for the user interface. + + + Other peers may include it in their local translation list. + + + To publish a translation, use the integrated + + + translation editor + + + to add a translation and publish it afterwards. + Above you can see four menues: @@ -6008,6 +6023,24 @@ Attributes + + Process Selected News + + + Delete Selected News + + + Abort Publication of Selected News + + + Process All News + + + Delete All News + + + Abort Publication of All News + "#(page)#::Process Selected News::Delete Selected News::Abort Publication of Selected News::Delete Selected News#(/page)#" @@ -10124,6 +10157,9 @@ >Local Peer Wiki< + + UI Translations + diff --git a/source/net/yacy/cora/sorting/ConcurrentScoreMap.java b/source/net/yacy/cora/sorting/ConcurrentScoreMap.java index 95b2ce3b8..77304e576 100644 --- a/source/net/yacy/cora/sorting/ConcurrentScoreMap.java +++ b/source/net/yacy/cora/sorting/ConcurrentScoreMap.java @@ -133,7 +133,10 @@ public class ConcurrentScoreMap extends AbstractScoreMap implements ScoreM if (obj == null) return; // use atomic operations - this.map.putIfAbsent(obj, new AtomicLong(0)); + final AtomicLong old = this.map.putIfAbsent(obj, new AtomicLong(0)); + // adjust overall counter if value replaced + if (old != null) this.gcount -= old.longValue(); // must use old befor setting a new value (it's a object reference) + this.map.get(obj).set(newScore); // increase overall counter diff --git a/source/net/yacy/peers/NewsPool.java b/source/net/yacy/peers/NewsPool.java index 1f13232f8..a16a8bcce 100644 --- a/source/net/yacy/peers/NewsPool.java +++ b/source/net/yacy/peers/NewsPool.java @@ -325,6 +325,8 @@ public class NewsPool { if (this.newsDB.get(record.id()) == null) { this.incomingNews.push(record); // we want to see our own news.. this.outgoingNews.push(record); // .. and put it on the publishing list + } else { + ConcurrentLog.info("NewsPool", "publishing of news aborted, news with same id (time + originator) exists id=" + record.id()); } } catch (final Exception e) { ConcurrentLog.logException(e); diff --git a/source/net/yacy/peers/PeerActions.java b/source/net/yacy/peers/PeerActions.java index b249d5c95..46bb263aa 100644 --- a/source/net/yacy/peers/PeerActions.java +++ b/source/net/yacy/peers/PeerActions.java @@ -30,7 +30,6 @@ import net.yacy.cora.document.encoding.ASCII; import net.yacy.cora.document.feed.RSSMessage; import net.yacy.cora.storage.ConcurrentARC; import net.yacy.kelondro.util.MapTools; -import net.yacy.peers.operation.yacyVersion; public class PeerActions { @@ -261,7 +260,7 @@ public class PeerActions { final String cre1 = MapTools.string2map(decodedString, ",").get("cre"); final String cre2 = MapTools.string2map(record.toString(), ",").get("cre"); if ((cre1 == null) || (cre2 == null) || (!(cre1.equals(cre2)))) { - System.out.println("### ERROR - cre are not equal: cre1=" + cre1 + ", cre2=" + cre2); + Network.log.warn("processPeerArrival: ### ERROR - message creation date verification not equal: cre1=" + cre1 + ", cre2=" + cre2); return; } try { diff --git a/source/net/yacy/search/query/SearchEvent.java b/source/net/yacy/search/query/SearchEvent.java index 11a322a4f..a291451d4 100644 --- a/source/net/yacy/search/query/SearchEvent.java +++ b/source/net/yacy/search/query/SearchEvent.java @@ -1329,10 +1329,32 @@ public final class SearchEvent { public long getSnippetComputationTime() { return this.snippetComputationAllTime; } - - public ScoreMap getTopicNavigator(final int count ) { + + /** + * Get topics in a ScoreMap if config allows topic navigator + * (the topics are filtered by badwords, stopwords and words included in the query) + * + * @param count max number of topics returned + * @return ScoreMap with max number of topics or null if + */ + public ScoreMap getTopicNavigator(final int count) { if (this.topicNavigatorCount > 0 && count >= 0) { //topicNavigatorCount set during init, 0=no nav - return this.getTopics(count != 0 ? count : this.topicNavigatorCount, 500); + if (!this.ref.sizeSmaller(2)) { + ScoreMap result; + int ic = count != 0 ? count : this.topicNavigatorCount; + + if (this.ref.size() <= ic) { // size matches return map directly + result = this.getTopics(/*ic, 500*/); + } else { // collect top most count topics + result = new ConcurrentScoreMap(); + Iterator it = this.getTopics(/*ic, 500*/).keys(false); + while (ic-- > 0 && it.hasNext()) { + String word = it.next(); + result.set(word, this.ref.get(word)); + } + } + return result; + } } return null; } @@ -1428,7 +1450,11 @@ public final class SearchEvent { */ public void addResult(URIMetadataNode resultEntry, final float score) { if (resultEntry == null) return; - final long ranking = ((long) (score * 128.f)) + postRanking(resultEntry, new ConcurrentScoreMap() /*this.snippetProcess.rankingProcess.getTopicNavigator(10)*/); + final long ranking = ((long) (score * 128.f)) + postRanking(resultEntry, this.ref /*this.getTopicNavigator(MAX_TOPWORDS)*/); + // TODO: above was originally using (see below), but getTopicNavigator returns this.ref and possibliy alters this.ref on first call (this.ref.size < 2 -> this.ref.clear) + // TODO: verify and straighten the use of addTopic, getTopic and getTopicNavigator and related score calculation + // final long ranking = ((long) (score * 128.f)) + postRanking(resultEntry, this.getTopicNavigator(MAX_TOPWORDS)); + resultEntry.setScore(ranking); // update the score of resultEntry for access by search interface / api this.resultList.put(new ReverseElement(resultEntry, ranking)); // remove smallest in case of overflow if (pollImmediately) this.resultList.poll(); // prevent re-ranking in case there is only a single index source which has already ranked entries. @@ -1467,24 +1493,27 @@ public final class SearchEvent { final String urlstring = rentry.url().toNormalform(true); final String[] urlcomps = MultiProtocolURL.urlComps(urlstring); final String[] descrcomps = MultiProtocolURL.splitpattern.split(rentry.title().toLowerCase()); - for (final String urlcomp : urlcomps) { + + // apply query-in-result matching + final QueryGoal.NormalizedWords urlcompmap = new QueryGoal.NormalizedWords(urlcomps); + final QueryGoal.NormalizedWords descrcompmap = new QueryGoal.NormalizedWords(descrcomps); + // the token map is used (instead of urlcomps/descrcomps) to determine appearance in url/title and eliminate double occurances + // (example Title="News News News News News News - today is party -- News News News News News News" to add one score instead of 12 * score !) + for (final String urlcomp : urlcompmap) { int tc = topwords.get(urlcomp); if (tc > 0) r += Math.max(1, tc) << this.query.ranking.coeff_urlcompintoplist; } - for (final String descrcomp : descrcomps) { + for (final String descrcomp : descrcompmap) { int tc = topwords.get(descrcomp); if (tc > 0) r += Math.max(1, tc) << this.query.ranking.coeff_descrcompintoplist; } - // apply query-in-result matching - final QueryGoal.NormalizedWords urlcomph = new QueryGoal.NormalizedWords(urlcomps); - final QueryGoal.NormalizedWords descrcomph = new QueryGoal.NormalizedWords(descrcomps); final Iterator shi = this.query.getQueryGoal().getIncludeWords(); String queryword; while (shi.hasNext()) { queryword = shi.next(); - if (urlcomph.contains(queryword)) r += 256 << this.query.ranking.coeff_appurl; - if (descrcomph.contains(queryword)) r += 256 << this.query.ranking.coeff_app_dc_title; + if (urlcompmap.contains(queryword)) r += 256 << this.query.ranking.coeff_appurl; + if (descrcompmap.contains(queryword)) r += 256 << this.query.ranking.coeff_app_dc_title; } return r; } @@ -1827,14 +1856,23 @@ public final class SearchEvent { // this is only available if execQuery() was called before return this.localSearchInclusion; } - - public ScoreMap getTopics(final int maxcount, final long maxtime) { - // create a list of words that had been computed by statistics over all - // words that appeared in the url or the description of all urls + + /** + * Return the list of words that had been computed by statistics over all + * words that appeared in the url or the description of all urls + * + * @return ScoreMap + */ + public ScoreMap getTopics(/* final int maxcount, final long maxtime */) { + /* ---------------------------------- start of rem (2016-09-03) + // TODO: result map is not used currently, verify if it should and use or delete this code block + // TODO: as it is not used now - in favour of performance this code block is rem'ed (2016-09-03) + final ScoreMap result = new ConcurrentScoreMap(); if ( this.ref.sizeSmaller(2) ) { this.ref.clear(); // navigators with one entry are not useful } + final Map counts = new HashMap(); final Iterator i = this.ref.keys(false); String word; @@ -1860,11 +1898,17 @@ public final class SearchEvent { result.set(ce.getKey(), (int) (((double) maxcount) * (ce.getValue() - min) / (max - min))); } } + /* ------------------------------------ end of rem (2016-09-03) */ return this.ref; } private final static Pattern lettermatch = Pattern.compile("[a-z]+"); + /** + * Collects topics in a ScoreMap for words not included in the query words. + * Words are also filtered by badword blacklist and stopword list. + * @param words + */ public void addTopic(final String[] words) { String word; for ( final String w : words ) { @@ -1881,6 +1925,10 @@ public final class SearchEvent { } } + /** + * Ad title words to this searchEvent's topic score map + * @param resultEntry + */ protected void addTopics(final URIMetadataNode resultEntry) { // take out relevant information for reference computation if ((resultEntry.url() == null) || (resultEntry.title() == null)) return; diff --git a/test/java/net/yacy/cora/sorting/ConcurrentScoreMapTest.java b/test/java/net/yacy/cora/sorting/ConcurrentScoreMapTest.java new file mode 100644 index 000000000..443a5436a --- /dev/null +++ b/test/java/net/yacy/cora/sorting/ConcurrentScoreMapTest.java @@ -0,0 +1,34 @@ + +package net.yacy.cora.sorting; + +import java.util.Iterator; +import static org.junit.Assert.assertEquals; +import org.junit.Test; + + +public class ConcurrentScoreMapTest { + + /** + * Test of totalCount method, of class ConcurrentScoreMap. + */ + @Test + public void testTotalCount() { + final ConcurrentScoreMap csm = new ConcurrentScoreMap(); + csm.set("first", 10); + csm.set("second", 5); + csm.set("third", 13); + + csm.set("first", 100); + + final Iterator it = csm.keys(true); + long sum = 0; + while (it.hasNext()) { + String x = it.next(); + long val = csm.get(x); + sum += val; + } + + assertEquals(sum, csm.totalCount()); + } + +}