boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into dist_macOS

Browse Source
pull/72/head
luccioman 8 years ago
parent 39ea28adfd 51c077f493
commit 74f9927ddc
12 changed files with 164 additions and 48 deletions
Show Stats Download Patch File Download Diff File

4
htroot/News.html
Unescape View File

@ -43,6 +43,10 @@
A change in the personal profile will create a news entry. You can see recently made changes of
profile entries on the Network page, where that profile change is visualized with a '*' beside the 'P' (profile) - selector.
</li>
<li>
Publishing of added or modified translation for the user interface. Other peers may include it in their local translation list.
To publish a translation, use the integrated <a href="Translator_p.html">translation editor</a> to add a translation and publish it afterwards.
</li>
</ul>
<p>
More news services will follow.

2
htroot/TransNews_p.html
Unescape View File

@ -6,7 +6,7 @@
</head>
<body>
#%env/templates/header.template%#
#%env/templates/submenuComputation.template%#
<h2 class="yacy">Translation News for Language #[currentlang]#</h2>
<form method="post" enctype="multipart/form-data">

33
htroot/TransNews_p.java
Unescape View File

@ -42,7 +42,6 @@ import net.yacy.server.serverObjects;
import net.yacy.server.serverSwitch;
import net.yacy.utils.crypt;
import net.yacy.utils.translation.TranslationManager;
import net.yacy.utils.translation.TranslatorXliff;
public class TransNews_p {
@ -90,32 +89,13 @@ public class TransNews_p {
continue;
}
if (NewsPool.CATEGORY_TRANSLATION_ADD.equals(rtmp.category())) {
//String tmplng = rtmp.attribute("language", null);
String tmplng = rtmp.attribute("language", null);
String tmpfile = rtmp.attribute("file", null);
String tmpsource = rtmp.attribute("source", null);
String tmptarget = rtmp.attribute("target", null);
//String tmptarget = rtmp.attribute("target", null);
if (sb.peers.mySeed().hash.equals(rtmp.originator())) {
/*
if (tmplng != null && tmplng.equals(currentlang)) {
sendit = false;
break;
}*/
if (tmpfile != null && tmpfile.equals(file)) {
sendit = false;
break;
}
if (tmpsource != null && tmpsource.equals(sourcetxt)) {
sendit = false;
break;
}
if (tmptarget != null && tmptarget.equals(targettxt)) {
sendit = false;
break;
}
}
// if news with file and source exist (maybe from other peer) - skip sending another msg (to avoid confusion)
if ((tmpfile != null && tmpfile.equals(file))
if ((tmplng != null && tmplng.equals(currentlang)) && (tmpfile != null && tmpfile.equals(file))
&& (tmpsource != null && tmpsource.equals(sourcetxt))) {
sendit = false;
break;
@ -189,8 +169,8 @@ public class TransNews_p {
final HashMap<String, Integer> positiveHashes = new HashMap<String, Integer>(); // a mapping from an url hash to Integer (count of votes)
accumulateVotes(sb, negativeHashes, positiveHashes, NewsPool.INCOMING_DB);
final ScoreMap<String> ranking = new ConcurrentScoreMap<String>(); // score cluster for url hashes
final HashMap<String, NewsDB.Record> Translation = new HashMap<String, NewsDB.Record>(); // a mapping from an url hash to a kelondroRow.Entry with display properties
accumulateTranslations(sb, Translation, ranking, negativeHashes, positiveHashes, NewsPool.INCOMING_DB);
final HashMap<String, NewsDB.Record> translation = new HashMap<String, NewsDB.Record>(); // a mapping from an url hash to a kelondroRow.Entry with display properties
accumulateTranslations(sb, translation, ranking, negativeHashes, positiveHashes, NewsPool.INCOMING_DB);
// read out translation-news array and create property entries
final Iterator<String> k = ranking.keys(false);
@ -207,7 +187,7 @@ public class TransNews_p {
continue;
}
row = Translation.get(refid);
row = translation.get(refid);
if (row == null) {
continue;
}
@ -220,6 +200,7 @@ public class TransNews_p {
continue;
}
if (!lang.equals(currentlang)) continue;
String existingtarget = null; //transMgr.getTranslation(filename, source);
Map<String, String> tmpMap = localTrans.get(filename);

1
htroot/env/templates/submenuComputation.template vendored
Unescape View File

@ -33,6 +33,7 @@
<ul class="SubMenu">
<li><a href="Surftips.html" class="MenuItemLink">Surftips</a></li>
<li><a href="Wiki.html?display=1" class="MenuItemLink">Local Peer Wiki</a></li>
<li><a href="TransNews_p.html" class="MenuItemLink">UI Translations</a></li>
</ul>
</div>
</div>

4
htroot/yacy/search.java
Unescape View File

@ -380,7 +380,9 @@ public final class search {
// prepare reference hints
final long timer = System.currentTimeMillis();
final ScoreMap<String> topicNavigator = sb.index.connectedRWI() ? theSearch.getTopics(5, 100) : new ConcurrentScoreMap<String>();
//final ScoreMap<String> topicNavigator = sb.index.connectedRWI() ? theSearch.getTopics(5, 100) : new ConcurrentScoreMap<String>();
final ScoreMap<String> topicNavigator = theSearch.getTopics(); // as there is currently no index interaction in getTopics(), we can use it by default
final StringBuilder refstr = new StringBuilder(6000);
final Iterator<String> navigatorIterator = topicNavigator.keys(false);
int i = 0;

10
locales/fr.lng
Unescape View File

@ -1212,8 +1212,14 @@ Category==Categorie
Received==Reçu
Distributed==Distribu&eacute;
Attributes==Attributs
"#(page)#::Process Selected News::Delete Selected News::Abort Publication of Selected News::Delete Selected News#(/page)#"==#(page)#::Traiter les nouvelles s&eacute;lectionn&eacute;es::Supprimer les nouvelles s&eacute;lectionn&eacute;es::Annuler la publication des nouvelles s&eacute;lectionn&eacute;es::Supprimer les nouvelles s&eacute;lection&eacute;es#(/page)#
"#(page)#::Process All News::Delete All News::Abort Publication of All News::Delete All News#(/page)#"==#(page)#::Traiter toutes les nouvelles::Supprimer toutes les nouvelles::Annuler la publication de toutes les nouvelles::Supprimer toutes les nouvelles#(/page)#
Process Selected News==Traiter les nouvelles s&eacute;lectionn&eacute;es
Delete Selected News==Supprimer les nouvelles s&eacute;lectionn&eacute;es
Abort Publication of Selected News==Annuler la publication des nouvelles s&eacute;lectionn&eacute;es
Delete Selected News==Supprimer les nouvelles s&eacute;lection&eacute;es
Process All News==Traiter toutes les nouvelles
Delete All News==Supprimer toutes les nouvelles
Abort Publication of All News==Annuler la publication de toutes les nouvelles
Delete All News==Supprimer toutes les nouvelles
#-----------------------------
#File: Performance_p.html

36
locales/master.lng.xlf
Unescape View File

@ -5966,6 +5966,21 @@
<trans-unit id="49006831" xml:space="preserve" approved="no" translate="yes">
<source>profile entries on the Network page, where that profile change is visualized with a '*' beside the 'P' (profile) - selector.</source>
</trans-unit>
<trans-unit id="80e8551f" xml:space="preserve" approved="no" translate="yes">
<source>Publishing of added or modified translation for the user interface.</source>
</trans-unit>
<trans-unit id="aff2f3ce" xml:space="preserve" approved="no" translate="yes">
<source>Other peers may include it in their local translation list.</source>
</trans-unit>
<trans-unit id="304f12eb" xml:space="preserve" approved="no" translate="yes">
<source>To publish a translation, use the integrated</source>
</trans-unit>
<trans-unit id="f7855dbc" xml:space="preserve" approved="no" translate="yes">
<source>translation editor</source>
</trans-unit>
<trans-unit id="7e95942a" xml:space="preserve" approved="no" translate="yes">
<source>to add a translation and publish it afterwards.</source>
</trans-unit>
<trans-unit id="ac7ecc34" xml:space="preserve" approved="no" translate="yes">
<source>Above you can see four menues:</source>
</trans-unit>
@ -6008,6 +6023,24 @@
<trans-unit id="8eff8577" xml:space="preserve" approved="no" translate="yes">
<source>Attributes</source>
</trans-unit>
<trans-unit id="8ae5c927" xml:space="preserve" approved="no" translate="yes">
<source>Process Selected News</source>
</trans-unit>
<trans-unit id="d40fbba3" xml:space="preserve" approved="no" translate="yes">
<source>Delete Selected News</source>
</trans-unit>
<trans-unit id="43ac9953" xml:space="preserve" approved="no" translate="yes">
<source>Abort Publication of Selected News</source>
</trans-unit>
<trans-unit id="49ff88e3" xml:space="preserve" approved="no" translate="yes">
<source>Process All News</source>
</trans-unit>
<trans-unit id="6fcef8e7" xml:space="preserve" approved="no" translate="yes">
<source>Delete All News</source>
</trans-unit>
<trans-unit id="30aa4d37" xml:space="preserve" approved="no" translate="yes">
<source>Abort Publication of All News</source>
</trans-unit>
<trans-unit id="8c39153f" xml:space="preserve" approved="no" translate="yes">
<source>"#(page)#::Process Selected News::Delete Selected News::Abort Publication of Selected News::Delete Selected News#(/page)#"</source>
</trans-unit>
@ -10124,6 +10157,9 @@
<trans-unit id="a66de681" xml:space="preserve" approved="no" translate="yes">
<source>&gt;Local Peer Wiki&lt;</source>
</trans-unit>
<trans-unit id="cf394e0e" xml:space="preserve" approved="no" translate="yes">
<source>UI Translations</source>
</trans-unit>
</body>
</file>

5
source/net/yacy/cora/sorting/ConcurrentScoreMap.java
Unescape View File

@ -133,7 +133,10 @@ public class ConcurrentScoreMap<E> extends AbstractScoreMap<E> implements ScoreM
if (obj == null) return;
// use atomic operations
this.map.putIfAbsent(obj, new AtomicLong(0));
final AtomicLong old = this.map.putIfAbsent(obj, new AtomicLong(0));
// adjust overall counter if value replaced
if (old != null) this.gcount -= old.longValue(); // must use old befor setting a new value (it's a object reference)
this.map.get(obj).set(newScore);
// increase overall counter

2
source/net/yacy/peers/NewsPool.java
Unescape View File

@ -325,6 +325,8 @@ public class NewsPool {
if (this.newsDB.get(record.id()) == null) {
this.incomingNews.push(record); // we want to see our own news..
this.outgoingNews.push(record); // .. and put it on the publishing list
} else {
ConcurrentLog.info("NewsPool", "publishing of news aborted, news with same id (time + originator) exists id=" + record.id());
}
} catch (final Exception e) {
ConcurrentLog.logException(e);

3
source/net/yacy/peers/PeerActions.java
Unescape View File

@ -30,7 +30,6 @@ import net.yacy.cora.document.encoding.ASCII;
import net.yacy.cora.document.feed.RSSMessage;
import net.yacy.cora.storage.ConcurrentARC;
import net.yacy.kelondro.util.MapTools;
import net.yacy.peers.operation.yacyVersion;
public class PeerActions {
@ -261,7 +260,7 @@ public class PeerActions {
final String cre1 = MapTools.string2map(decodedString, ",").get("cre");
final String cre2 = MapTools.string2map(record.toString(), ",").get("cre");
if ((cre1 == null) || (cre2 == null) || (!(cre1.equals(cre2)))) {
System.out.println("### ERROR - cre are not equal: cre1=" + cre1 + ", cre2=" + cre2);
Network.log.warn("processPeerArrival: ### ERROR - message creation date verification not equal: cre1=" + cre1 + ", cre2=" + cre2);
return;
}
try {

78
source/net/yacy/search/query/SearchEvent.java
Unescape View File

@ -1329,10 +1329,32 @@ public final class SearchEvent {
public long getSnippetComputationTime() {
return this.snippetComputationAllTime;
}
public ScoreMap<String> getTopicNavigator(final int count ) {
/**
* Get topics in a ScoreMap if config allows topic navigator
* (the topics are filtered by badwords, stopwords and words included in the query)
*
* @param count max number of topics returned
* @return ScoreMap with max number of topics or null if
*/
public ScoreMap<String> getTopicNavigator(final int count) {
if (this.topicNavigatorCount > 0 && count >= 0) { //topicNavigatorCount set during init, 0=no nav
return this.getTopics(count != 0 ? count : this.topicNavigatorCount, 500);
if (!this.ref.sizeSmaller(2)) {
ScoreMap<String> result;
int ic = count != 0 ? count : this.topicNavigatorCount;
if (this.ref.size() <= ic) { // size matches return map directly
result = this.getTopics(/*ic, 500*/);
} else { // collect top most count topics
result = new ConcurrentScoreMap<String>();
Iterator<String> it = this.getTopics(/*ic, 500*/).keys(false);
while (ic-- > 0 && it.hasNext()) {
String word = it.next();
result.set(word, this.ref.get(word));
}
}
return result;
}
}
return null;
}
@ -1428,7 +1450,11 @@ public final class SearchEvent {
*/
public void addResult(URIMetadataNode resultEntry, final float score) {
if (resultEntry == null) return;
final long ranking = ((long) (score * 128.f)) + postRanking(resultEntry, new ConcurrentScoreMap<String>() /*this.snippetProcess.rankingProcess.getTopicNavigator(10)*/);
final long ranking = ((long) (score * 128.f)) + postRanking(resultEntry, this.ref /*this.getTopicNavigator(MAX_TOPWORDS)*/);
// TODO: above was originally using (see below), but getTopicNavigator returns this.ref and possibliy alters this.ref on first call (this.ref.size < 2 -> this.ref.clear)
// TODO: verify and straighten the use of addTopic, getTopic and getTopicNavigator and related score calculation
// final long ranking = ((long) (score * 128.f)) + postRanking(resultEntry, this.getTopicNavigator(MAX_TOPWORDS));
resultEntry.setScore(ranking); // update the score of resultEntry for access by search interface / api
this.resultList.put(new ReverseElement<URIMetadataNode>(resultEntry, ranking)); // remove smallest in case of overflow
if (pollImmediately) this.resultList.poll(); // prevent re-ranking in case there is only a single index source which has already ranked entries.
@ -1467,24 +1493,27 @@ public final class SearchEvent {
final String urlstring = rentry.url().toNormalform(true);
final String[] urlcomps = MultiProtocolURL.urlComps(urlstring);
final String[] descrcomps = MultiProtocolURL.splitpattern.split(rentry.title().toLowerCase());
for (final String urlcomp : urlcomps) {
// apply query-in-result matching
final QueryGoal.NormalizedWords urlcompmap = new QueryGoal.NormalizedWords(urlcomps);
final QueryGoal.NormalizedWords descrcompmap = new QueryGoal.NormalizedWords(descrcomps);
// the token map is used (instead of urlcomps/descrcomps) to determine appearance in url/title and eliminate double occurances
// (example Title="News News News News News News - today is party -- News News News News News News" to add one score instead of 12 * score !)
for (final String urlcomp : urlcompmap) {
int tc = topwords.get(urlcomp);
if (tc > 0) r += Math.max(1, tc) << this.query.ranking.coeff_urlcompintoplist;
}
for (final String descrcomp : descrcomps) {
for (final String descrcomp : descrcompmap) {
int tc = topwords.get(descrcomp);
if (tc > 0) r += Math.max(1, tc) << this.query.ranking.coeff_descrcompintoplist;
}
// apply query-in-result matching
final QueryGoal.NormalizedWords urlcomph = new QueryGoal.NormalizedWords(urlcomps);
final QueryGoal.NormalizedWords descrcomph = new QueryGoal.NormalizedWords(descrcomps);
final Iterator<String> shi = this.query.getQueryGoal().getIncludeWords();
String queryword;
while (shi.hasNext()) {
queryword = shi.next();
if (urlcomph.contains(queryword)) r += 256 << this.query.ranking.coeff_appurl;
if (descrcomph.contains(queryword)) r += 256 << this.query.ranking.coeff_app_dc_title;
if (urlcompmap.contains(queryword)) r += 256 << this.query.ranking.coeff_appurl;
if (descrcompmap.contains(queryword)) r += 256 << this.query.ranking.coeff_app_dc_title;
}
return r;
}
@ -1827,14 +1856,23 @@ public final class SearchEvent {
// this is only available if execQuery() was called before
return this.localSearchInclusion;
}
public ScoreMap<String> getTopics(final int maxcount, final long maxtime) {
// create a list of words that had been computed by statistics over all
// words that appeared in the url or the description of all urls
/**
* Return the list of words that had been computed by statistics over all
* words that appeared in the url or the description of all urls
*
* @return ScoreMap
*/
public ScoreMap<String> getTopics(/* final int maxcount, final long maxtime */) {
/* ---------------------------------- start of rem (2016-09-03)
// TODO: result map is not used currently, verify if it should and use or delete this code block
// TODO: as it is not used now - in favour of performance this code block is rem'ed (2016-09-03)
final ScoreMap<String> result = new ConcurrentScoreMap<String>();
if ( this.ref.sizeSmaller(2) ) {
this.ref.clear(); // navigators with one entry are not useful
}
final Map<String, Float> counts = new HashMap<String, Float>();
final Iterator<String> i = this.ref.keys(false);
String word;
@ -1860,11 +1898,17 @@ public final class SearchEvent {
result.set(ce.getKey(), (int) (((double) maxcount) * (ce.getValue() - min) / (max - min)));
}
}
/* ------------------------------------ end of rem (2016-09-03) */
return this.ref;
}
private final static Pattern lettermatch = Pattern.compile("[a-z]+");
/**
* Collects topics in a ScoreMap for words not included in the query words.
* Words are also filtered by badword blacklist and stopword list.
* @param words
*/
public void addTopic(final String[] words) {
String word;
for ( final String w : words ) {
@ -1881,6 +1925,10 @@ public final class SearchEvent {
}
}
/**
* Ad title words to this searchEvent's topic score map
* @param resultEntry
*/
protected void addTopics(final URIMetadataNode resultEntry) {
// take out relevant information for reference computation
if ((resultEntry.url() == null) || (resultEntry.title() == null)) return;

34
test/java/net/yacy/cora/sorting/ConcurrentScoreMapTest.java
Unescape View File

@ -0,0 +1,34 @@
package net.yacy.cora.sorting;
import java.util.Iterator;
import static org.junit.Assert.assertEquals;
import org.junit.Test;
public class ConcurrentScoreMapTest {
/**
* Test of totalCount method, of class ConcurrentScoreMap.
*/
@Test
public void testTotalCount() {
final ConcurrentScoreMap<String> csm = new ConcurrentScoreMap<String>();
csm.set("first", 10);
csm.set("second", 5);
csm.set("third", 13);
csm.set("first", 100);
final Iterator<String> it = csm.keys(true);
long sum = 0;
while (it.hasNext()) {
String x = it.next();
long val = csm.get(x);
sum += val;
}
assertEquals(sum, csm.totalCount());
}
}
Write Preview
Loading…
Cancel
Save