Merge branch 'master' of https://github.com/yacy/yacy_search_server.git
Michael Peter Christen
9 years ago
commit
8f4a341735
@ -0,0 +1,8 @@
|
||||
#!/usr/bin/env sh
|
||||
|
||||
# Launcher for YaCy in a MacOS bundle :
|
||||
# rely on the generic startYACY.sh, but specifies the user home relative path for YaCy data
|
||||
# This data directory is set in conforming to OS X File System Programming Guide
|
||||
# see : https://developer.apple.com/library/ios/documentation/FileManagement/Conceptual/FileSystemProgrammingGuide/MacOSXDirectories/MacOSXDirectories.html
|
||||
|
||||
"`dirname $0`"/startYACY.sh -s "'Library/Application Support/net.yacy.YaCy'"
|
||||
@ -0,0 +1,62 @@
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<title>YaCy '#[clientname]#': Translation News</title>
|
||||
#%env/templates/metas.template%#
|
||||
</head>
|
||||
<body>
|
||||
#%env/templates/header.template%#
|
||||
#%env/templates/submenuComputation.template%#
|
||||
<h2 class="yacy">Translation News for Language #[currentlang]#</h2>
|
||||
|
||||
<form method="post" enctype="multipart/form-data">
|
||||
<p>
|
||||
You can share your local addition to translations and distribute it to other peers.
|
||||
The remote peer can vote on your translation and add it to the own local translation.<br>
|
||||
(#[transsize]# entries available) <input type="submit" class="btn btn-default" name="publishtranslation" value="Publish">
|
||||
<small>You can check your outgoing messages <a href="News.html?page=3">here</a></small>
|
||||
</p>
|
||||
</form>
|
||||
#(errmsg)#::<p class="error">Please activate a different language <a href='ConfigBasic.html'>here</a></p>#(/errmsg)#
|
||||
#{results}#
|
||||
<!-- link begin -->
|
||||
|
||||
<fieldset>
|
||||
<table>
|
||||
<tr>
|
||||
<th>File:</th><th><a href="#[url]#" target="transnewsfile">#[filename]#</a></th><th>Originator</th>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>English:</td><td>#[source]#</td><td></td>
|
||||
</tr>
|
||||
#(existing)#::
|
||||
<tr>
|
||||
<td>existing</td><td class="warning">#[target]#</td><td></td>
|
||||
</tr>
|
||||
#(/existing)#
|
||||
<tr>
|
||||
<td>Translation:</td><td>#[target]#</td><td>#[peername]#</td>
|
||||
</tr>
|
||||
|
||||
<tr><td><small>score #[score]#</small></td>
|
||||
<td>
|
||||
<a href="TransNews_p.html?voteNegative=#[refid]#" title="negative vote">
|
||||
<span class="warning glyphicon glyphicon-thumbs-down"</span></a>
|
||||
|
||||
<a href="TransNews_p.html?votePositive=#[refid]#&filename=#[filename]#&source=#[source]#&target=#[target]#" title="positive vote" >
|
||||
<span class="success glyphicon glyphicon-thumbs-up"></span></a>
|
||||
<small>Vote on this translation. If you vote positive the translation is added to your local translation list.</small>
|
||||
</td><td></td>
|
||||
</tr>
|
||||
</table>
|
||||
</fieldset>
|
||||
|
||||
<!-- link end -->
|
||||
#{/results}#
|
||||
<p>
|
||||
<br>
|
||||
</p>
|
||||
|
||||
#%env/templates/footer.template%#
|
||||
</body>
|
||||
</html>
|
||||
@ -0,0 +1,323 @@
|
||||
// TransNews_p.java
|
||||
//
|
||||
// This is a part of YaCy, a peer-to-peer based web search engine
|
||||
// published on http://yacy.net
|
||||
//
|
||||
// This file is contributed by Burkhard Buelte
|
||||
//
|
||||
// $LastChangedDate$
|
||||
// $LastChangedRevision$
|
||||
// $LastChangedBy$
|
||||
//
|
||||
// LICENSE
|
||||
//
|
||||
// This program is free software; you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation; either version 2 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with this program; if not, write to the Free Software
|
||||
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.util.HashMap;
|
||||
import java.util.Iterator;
|
||||
import java.util.Map;
|
||||
|
||||
import net.yacy.cora.protocol.RequestHeader;
|
||||
import net.yacy.cora.sorting.ConcurrentScoreMap;
|
||||
import net.yacy.cora.sorting.ScoreMap;
|
||||
import net.yacy.cora.util.SpaceExceededException;
|
||||
import net.yacy.peers.NewsDB;
|
||||
import net.yacy.peers.NewsPool;
|
||||
import net.yacy.search.Switchboard;
|
||||
import net.yacy.server.serverObjects;
|
||||
import net.yacy.server.serverSwitch;
|
||||
import net.yacy.utils.crypt;
|
||||
import net.yacy.utils.translation.TranslationManager;
|
||||
|
||||
public class TransNews_p {
|
||||
|
||||
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
|
||||
final Switchboard sb = (Switchboard) env;
|
||||
final serverObjects prop = new serverObjects();
|
||||
|
||||
String currentlang = sb.getConfig("locale.language", "default");
|
||||
prop.put("currentlang", currentlang);
|
||||
|
||||
if ("default".equals(currentlang) || "browser".equals(currentlang)) {
|
||||
prop.put("errmsg", 1); // msg: activate diff lng
|
||||
prop.put("transsize", 0);
|
||||
return prop;
|
||||
} else {
|
||||
prop.put("errmsg", 0);
|
||||
}
|
||||
|
||||
TranslationManager transMgr = new TranslationManager();
|
||||
File locallangFile = transMgr.getScratchFile(new File(currentlang + ".lng"));
|
||||
Map<String, Map<String, String>> localTrans = transMgr.loadTranslationsLists(locallangFile);
|
||||
// calculate size of local translations list
|
||||
int size = 0;
|
||||
for (Map<String, String> lst : localTrans.values()) {
|
||||
size += lst.size();
|
||||
}
|
||||
prop.put("transsize", size);
|
||||
|
||||
|
||||
// read voting
|
||||
if ((post != null) && post.containsKey("publishtranslation")) {
|
||||
Iterator<String> filenameit = localTrans.keySet().iterator();
|
||||
while (filenameit.hasNext()) {
|
||||
String file = filenameit.next();
|
||||
Map<String, String> tmptrans = localTrans.get(file);
|
||||
for (String sourcetxt : tmptrans.keySet()) {
|
||||
String targettxt = tmptrans.get(sourcetxt);
|
||||
if (targettxt != null && !targettxt.isEmpty()) {
|
||||
boolean sendit = true;
|
||||
// check if already published (in newsPool)
|
||||
Iterator<NewsDB.Record> it = sb.peers.newsPool.recordIterator(NewsPool.INCOMING_DB);
|
||||
while (it.hasNext()) {
|
||||
NewsDB.Record rtmp = it.next();
|
||||
if (rtmp == null) {
|
||||
continue;
|
||||
}
|
||||
if (NewsPool.CATEGORY_TRANSLATION_ADD.equals(rtmp.category())) {
|
||||
String tmplng = rtmp.attribute("language", null);
|
||||
String tmpfile = rtmp.attribute("file", null);
|
||||
String tmpsource = rtmp.attribute("source", null);
|
||||
//String tmptarget = rtmp.attribute("target", null);
|
||||
|
||||
// if news with file and source exist (maybe from other peer) - skip sending another msg (to avoid confusion)
|
||||
if ((tmplng != null && tmplng.equals(currentlang)) && (tmpfile != null && tmpfile.equals(file))
|
||||
&& (tmpsource != null && tmpsource.equals(sourcetxt))) {
|
||||
sendit = false;
|
||||
break;
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
if (sendit) {
|
||||
final HashMap<String, String> map = new HashMap<String, String>();
|
||||
map.put("language", currentlang);
|
||||
map.put("file", file);
|
||||
map.put("source", sourcetxt);
|
||||
map.put("target", targettxt);
|
||||
sb.peers.newsPool.publishMyNews(sb.peers.mySeed(), NewsPool.CATEGORY_TRANSLATION_ADD, map);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
String refid;
|
||||
if ((post != null) && ((refid = post.get("voteNegative", null)) != null)) {
|
||||
|
||||
// make new news message with voting
|
||||
if (!sb.isRobinsonMode()) {
|
||||
final HashMap<String, String> map = new HashMap<String, String>();
|
||||
map.put("language", currentlang);
|
||||
map.put("file", crypt.simpleDecode(post.get("filename", "")));
|
||||
map.put("source", crypt.simpleDecode(post.get("source", "")));
|
||||
map.put("target", crypt.simpleDecode(post.get("target", "")));
|
||||
map.put("vote", "negative");
|
||||
map.put("refid", refid);
|
||||
sb.peers.newsPool.publishMyNews(sb.peers.mySeed(), NewsPool.CATEGORY_TRANSLATION_VOTE_ADD, map);
|
||||
try {
|
||||
sb.peers.newsPool.moveOff(NewsPool.INCOMING_DB, refid);
|
||||
} catch (IOException | SpaceExceededException ex) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if ((post != null) && ((refid = post.get("votePositive", null)) != null)) {
|
||||
|
||||
final String filename = post.get("filename");
|
||||
|
||||
File lngfile = new File(sb.getAppPath("locale.source", "locales"), currentlang + ".lng");
|
||||
transMgr = new TranslationManager(lngfile); // load full language for check if entry is new (globally)
|
||||
if (transMgr.addTranslation(filename, post.get("source"), post.get("target"))) {
|
||||
// add to local translation extension
|
||||
transMgr.addTranslation(localTrans, filename, post.get("source"), post.get("target"));
|
||||
transMgr.saveAsLngFile(currentlang, locallangFile, localTrans); // save local-trans to local-file
|
||||
transMgr.translateFile(filename); // ad-hoc translate file with new/added text
|
||||
} // TODO: shall we post voting if translation is not new ?
|
||||
|
||||
// make new news message with voting
|
||||
final HashMap<String, String> map = new HashMap<String, String>();
|
||||
map.put("language", currentlang);
|
||||
map.put("file", crypt.simpleDecode(filename));
|
||||
map.put("source", crypt.simpleDecode(post.get("source", "")));
|
||||
map.put("target", crypt.simpleDecode(post.get("target", "")));
|
||||
map.put("vote", "positive");
|
||||
map.put("refid", refid);
|
||||
sb.peers.newsPool.publishMyNews(sb.peers.mySeed(), NewsPool.CATEGORY_TRANSLATION_VOTE_ADD, map);
|
||||
try {
|
||||
sb.peers.newsPool.moveOff(NewsPool.INCOMING_DB, refid);
|
||||
} catch (IOException | SpaceExceededException ex) {
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// create Translation voting list
|
||||
final HashMap<String, Integer> negativeHashes = new HashMap<String, Integer>(); // a mapping from an url hash to Integer (count of votes)
|
||||
final HashMap<String, Integer> positiveHashes = new HashMap<String, Integer>(); // a mapping from an url hash to Integer (count of votes)
|
||||
accumulateVotes(sb, negativeHashes, positiveHashes, NewsPool.INCOMING_DB);
|
||||
final ScoreMap<String> ranking = new ConcurrentScoreMap<String>(); // score cluster for url hashes
|
||||
final HashMap<String, NewsDB.Record> translation = new HashMap<String, NewsDB.Record>(); // a mapping from an url hash to a kelondroRow.Entry with display properties
|
||||
accumulateTranslations(sb, translation, ranking, negativeHashes, positiveHashes, NewsPool.INCOMING_DB);
|
||||
|
||||
// read out translation-news array and create property entries
|
||||
final Iterator<String> k = ranking.keys(false);
|
||||
int i = 0;
|
||||
NewsDB.Record row;
|
||||
String filename;
|
||||
String source;
|
||||
String target;
|
||||
|
||||
while (k.hasNext()) {
|
||||
|
||||
refid = k.next();
|
||||
if (refid == null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
row = translation.get(refid);
|
||||
if (row == null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
String lang = row.attribute("language", null);
|
||||
filename = row.attribute("file", null);
|
||||
source = row.attribute("source", null);
|
||||
target = row.attribute("target", null);
|
||||
if ((lang == null) || (filename == null) || (source == null) || (target == null)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!lang.equals(currentlang)) continue;
|
||||
|
||||
String existingtarget = null; //transMgr.getTranslation(filename, source);
|
||||
Map<String, String> tmpMap = localTrans.get(filename);
|
||||
if (tmpMap != null) {
|
||||
existingtarget = tmpMap.get(source);
|
||||
}
|
||||
|
||||
boolean altexist = existingtarget != null && !target.isEmpty() && !existingtarget.isEmpty() && !existingtarget.equals(target);
|
||||
|
||||
prop.put("results_" + i + "_refid", refid);
|
||||
prop.put("results_" + i + "_url", filename); // url to local file
|
||||
prop.put("results_" + i + "_targetlanguage", lang);
|
||||
prop.put("results_" + i + "_filename", filename);
|
||||
prop.putHTML("results_" + i + "_source", source);
|
||||
prop.putHTML("results_" + i + "_target", target);
|
||||
prop.put("results_" + i + "_existing", altexist);
|
||||
prop.putHTML("results_" + i + "_existing_target", existingtarget);
|
||||
prop.put("results_" + i + "_score", ranking.get(refid));
|
||||
prop.put("results_" + i + "_peername", sb.peers.get(row.originator()).getName());
|
||||
i++;
|
||||
|
||||
if (i >= 50) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
prop.put("results", i);
|
||||
|
||||
return prop;
|
||||
}
|
||||
|
||||
private static void accumulateVotes(final Switchboard sb, final HashMap<String, Integer> negativeHashes, final HashMap<String, Integer> positiveHashes, final int dbtype) {
|
||||
final int maxCount = Math.min(1000, sb.peers.newsPool.size(dbtype));
|
||||
NewsDB.Record newsrecord;
|
||||
final Iterator<NewsDB.Record> recordIterator = sb.peers.newsPool.recordIterator(dbtype);
|
||||
int j = 0;
|
||||
while ((recordIterator.hasNext()) && (j++ < maxCount)) {
|
||||
newsrecord = recordIterator.next();
|
||||
if (newsrecord == null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (newsrecord.category().equals(NewsPool.CATEGORY_TRANSLATION_VOTE_ADD)) {
|
||||
final String refid = newsrecord.attribute("refid", "");
|
||||
final String vote = newsrecord.attribute("vote", "");
|
||||
final int factor = ((dbtype == NewsPool.OUTGOING_DB) || (dbtype == NewsPool.PUBLISHED_DB)) ? 2 : 1;
|
||||
if (vote.equals("negative")) {
|
||||
final Integer i = negativeHashes.get(refid);
|
||||
if (i == null) {
|
||||
negativeHashes.put(refid, Integer.valueOf(factor));
|
||||
} else {
|
||||
negativeHashes.put(refid, Integer.valueOf(i.intValue() + factor));
|
||||
}
|
||||
}
|
||||
if (vote.equals("positive")) {
|
||||
final Integer i = positiveHashes.get(refid);
|
||||
if (i == null) {
|
||||
positiveHashes.put(refid, Integer.valueOf(factor));
|
||||
} else {
|
||||
positiveHashes.put(refid, Integer.valueOf(i.intValue() + factor));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void accumulateTranslations(
|
||||
final Switchboard sb,
|
||||
final HashMap<String, NewsDB.Record> translationmsg, final ScoreMap<String> ranking,
|
||||
final HashMap<String, Integer> negativeHashes, final HashMap<String, Integer> positiveHashes, final int dbtype) {
|
||||
final int maxCount = Math.min(1000, sb.peers.newsPool.size(dbtype));
|
||||
NewsDB.Record newsrecord;
|
||||
final Iterator<NewsDB.Record> recordIterator = sb.peers.newsPool.recordIterator(dbtype);
|
||||
int j = 0;
|
||||
String refid = "";
|
||||
String targetlanguage ="";
|
||||
String filename="";
|
||||
String source="";
|
||||
String target="";
|
||||
|
||||
int score = 0;
|
||||
Integer vote;
|
||||
|
||||
while ((recordIterator.hasNext()) && (j++ < maxCount)) {
|
||||
newsrecord = recordIterator.next();
|
||||
if (newsrecord == null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if ((newsrecord.category().equals(NewsPool.CATEGORY_TRANSLATION_ADD))
|
||||
&& ((sb.peers.get(newsrecord.originator())) != null)) {
|
||||
refid = newsrecord.id();
|
||||
targetlanguage = newsrecord.attribute("language", "");
|
||||
filename = newsrecord.attribute("file", "");
|
||||
source = newsrecord.attribute("source", "");
|
||||
target = newsrecord.attribute("target", "");
|
||||
if (refid.isEmpty() || targetlanguage.isEmpty() || filename.isEmpty() || source.isEmpty() || target.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
score = 0;
|
||||
}
|
||||
|
||||
// add/subtract votes and write record
|
||||
|
||||
if ((vote = negativeHashes.get(refid)) != null) {
|
||||
score -= vote.intValue();
|
||||
}
|
||||
if ((vote = positiveHashes.get(refid)) != null) {
|
||||
score += vote.intValue();
|
||||
}
|
||||
// consider double-entries
|
||||
if (translationmsg.containsKey(refid)) {
|
||||
ranking.inc(refid, score);
|
||||
} else {
|
||||
ranking.set(refid, score);
|
||||
translationmsg.put(refid, newsrecord);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -0,0 +1,34 @@
|
||||
|
||||
package net.yacy.cora.sorting;
|
||||
|
||||
import java.util.Iterator;
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import org.junit.Test;
|
||||
|
||||
|
||||
public class ConcurrentScoreMapTest {
|
||||
|
||||
/**
|
||||
* Test of totalCount method, of class ConcurrentScoreMap.
|
||||
*/
|
||||
@Test
|
||||
public void testTotalCount() {
|
||||
final ConcurrentScoreMap<String> csm = new ConcurrentScoreMap<String>();
|
||||
csm.set("first", 10);
|
||||
csm.set("second", 5);
|
||||
csm.set("third", 13);
|
||||
|
||||
csm.set("first", 100);
|
||||
|
||||
final Iterator<String> it = csm.keys(true);
|
||||
long sum = 0;
|
||||
while (it.hasNext()) {
|
||||
String x = it.next();
|
||||
long val = csm.get(x);
|
||||
sum += val;
|
||||
}
|
||||
|
||||
assertEquals(sum, csm.totalCount());
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,39 @@
|
||||
|
||||
package net.yacy.document;
|
||||
|
||||
import java.net.MalformedURLException;
|
||||
import java.util.Map;
|
||||
import net.yacy.cora.document.WordCache;
|
||||
import net.yacy.kelondro.data.word.Word;
|
||||
import org.junit.Test;
|
||||
import static org.junit.Assert.*;
|
||||
|
||||
|
||||
public class TokenizerTest {
|
||||
|
||||
/**
|
||||
* Test of words method, of class Tokenizer.
|
||||
*/
|
||||
@Test
|
||||
public void testWords() throws MalformedURLException {
|
||||
// pos = 1 2 3 4 5 6 7 8 9 10 // 1-letter words don't count
|
||||
String text = "One word is not a sentence because words are just words.";
|
||||
WordCache meaningLib = new WordCache(null);
|
||||
boolean doAutotagging = false;
|
||||
VocabularyScraper scraper = null;
|
||||
|
||||
Tokenizer t = new Tokenizer(null, text, meaningLib, doAutotagging, scraper);
|
||||
|
||||
Map<String, Word> words = t.words;
|
||||
|
||||
// test extracted word information (position)
|
||||
Word w = words.get("word");
|
||||
assertEquals("position of 'word' ", 2, w.posInText);
|
||||
assertEquals("occurence of 'word' ", 1, w.occurrences());
|
||||
|
||||
w = words.get("words");
|
||||
assertEquals("position of 'words' ", 7, w.posInText);
|
||||
assertEquals("occurence of 'words' ", 2, w.occurrences());
|
||||
}
|
||||
|
||||
}
|
||||
Loading…
Reference in new issue