Merge remote-tracking branch 'origin/master' into docker

pull/55/head
luccioman 9 years ago
commit d3f07acee7

@ -20,7 +20,7 @@
<classpathentry kind="lib" path="lib/commons-jxpath-1.3.jar"/>
<classpathentry kind="lib" path="lib/jsch-0.1.53.jar"/>
<classpathentry kind="lib" path="lib/jakarta-oro-2.0.8.jar"/>
<classpathentry kind="lib" path="lib/commons-fileupload-1.3.1.jar"/>
<classpathentry kind="lib" path="lib/commons-fileupload-1.3.2.jar"/>
<classpathentry kind="lib" path="lib/json-simple-1.1.1.jar"/>
<classpathentry kind="lib" path="lib/xercesImpl.jar"/>
<classpathentry kind="lib" path="lib/xml-apis.jar"/>
@ -48,20 +48,20 @@
<classpathentry kind="lib" path="lib/commons-io-2.5.jar"/>
<classpathentry kind="lib" path="lib/slf4j-api-1.7.18.jar"/>
<classpathentry kind="lib" path="lib/chardet.jar"/>
<classpathentry kind="lib" path="lib/jetty-client-9.2.16.v20160414.jar"/>
<classpathentry kind="lib" path="lib/jetty-continuation-9.2.16.v20160414.jar"/>
<classpathentry kind="lib" path="lib/jetty-deploy-9.2.16.v20160414.jar"/>
<classpathentry kind="lib" path="lib/jetty-http-9.2.16.v20160414.jar"/>
<classpathentry kind="lib" path="lib/jetty-io-9.2.16.v20160414.jar"/>
<classpathentry kind="lib" path="lib/jetty-jmx-9.2.16.v20160414.jar"/>
<classpathentry kind="lib" path="lib/jetty-proxy-9.2.16.v20160414.jar"/>
<classpathentry kind="lib" path="lib/jetty-security-9.2.16.v20160414.jar"/>
<classpathentry kind="lib" path="lib/jetty-server-9.2.16.v20160414.jar"/>
<classpathentry kind="lib" path="lib/jetty-servlet-9.2.16.v20160414.jar"/>
<classpathentry kind="lib" path="lib/jetty-servlets-9.2.16.v20160414.jar"/>
<classpathentry kind="lib" path="lib/jetty-util-9.2.16.v20160414.jar"/>
<classpathentry kind="lib" path="lib/jetty-webapp-9.2.16.v20160414.jar"/>
<classpathentry kind="lib" path="lib/jetty-xml-9.2.16.v20160414.jar"/>
<classpathentry kind="lib" path="lib/jetty-client-9.2.17.v20160517.jar"/>
<classpathentry kind="lib" path="lib/jetty-continuation-9.2.17.v20160517.jar"/>
<classpathentry kind="lib" path="lib/jetty-deploy-9.2.17.v20160517.jar"/>
<classpathentry kind="lib" path="lib/jetty-http-9.2.17.v20160517.jar"/>
<classpathentry kind="lib" path="lib/jetty-io-9.2.17.v20160517.jar"/>
<classpathentry kind="lib" path="lib/jetty-jmx-9.2.17.v20160517.jar"/>
<classpathentry kind="lib" path="lib/jetty-proxy-9.2.17.v20160517.jar"/>
<classpathentry kind="lib" path="lib/jetty-security-9.2.17.v20160517.jar"/>
<classpathentry kind="lib" path="lib/jetty-server-9.2.17.v20160517.jar"/>
<classpathentry kind="lib" path="lib/jetty-servlet-9.2.17.v20160517.jar"/>
<classpathentry kind="lib" path="lib/jetty-servlets-9.2.17.v20160517.jar"/>
<classpathentry kind="lib" path="lib/jetty-util-9.2.17.v20160517.jar"/>
<classpathentry kind="lib" path="lib/jetty-webapp-9.2.17.v20160517.jar"/>
<classpathentry kind="lib" path="lib/jetty-xml-9.2.17.v20160517.jar"/>
<classpathentry kind="lib" path="lib/httpclient-4.5.2.jar"/>
<classpathentry kind="lib" path="lib/httpmime-4.5.2.jar"/>
<classpathentry kind="lib" path="lib/noggit-0.6.jar"/>

@ -168,7 +168,7 @@
<pathelement location="${lib}/common-lang-3.2.1.jar" />
<pathelement location="${lib}/commons-codec-1.10.jar" />
<pathelement location="${lib}/commons-compress-1.11.jar" />
<pathelement location="${lib}/commons-fileupload-1.3.1.jar" />
<pathelement location="${lib}/commons-fileupload-1.3.2.jar" />
<pathelement location="${lib}/commons-io-2.5.jar" />
<pathelement location="${lib}/commons-jxpath-1.3.jar" />
<pathelement location="${lib}/commons-lang-2.6.jar" />
@ -190,20 +190,20 @@
<pathelement location="${lib}/javax.servlet-api-3.1.0.jar" />
<pathelement location="${lib}/jcifs-1.3.17.jar" />
<pathelement location="${lib}/jcl-over-slf4j-1.7.18.jar" />
<pathelement location="${lib}/jetty-client-9.2.16.v20160414.jar" />
<pathelement location="${lib}/jetty-continuation-9.2.16.v20160414.jar" />
<pathelement location="${lib}/jetty-deploy-9.2.16.v20160414.jar" />
<pathelement location="${lib}/jetty-http-9.2.16.v20160414.jar" />
<pathelement location="${lib}/jetty-io-9.2.16.v20160414.jar" />
<pathelement location="${lib}/jetty-jmx-9.2.16.v20160414.jar" />
<pathelement location="${lib}/jetty-proxy-9.2.16.v20160414.jar" />
<pathelement location="${lib}/jetty-security-9.2.16.v20160414.jar" />
<pathelement location="${lib}/jetty-server-9.2.16.v20160414.jar" />
<pathelement location="${lib}/jetty-servlet-9.2.16.v20160414.jar" />
<pathelement location="${lib}/jetty-servlets-9.2.16.v20160414.jar" />
<pathelement location="${lib}/jetty-util-9.2.16.v20160414.jar" />
<pathelement location="${lib}/jetty-webapp-9.2.16.v20160414.jar" />
<pathelement location="${lib}/jetty-xml-9.2.16.v20160414.jar" />
<pathelement location="${lib}/jetty-client-9.2.17.v20160517.jar" />
<pathelement location="${lib}/jetty-continuation-9.2.17.v20160517.jar" />
<pathelement location="${lib}/jetty-deploy-9.2.17.v20160517.jar" />
<pathelement location="${lib}/jetty-http-9.2.17.v20160517.jar" />
<pathelement location="${lib}/jetty-io-9.2.17.v20160517.jar" />
<pathelement location="${lib}/jetty-jmx-9.2.17.v20160517.jar" />
<pathelement location="${lib}/jetty-proxy-9.2.17.v20160517.jar" />
<pathelement location="${lib}/jetty-security-9.2.17.v20160517.jar" />
<pathelement location="${lib}/jetty-server-9.2.17.v20160517.jar" />
<pathelement location="${lib}/jetty-servlet-9.2.17.v20160517.jar" />
<pathelement location="${lib}/jetty-servlets-9.2.17.v20160517.jar" />
<pathelement location="${lib}/jetty-util-9.2.17.v20160517.jar" />
<pathelement location="${lib}/jetty-webapp-9.2.17.v20160517.jar" />
<pathelement location="${lib}/jetty-xml-9.2.17.v20160517.jar" />
<pathelement location="${lib}/jsch-0.1.53.jar" />
<pathelement location="${lib}/json-simple-1.1.1.jar" />
<pathelement location="${lib}/jsoup-1.9.1.jar" />

@ -36,7 +36,6 @@ import java.util.regex.Pattern;
import net.yacy.cora.protocol.Domains;
import net.yacy.cora.protocol.HeaderFramework;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.data.Translator;
import net.yacy.data.WorkTables;
import net.yacy.http.YaCyHttpServer;
import net.yacy.kelondro.workflow.InstantBusyThread;
@ -46,6 +45,7 @@ import net.yacy.search.SwitchboardConstants;
import net.yacy.server.serverObjects;
import net.yacy.server.serverSwitch;
import net.yacy.server.http.HTTPDFileHandler;
import net.yacy.utils.translation.TranslatorXliff;
import net.yacy.utils.upnp.UPnPMappingType;
import net.yacy.utils.upnp.UPnP;
@ -85,7 +85,7 @@ public class ConfigBasic {
// language settings
if (post != null && post.containsKey("language") && !lang.equals(post.get("language", "default")) &&
(Translator.changeLang(env, langPath, post.get("language", "default") + ".lng"))) {
(new TranslatorXliff().changeLang(env, langPath, post.get("language", "default") + ".lng"))) {
prop.put("changedLanguage", "1");
}

@ -57,6 +57,7 @@
<p>Make sure that you only download data from trustworthy sources. The new language file
might overwrite existing data if a file of the same name exists already.</p>
</fieldset>
<p><a href="Translator_p.html">Simple Editor</a> to add untranslated text</p>
</form>
#(status)#
::<p><strong>Unable to get URL: #[url]#</strong></p>

@ -82,7 +82,7 @@ public class ConfigLanguage_p {
* directory traversal attacks!
*/
if (langFiles.contains(selectedLanguage) || selectedLanguage.startsWith("default")) {
Translator.changeLang(env, langPath, selectedLanguage);
new TranslatorXliff().changeLang(env, langPath, selectedLanguage);
}
//delete language file
@ -95,6 +95,7 @@ public class ConfigLanguage_p {
if (langFiles.contains(selectedLanguage)) {
final File langfile= new File(langPath, selectedLanguage);
FileUtils.deletedelete(langfile);
new TranslatorXliff().getScratchFile(langfile).delete();
}
//load language file from URL
@ -105,7 +106,8 @@ public class ConfigLanguage_p {
final DigestURL u = new DigestURL(url);
it = FileUtils.strings(u.get(ClientIdentification.yacyInternetCrawlerAgent, null, null));
try {
File langFile = new File(langPath, u.getFileName());
TranslatorXliff tx = new TranslatorXliff();
File langFile = tx.getScratchFile(new File(langPath, u.getFileName()));
final OutputStreamWriter bw = new OutputStreamWriter(new FileOutputStream(langFile), StandardCharsets.UTF_8.name());
while (it.hasNext()) {
@ -116,14 +118,13 @@ public class ConfigLanguage_p {
// convert downloaded xliff to internal lng file
final String ext = Files.getFileExtension(langFile.getName());
if (ext.equalsIgnoreCase("xlf") || ext.equalsIgnoreCase("xliff")) {
TranslatorXliff tx = new TranslatorXliff();
Map<String,Map<String,String>> lng = TranslatorXliff.loadTranslationsListsFromXliff(langFile);
Map<String,Map<String,String>> lng = tx.loadTranslationsListsFromXliff(langFile);
langFile = new File(langPath, Files.getNameWithoutExtension(langFile.getName())+".lng");
tx.saveAsLngFile(null, langFile, lng);
}
if (post.containsKey("use_lang") && "on".equals(post.get("use_lang"))) {
Translator.changeLang(env, langPath, langFile.getName());
tx.changeLang(env, langPath, langFile.getName());
}
} catch (final IOException e) {
prop.put("status", "2");//error saving the language file

@ -0,0 +1,53 @@
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" >
<head>
<title>YaCy '#[clientname]#': Translation Editor</title>
#%env/templates/metas.template%#
</head>
<body id="Translator">
#%env/templates/header.template%#
#%env/templates/submenuDesign.template%#
<h2>Translation Editor</h2>
<p>Translate untranslated text of the user interface (current language). The modified translation file is stored in DATA/LOCALE directory</p>
<fieldset>
<legend>
<label>UI Translation</label>
</legend>
<p>Target Language: <b>#[targetlang]#</b></p><p class="error">#[errmsg]#</p>
<form id="Translation" method="post" action="Translator_p.html" enctype="multipart/form-data" accept-charset="UTF-8">
<label for="sourcefile">Source File</label>
<select name="sourcefile" onchange="submit();">
#{filelist}#
<option value="#[filename]#" #(selected)#::selected="selected"#(/selected)#>#[filename]#</option>
#{/filelist}#
</select> <a href="#[sourcefile]#" target="sourcefilewindow" >view it</a>
<label for="filteruntranslated">filter untranslated</label>
<input type="checkbox" name="filteruntranslated" id="filteruntranslated" value="true" onclick="submit();" #(filter.checked)#::checked="checked"#(/filter.checked)# />
<table>
<tr><th style="border-bottom: solid gray; border-bottom-width: 1px;">Source Text</th>
<th style="border-bottom: solid gray; border-bottom-width: 1px;">Translated Text ( #[targetlang]# )</th></tr>
#{textlist}#
<tr>
<td style="border-bottom: solid gray; border-bottom-width: 1px;" valign="top">
<!--<input type="text" name="sourcetxt" id="sourcetxt#[tokenid]#" size="80" disabled="true" value="#[sourcetxt]#"/>-->
<label for="targettxt#[tokenid]#" >#[sourcetxt]#</label>
</td>
<td style="border-bottom: solid gray; border-bottom-width: 1px;" valign="top" nowrap>
<input type="text" name="targettxt#[tokenid]#" id="targettxt#[tokenid]#" size="80" value="#[targettxt]#"/>#(filteruntranslated)#::<button name="approve" type="submit" value="#[tokenid]#" class="btn btn-sm"><span class="glyphicon glyphicon-ok-sign" style="color: green"/></button>#(/filteruntranslated)#
</td>
</tr>
#{/textlist}#
</table>
<input type="submit" name="savetranslationlist" value="Save translation" class="btn btn-primary"/>
</form>
</fieldset>
#%env/templates/footer.template%#
</body>
</html>

@ -0,0 +1,127 @@
/**
* Translator_p
* Copyright 2012 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany
* First released 14.09.2011 at http://yacy.net
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program in the file lgpl21.txt
* If not, see <http://www.gnu.org/licenses/>.
*/
import java.io.File;
import java.io.IOException;
import java.util.Iterator;
import java.util.Map;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.cora.util.ConcurrentLog;
import net.yacy.search.Switchboard;
import net.yacy.server.serverObjects;
import net.yacy.server.serverSwitch;
import net.yacy.server.servletProperties;
import net.yacy.utils.translation.CreateTranslationMasters;
public class Translator_p {
public static servletProperties respond(@SuppressWarnings("unused") final RequestHeader requestHeader, @SuppressWarnings("unused") final serverObjects post, @SuppressWarnings("unused") final serverSwitch env) {
try {
final servletProperties prop = new servletProperties();
final Switchboard sb = (Switchboard) env;
String langcfg = env.getConfig("locale.language", "default");
prop.put("targetlang", langcfg);
if ("default".equals(langcfg)) {
prop.put("errmsg", "activate a different language");
return prop;
} else {
prop.put("errmsg", "");
}
File lngfile = new File("locales", langcfg + ".lng");
CreateTranslationMasters ctm = new CreateTranslationMasters(/*new File ("locales","master.lng.xlf")*/);
File masterxlf = new File("locales", "master.lng.xlf");
if (!masterxlf.exists()) ctm.createMasterTranslationLists(masterxlf);
Map<String, Map<String, String>> origTrans = ctm.joinMasterTranslationLists(masterxlf, lngfile);
int i = 0;
if (origTrans.size() > 0) {
String filename = origTrans.keySet().iterator().next();
if (post != null && post.containsKey("sourcefile")) {
filename = post.get("sourcefile", filename);
}
Iterator<String> filenameit = origTrans.keySet().iterator();
while (filenameit.hasNext()) {
String tmp = filenameit.next();
prop.put("filelist_" + i + "_filename", tmp);
prop.put("filelist_" + i + "_selected", tmp.equals(filename));
i++;
}
prop.put("filelist", i);
prop.add("sourcefile", filename);
Map<String, String> origTextList = origTrans.get(filename);
i = 0;
boolean filteruntranslated = false;
int textlistid = -1;
if (post != null) {
filteruntranslated = post.getBoolean("filteruntranslated");
if (filteruntranslated) {
prop.put("filter.checked", 1);
} else {
prop.put("filter.checked", 0);
}
textlistid = post.getInt("approve", -1);
}
boolean changed = false;
for (String sourcetext : origTextList.keySet()) {
String targettxt = origTextList.get(sourcetext);
if (targettxt == null || targettxt.isEmpty()) {
prop.put("textlist_" + i + "_filteruntranslated", true);
} else if (filteruntranslated) {
continue;
}
if (i == textlistid && post != null) {
String t = post.get("targettxt" + Integer.toString(textlistid));
// correct common partial html markup (part of text identification for words also used as html parameter)
if (!t.isEmpty()) {
if (sourcetext.startsWith(">") && !t.startsWith(">")) t=">"+t;
if (sourcetext.endsWith("<") && !t.endsWith("<")) t=t+"<";
}
targettxt = t;
origTextList.replace(sourcetext, targettxt);
changed = true;
}
prop.putHTML("textlist_" + i + "_sourcetxt", sourcetext);
prop.putHTML("textlist_" + i + "_targettxt", targettxt);
prop.put("textlist_" + i + "_tokenid", Integer.toString(i));
prop.put("textlist_" + i + "_filteruntranslated_tokenid", Integer.toString(i));
//prop.put("textlist_" + i +"_filteruntranslated", filteruntranslated);
i++;
}
if (post != null && post.containsKey("savetranslationlist")) {
changed = true;
}
if (changed) {
ctm.saveAsLngFile(langcfg, ctm.getScratchFile(lngfile), origTrans);
}
}
prop.put("textlist", i);
return prop;
} catch (IOException ex) {
ConcurrentLog.logException(ex);
}
return null;
}
}

Binary file not shown.

Binary file not shown.

@ -36,7 +36,7 @@
<!-- the Solr version used in dependency section for all related dependencies -->
<solr.version>5.5.1</solr.version>
<!-- the Jetty version used in dependency section for all related dependencies -->
<jetty.version>9.2.16.v20160414</jetty.version>
<jetty.version>9.2.17.v20160517</jetty.version>
<!-- properties used for filtering yacyBuildProperties.java -->
<REPL_DATE>${DSTAMP}</REPL_DATE>
@ -332,7 +332,8 @@
<dependency>
<groupId>commons-fileupload</groupId>
<artifactId>commons-fileupload</artifactId>
<version>1.3.1</version>
<version>1.3.2</version>
<type>jar</type>
</dependency>
<dependency>
<groupId>commons-io</groupId>

@ -101,8 +101,6 @@ public class Domains {
private static Set<InetAddress> publicIPv4HostAddresses = new HashSet<InetAddress>(); // subset of myHostAddresses
private static Set<InetAddress> publicIPv6HostAddresses = new HashSet<InetAddress>(); // subset of myHostAddresses
private static Set<String> localHostNames = new HashSet<String>(); // subset of myHostNames
private static Set<String> publicIPv4HostNames = new HashSet<String>(); // subset of myHostNames
private static Set<String> publicIPv6HostNames = new HashSet<String>(); // subset of myHostNames
static {
localHostNames.add(LOCALHOST);
try {
@ -179,10 +177,8 @@ public class Domains {
ConcurrentLog.info("Domain Init", "local host address: " + hostaddress + " (public)");
if (a instanceof Inet4Address) {
publicIPv4HostAddresses.add(a);
if (hostname != null) {publicIPv4HostNames.add(hostname); publicIPv4HostNames.add(hostaddress);}
} else {
publicIPv6HostAddresses.add(a);
if (hostname != null) {publicIPv6HostNames.add(hostname); publicIPv6HostNames.add(hostaddress);}
}
}
}

@ -521,8 +521,8 @@ public final class CrawlSwitchboard {
CrawlProfile.getRecrawlDate(CRAWL_PROFILE_SNIPPET_LOCAL_MEDIA_RECRAWL_CYCLE),
-1,
true, true, true, false, // crawlingQ, followFrames, obeyHtmlRobotsNoindex, obeyHtmlRobotsNofollow,
false,
false,
false, // indexText
false, // indexMedia
true,
false,
-1, false, true, CrawlProfile.MATCH_NEVER_STRING,

@ -107,7 +107,7 @@ public class Translator {
* @param translationFile the File, which contains the Lists
* @return a HashMap, which contains for each File a HashMap with translations.
*/
public static Map<String, Map<String, String>> loadTranslationsLists(final File translationFile) {
public Map<String, Map<String, String>> loadTranslationsLists(final File translationFile) {
final Map<String, Map<String, String>> lists = new HashMap<String, Map<String, String>>(); //list of translationLists for different files.
Map<String, String> translationList = new LinkedHashMap<String, String>(); //current Translation Table (maintaining input order)
@ -186,7 +186,7 @@ public class Translator {
return true;
}
public static boolean translateFiles(final File sourceDir, final File destDir, final File baseDir, final File translationFile, final String extensions){
public boolean translateFiles(final File sourceDir, final File destDir, final File baseDir, final File translationFile, final String extensions){
return translateFiles(sourceDir, destDir, baseDir, loadTranslationsLists(translationFile), extensions);
}
@ -219,7 +219,7 @@ public class Translator {
return true;
}
public static boolean translateFilesRecursive(final File sourceDir, final File destDir, final File translationFile, final String extensions, final String notdir){
public boolean translateFilesRecursive(final File sourceDir, final File destDir, final File translationFile, final String extensions, final String notdir){
final List<File> dirList=FileUtils.getDirsRecursive(sourceDir, notdir);
dirList.add(sourceDir);
for (final File file : dirList) {
@ -248,7 +248,7 @@ public class Translator {
return map;
}
public static boolean changeLang(final serverSwitch env, final File langPath, final String lang) {
public boolean changeLang(final serverSwitch env, final File langPath, final String lang) {
boolean ret = false;
if ("default".equals(lang) || "default.lng".equals(lang)) {
@ -257,14 +257,10 @@ public class Translator {
} else {
final String htRootPath = env.getConfig(SwitchboardConstants.HTROOT_PATH, SwitchboardConstants.HTROOT_PATH_DEFAULT);
final File sourceDir = new File(env.getAppPath(), htRootPath);
final File destDir = new File(env.getDataPath("locale.translated_html", "DATA/LOCALE/htroot"), lang.substring(0, lang.length() - 4));// cut
// .lng
//File destDir = new File(env.getRootPath(), htRootPath + "/locale/" + lang.substring(0, lang.length() - 4));// cut
// .lng
final File destDir = new File(env.getDataPath("locale.translated_html", "DATA/LOCALE/htroot"), lang.substring(0, lang.length() - 4));// cut .lng
final File translationFile = new File(langPath, lang);
//if (translator.translateFiles(sourceDir, destDir, translationFile, "html")) {
if (Translator.translateFilesRecursive(sourceDir, destDir, translationFile, "html,template,inc", "locale")) {
if (translateFilesRecursive(sourceDir, destDir, translationFile, "html,template,inc", "locale")) {
env.setConfig("locale.language", lang.substring(0, lang.length() - 4));
Formatter.setLocale(env.getConfig("locale.language", "en"));
try {

@ -289,7 +289,7 @@ public class AutoSearch extends AbstractBusyThread {
urls.add(new DigestURL(entry, (byte[]) null));
this.gotresults++;
}
sb.addToIndex(urls, null, "AutoSearch", null, true);
sb.addToCrawler(urls, false);
ConcurrentLog.info(AutoSearch.class.getName(), "added " + urls.size() + " results from " + seed.getName() + " to index for query=" + currentQuery);
} catch (IOException ex) {
ConcurrentLog.info(AutoSearch.class.getName(), "no answer from " + seed.getName());

@ -3292,6 +3292,8 @@ public final class Switchboard extends serverSwitch {
/**
* load the content of a URL, parse the content and add the content to the index This process is started
* concurrently. The method returns immediately after the call.
* Loaded/indexed pages are added to the given SearchEvent. If this is not required prefer addToCrawler
* to spare concurrent processes, bandwidth and intransparent crawl/load activity
*
* @param url the url that shall be indexed
* @param searchEvent (optional) a search event that shall get results from the indexed pages directly
@ -3328,7 +3330,7 @@ public final class Switchboard extends serverSwitch {
continue;
}
requests.add(request);
}
}
new Thread() {
@Override

@ -89,7 +89,7 @@ public class CreateTranslationMasters extends TranslatorXliff {
public void createMasterTranslationLists(File masterOutputFile) throws IOException {
Map<String, Map<String, String>> xliffTrans;
if (masterOutputFile.exists()) // if file exists, conserve existing master content (may be updated by external tool)
xliffTrans = TranslatorXliff.loadTranslationsListsFromXliff(masterOutputFile);
xliffTrans = loadTranslationsListsFromXliff(masterOutputFile);
else
xliffTrans = new TreeMap<String, Map<String, String>>();
@ -97,7 +97,7 @@ public class CreateTranslationMasters extends TranslatorXliff {
for (String filename : lngFiles) {
// load translation list
ConcurrentLog.info("TRANSLATOR", "include translation file " + filename);
Map<String, Map<String, String>> origTrans = Translator.loadTranslationsLists(new File("locales", filename));
Map<String, Map<String, String>> origTrans = loadTranslationsLists(new File("locales", filename));
for (String transfilename : origTrans.keySet()) { // get translation filename
File checkfile = new File("htroot", transfilename);
@ -154,10 +154,10 @@ public class CreateTranslationMasters extends TranslatorXliff {
public Map<String, Map<String, String>> joinMasterTranslationLists(File xlifmaster, File lngfile) throws IOException {
final String filename = lngfile.getName();
Map<String, Map<String, String>> xliffTrans = TranslatorXliff.loadTranslationsListsFromXliff(xlifmaster);
Map<String, Map<String, String>> xliffTrans = loadTranslationsListsFromXliff(xlifmaster);
// load translation list
System.out.println("join into master translation file " + filename);
Map<String, Map<String, String>> origTrans = Translator.loadTranslationsLists(lngfile);
Map<String, Map<String, String>> origTrans = loadTranslationsLists(lngfile);
for (String transfilename : origTrans.keySet()) { // get translation filename
// compare translation list

@ -87,7 +87,7 @@ public class ListNonTranslatedFiles extends TranslatorUtil {
+ translationFile);
try {
Set<String> translatedRelativePaths = Translator.loadTranslationsLists(translationFile).keySet();
Set<String> translatedRelativePaths = new Translator().loadTranslationsLists(translationFile).keySet();
List<File> srcFiles = FileUtils.getFilesRecursive(sourceDir, excludedDir, fileFilter);

@ -86,7 +86,7 @@ public class TranslateAll extends TranslatorUtil {
File localeDestDir = new File(destDir, localeCode);
localeDestDir.mkdirs();
Translator.translateFilesRecursive(sourceDir, localeDestDir,
new Translator().translateFilesRecursive(sourceDir, localeDestDir,
translationFile, extensions, "locale");
}
}

@ -66,7 +66,7 @@ public class TranslateAllToOneLanguage extends TranslatorUtil {
+ translationFile);
try {
Translator.translateFilesRecursive(sourceDir, destDir,
new Translator().translateFilesRecursive(sourceDir, destDir,
translationFile, extensions, "locale");
} finally {
ConcurrentLog.shutdown();

@ -44,6 +44,7 @@ import javax.xml.bind.Unmarshaller;
import net.yacy.cora.util.ConcurrentLog;
import net.yacy.data.Translator;
import net.yacy.search.Switchboard;
import org.oasis.xliff.core_12.Body;
import org.oasis.xliff.core_12.Target;
@ -66,7 +67,7 @@ public class TranslatorXliff extends Translator {
* @return a HashMap, which contains for each File a HashMap with
* translations.
*/
public static Map<String, Map<String, String>> loadTranslationsListsFromXliff(final File xliffFile) {
public Map<String, Map<String, String>> loadTranslationsListsFromXliff(final File xliffFile) {
final Map<String, Map<String, String>> lngLists = new TreeMap<String, Map<String, String>>(); //list of translationLists for different files.
/**
@ -144,11 +145,13 @@ public class TranslatorXliff extends Translator {
* @param xliffFile
* @return translatio map
*/
public static Map<String, Map<String, String>> loadTranslationsLists(final File xliffFile) {
@Override
public Map<String, Map<String, String>> loadTranslationsLists(final File xliffFile) {
File locallng = getScratchFile(xliffFile);
if (xliffFile.getName().toLowerCase().endsWith(".xlf") || xliffFile.getName().toLowerCase().endsWith(".xliff")) {
return loadTranslationsListsFromXliff(xliffFile);
return locallng.exists() ? loadTranslationsListsFromXliff(locallng) : loadTranslationsListsFromXliff(xliffFile);
} else {
return Translator.loadTranslationsLists(xliffFile);
return locallng.exists() ? super.loadTranslationsLists(locallng) : super.loadTranslationsLists(xliffFile);
}
}
@ -317,4 +320,21 @@ public class TranslatorXliff extends Translator {
}
return s;
}
/**
* Get the path to a work/scratch file in the DATA/LOCALE directory with the
* same name as given in the langPath
*
* @param langFile the path with filename to the language file
* @return a path to DATA/LOCALE/langFile.filename()
*/
public File getScratchFile(final File langFile) {
if (Switchboard.getSwitchboard() != null) { // for debug and testing were switchboard is null
File f = Switchboard.getSwitchboard().getDataPath("locale.translated_html", "DATA/LOCALE");
f = new File(f.getParentFile(), langFile.getName());
return f;
} else {
return langFile;
}
}
}

@ -46,7 +46,6 @@ import net.yacy.cora.protocol.TimeoutRequest;
import net.yacy.cora.protocol.http.HTTPClient;
import net.yacy.cora.sorting.Array;
import net.yacy.cora.util.ConcurrentLog;
import net.yacy.data.Translator;
import net.yacy.gui.YaCyApp;
import net.yacy.gui.framework.Browser;
import net.yacy.http.Jetty9HttpServerImpl;
@ -67,6 +66,7 @@ import net.yacy.cora.protocol.ConnectionInfo;
import net.yacy.crawler.retrieval.Response;
import net.yacy.peers.Seed;
import net.yacy.server.serverSwitch;
import net.yacy.utils.translation.TranslatorXliff;
/**
@ -362,7 +362,7 @@ public final class yacy {
if (currentRev == null || !currentRev.equals(sb.getConfig(Seed.VERSION, ""))) try { //is this another version?!
final File sourceDir = new File(sb.getConfig(SwitchboardConstants.HTROOT_PATH, SwitchboardConstants.HTROOT_PATH_DEFAULT));
final File destDir = new File(sb.getDataPath("locale.translated_html", "DATA/LOCALE/htroot"), lang);
if (Translator.translateFilesRecursive(sourceDir, destDir, new File(locale_source, lang + ".lng"), "html,template,inc", "locale")){ //translate it
if (new TranslatorXliff().translateFilesRecursive(sourceDir, destDir, new File(locale_source, lang + ".lng"), "html,template,inc", "locale")){ //translate it
//write the new Versionnumber
final BufferedWriter bw = new BufferedWriter(new PrintWriter(new FileWriter(new File(destDir, "version"))));
bw.write(sb.getConfig(Seed.VERSION, "Error getting Version"));

@ -31,7 +31,7 @@ public class TranslatorXliffTest {
for (String filename : lngFiles) {
// load translation list
System.out.println("Test translation file " + filename);
Map<String, Map<String, String>> origTrans = Translator.loadTranslationsLists(new File("locales", filename));
Map<String, Map<String, String>> origTrans = new Translator().loadTranslationsLists(new File("locales", filename));
TranslatorXliff txlif = new TranslatorXliff();
// save as xliff file

Loading…
Cancel
Save