Merge remote-tracking branch 'origin/master' into docker

pull/55/head
luccioman 9 years ago
commit d3f07acee7

@ -20,7 +20,7 @@
<classpathentry kind="lib" path="lib/commons-jxpath-1.3.jar"/> <classpathentry kind="lib" path="lib/commons-jxpath-1.3.jar"/>
<classpathentry kind="lib" path="lib/jsch-0.1.53.jar"/> <classpathentry kind="lib" path="lib/jsch-0.1.53.jar"/>
<classpathentry kind="lib" path="lib/jakarta-oro-2.0.8.jar"/> <classpathentry kind="lib" path="lib/jakarta-oro-2.0.8.jar"/>
<classpathentry kind="lib" path="lib/commons-fileupload-1.3.1.jar"/> <classpathentry kind="lib" path="lib/commons-fileupload-1.3.2.jar"/>
<classpathentry kind="lib" path="lib/json-simple-1.1.1.jar"/> <classpathentry kind="lib" path="lib/json-simple-1.1.1.jar"/>
<classpathentry kind="lib" path="lib/xercesImpl.jar"/> <classpathentry kind="lib" path="lib/xercesImpl.jar"/>
<classpathentry kind="lib" path="lib/xml-apis.jar"/> <classpathentry kind="lib" path="lib/xml-apis.jar"/>
@ -48,20 +48,20 @@
<classpathentry kind="lib" path="lib/commons-io-2.5.jar"/> <classpathentry kind="lib" path="lib/commons-io-2.5.jar"/>
<classpathentry kind="lib" path="lib/slf4j-api-1.7.18.jar"/> <classpathentry kind="lib" path="lib/slf4j-api-1.7.18.jar"/>
<classpathentry kind="lib" path="lib/chardet.jar"/> <classpathentry kind="lib" path="lib/chardet.jar"/>
<classpathentry kind="lib" path="lib/jetty-client-9.2.16.v20160414.jar"/> <classpathentry kind="lib" path="lib/jetty-client-9.2.17.v20160517.jar"/>
<classpathentry kind="lib" path="lib/jetty-continuation-9.2.16.v20160414.jar"/> <classpathentry kind="lib" path="lib/jetty-continuation-9.2.17.v20160517.jar"/>
<classpathentry kind="lib" path="lib/jetty-deploy-9.2.16.v20160414.jar"/> <classpathentry kind="lib" path="lib/jetty-deploy-9.2.17.v20160517.jar"/>
<classpathentry kind="lib" path="lib/jetty-http-9.2.16.v20160414.jar"/> <classpathentry kind="lib" path="lib/jetty-http-9.2.17.v20160517.jar"/>
<classpathentry kind="lib" path="lib/jetty-io-9.2.16.v20160414.jar"/> <classpathentry kind="lib" path="lib/jetty-io-9.2.17.v20160517.jar"/>
<classpathentry kind="lib" path="lib/jetty-jmx-9.2.16.v20160414.jar"/> <classpathentry kind="lib" path="lib/jetty-jmx-9.2.17.v20160517.jar"/>
<classpathentry kind="lib" path="lib/jetty-proxy-9.2.16.v20160414.jar"/> <classpathentry kind="lib" path="lib/jetty-proxy-9.2.17.v20160517.jar"/>
<classpathentry kind="lib" path="lib/jetty-security-9.2.16.v20160414.jar"/> <classpathentry kind="lib" path="lib/jetty-security-9.2.17.v20160517.jar"/>
<classpathentry kind="lib" path="lib/jetty-server-9.2.16.v20160414.jar"/> <classpathentry kind="lib" path="lib/jetty-server-9.2.17.v20160517.jar"/>
<classpathentry kind="lib" path="lib/jetty-servlet-9.2.16.v20160414.jar"/> <classpathentry kind="lib" path="lib/jetty-servlet-9.2.17.v20160517.jar"/>
<classpathentry kind="lib" path="lib/jetty-servlets-9.2.16.v20160414.jar"/> <classpathentry kind="lib" path="lib/jetty-servlets-9.2.17.v20160517.jar"/>
<classpathentry kind="lib" path="lib/jetty-util-9.2.16.v20160414.jar"/> <classpathentry kind="lib" path="lib/jetty-util-9.2.17.v20160517.jar"/>
<classpathentry kind="lib" path="lib/jetty-webapp-9.2.16.v20160414.jar"/> <classpathentry kind="lib" path="lib/jetty-webapp-9.2.17.v20160517.jar"/>
<classpathentry kind="lib" path="lib/jetty-xml-9.2.16.v20160414.jar"/> <classpathentry kind="lib" path="lib/jetty-xml-9.2.17.v20160517.jar"/>
<classpathentry kind="lib" path="lib/httpclient-4.5.2.jar"/> <classpathentry kind="lib" path="lib/httpclient-4.5.2.jar"/>
<classpathentry kind="lib" path="lib/httpmime-4.5.2.jar"/> <classpathentry kind="lib" path="lib/httpmime-4.5.2.jar"/>
<classpathentry kind="lib" path="lib/noggit-0.6.jar"/> <classpathentry kind="lib" path="lib/noggit-0.6.jar"/>

@ -168,7 +168,7 @@
<pathelement location="${lib}/common-lang-3.2.1.jar" /> <pathelement location="${lib}/common-lang-3.2.1.jar" />
<pathelement location="${lib}/commons-codec-1.10.jar" /> <pathelement location="${lib}/commons-codec-1.10.jar" />
<pathelement location="${lib}/commons-compress-1.11.jar" /> <pathelement location="${lib}/commons-compress-1.11.jar" />
<pathelement location="${lib}/commons-fileupload-1.3.1.jar" /> <pathelement location="${lib}/commons-fileupload-1.3.2.jar" />
<pathelement location="${lib}/commons-io-2.5.jar" /> <pathelement location="${lib}/commons-io-2.5.jar" />
<pathelement location="${lib}/commons-jxpath-1.3.jar" /> <pathelement location="${lib}/commons-jxpath-1.3.jar" />
<pathelement location="${lib}/commons-lang-2.6.jar" /> <pathelement location="${lib}/commons-lang-2.6.jar" />
@ -190,20 +190,20 @@
<pathelement location="${lib}/javax.servlet-api-3.1.0.jar" /> <pathelement location="${lib}/javax.servlet-api-3.1.0.jar" />
<pathelement location="${lib}/jcifs-1.3.17.jar" /> <pathelement location="${lib}/jcifs-1.3.17.jar" />
<pathelement location="${lib}/jcl-over-slf4j-1.7.18.jar" /> <pathelement location="${lib}/jcl-over-slf4j-1.7.18.jar" />
<pathelement location="${lib}/jetty-client-9.2.16.v20160414.jar" /> <pathelement location="${lib}/jetty-client-9.2.17.v20160517.jar" />
<pathelement location="${lib}/jetty-continuation-9.2.16.v20160414.jar" /> <pathelement location="${lib}/jetty-continuation-9.2.17.v20160517.jar" />
<pathelement location="${lib}/jetty-deploy-9.2.16.v20160414.jar" /> <pathelement location="${lib}/jetty-deploy-9.2.17.v20160517.jar" />
<pathelement location="${lib}/jetty-http-9.2.16.v20160414.jar" /> <pathelement location="${lib}/jetty-http-9.2.17.v20160517.jar" />
<pathelement location="${lib}/jetty-io-9.2.16.v20160414.jar" /> <pathelement location="${lib}/jetty-io-9.2.17.v20160517.jar" />
<pathelement location="${lib}/jetty-jmx-9.2.16.v20160414.jar" /> <pathelement location="${lib}/jetty-jmx-9.2.17.v20160517.jar" />
<pathelement location="${lib}/jetty-proxy-9.2.16.v20160414.jar" /> <pathelement location="${lib}/jetty-proxy-9.2.17.v20160517.jar" />
<pathelement location="${lib}/jetty-security-9.2.16.v20160414.jar" /> <pathelement location="${lib}/jetty-security-9.2.17.v20160517.jar" />
<pathelement location="${lib}/jetty-server-9.2.16.v20160414.jar" /> <pathelement location="${lib}/jetty-server-9.2.17.v20160517.jar" />
<pathelement location="${lib}/jetty-servlet-9.2.16.v20160414.jar" /> <pathelement location="${lib}/jetty-servlet-9.2.17.v20160517.jar" />
<pathelement location="${lib}/jetty-servlets-9.2.16.v20160414.jar" /> <pathelement location="${lib}/jetty-servlets-9.2.17.v20160517.jar" />
<pathelement location="${lib}/jetty-util-9.2.16.v20160414.jar" /> <pathelement location="${lib}/jetty-util-9.2.17.v20160517.jar" />
<pathelement location="${lib}/jetty-webapp-9.2.16.v20160414.jar" /> <pathelement location="${lib}/jetty-webapp-9.2.17.v20160517.jar" />
<pathelement location="${lib}/jetty-xml-9.2.16.v20160414.jar" /> <pathelement location="${lib}/jetty-xml-9.2.17.v20160517.jar" />
<pathelement location="${lib}/jsch-0.1.53.jar" /> <pathelement location="${lib}/jsch-0.1.53.jar" />
<pathelement location="${lib}/json-simple-1.1.1.jar" /> <pathelement location="${lib}/json-simple-1.1.1.jar" />
<pathelement location="${lib}/jsoup-1.9.1.jar" /> <pathelement location="${lib}/jsoup-1.9.1.jar" />

@ -36,7 +36,6 @@ import java.util.regex.Pattern;
import net.yacy.cora.protocol.Domains; import net.yacy.cora.protocol.Domains;
import net.yacy.cora.protocol.HeaderFramework; import net.yacy.cora.protocol.HeaderFramework;
import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.protocol.RequestHeader;
import net.yacy.data.Translator;
import net.yacy.data.WorkTables; import net.yacy.data.WorkTables;
import net.yacy.http.YaCyHttpServer; import net.yacy.http.YaCyHttpServer;
import net.yacy.kelondro.workflow.InstantBusyThread; import net.yacy.kelondro.workflow.InstantBusyThread;
@ -46,6 +45,7 @@ import net.yacy.search.SwitchboardConstants;
import net.yacy.server.serverObjects; import net.yacy.server.serverObjects;
import net.yacy.server.serverSwitch; import net.yacy.server.serverSwitch;
import net.yacy.server.http.HTTPDFileHandler; import net.yacy.server.http.HTTPDFileHandler;
import net.yacy.utils.translation.TranslatorXliff;
import net.yacy.utils.upnp.UPnPMappingType; import net.yacy.utils.upnp.UPnPMappingType;
import net.yacy.utils.upnp.UPnP; import net.yacy.utils.upnp.UPnP;
@ -85,7 +85,7 @@ public class ConfigBasic {
// language settings // language settings
if (post != null && post.containsKey("language") && !lang.equals(post.get("language", "default")) && if (post != null && post.containsKey("language") && !lang.equals(post.get("language", "default")) &&
(Translator.changeLang(env, langPath, post.get("language", "default") + ".lng"))) { (new TranslatorXliff().changeLang(env, langPath, post.get("language", "default") + ".lng"))) {
prop.put("changedLanguage", "1"); prop.put("changedLanguage", "1");
} }

@ -57,6 +57,7 @@
<p>Make sure that you only download data from trustworthy sources. The new language file <p>Make sure that you only download data from trustworthy sources. The new language file
might overwrite existing data if a file of the same name exists already.</p> might overwrite existing data if a file of the same name exists already.</p>
</fieldset> </fieldset>
<p><a href="Translator_p.html">Simple Editor</a> to add untranslated text</p>
</form> </form>
#(status)# #(status)#
::<p><strong>Unable to get URL: #[url]#</strong></p> ::<p><strong>Unable to get URL: #[url]#</strong></p>

@ -82,7 +82,7 @@ public class ConfigLanguage_p {
* directory traversal attacks! * directory traversal attacks!
*/ */
if (langFiles.contains(selectedLanguage) || selectedLanguage.startsWith("default")) { if (langFiles.contains(selectedLanguage) || selectedLanguage.startsWith("default")) {
Translator.changeLang(env, langPath, selectedLanguage); new TranslatorXliff().changeLang(env, langPath, selectedLanguage);
} }
//delete language file //delete language file
@ -95,6 +95,7 @@ public class ConfigLanguage_p {
if (langFiles.contains(selectedLanguage)) { if (langFiles.contains(selectedLanguage)) {
final File langfile= new File(langPath, selectedLanguage); final File langfile= new File(langPath, selectedLanguage);
FileUtils.deletedelete(langfile); FileUtils.deletedelete(langfile);
new TranslatorXliff().getScratchFile(langfile).delete();
} }
//load language file from URL //load language file from URL
@ -105,7 +106,8 @@ public class ConfigLanguage_p {
final DigestURL u = new DigestURL(url); final DigestURL u = new DigestURL(url);
it = FileUtils.strings(u.get(ClientIdentification.yacyInternetCrawlerAgent, null, null)); it = FileUtils.strings(u.get(ClientIdentification.yacyInternetCrawlerAgent, null, null));
try { try {
File langFile = new File(langPath, u.getFileName()); TranslatorXliff tx = new TranslatorXliff();
File langFile = tx.getScratchFile(new File(langPath, u.getFileName()));
final OutputStreamWriter bw = new OutputStreamWriter(new FileOutputStream(langFile), StandardCharsets.UTF_8.name()); final OutputStreamWriter bw = new OutputStreamWriter(new FileOutputStream(langFile), StandardCharsets.UTF_8.name());
while (it.hasNext()) { while (it.hasNext()) {
@ -116,14 +118,13 @@ public class ConfigLanguage_p {
// convert downloaded xliff to internal lng file // convert downloaded xliff to internal lng file
final String ext = Files.getFileExtension(langFile.getName()); final String ext = Files.getFileExtension(langFile.getName());
if (ext.equalsIgnoreCase("xlf") || ext.equalsIgnoreCase("xliff")) { if (ext.equalsIgnoreCase("xlf") || ext.equalsIgnoreCase("xliff")) {
TranslatorXliff tx = new TranslatorXliff(); Map<String,Map<String,String>> lng = tx.loadTranslationsListsFromXliff(langFile);
Map<String,Map<String,String>> lng = TranslatorXliff.loadTranslationsListsFromXliff(langFile);
langFile = new File(langPath, Files.getNameWithoutExtension(langFile.getName())+".lng"); langFile = new File(langPath, Files.getNameWithoutExtension(langFile.getName())+".lng");
tx.saveAsLngFile(null, langFile, lng); tx.saveAsLngFile(null, langFile, lng);
} }
if (post.containsKey("use_lang") && "on".equals(post.get("use_lang"))) { if (post.containsKey("use_lang") && "on".equals(post.get("use_lang"))) {
Translator.changeLang(env, langPath, langFile.getName()); tx.changeLang(env, langPath, langFile.getName());
} }
} catch (final IOException e) { } catch (final IOException e) {
prop.put("status", "2");//error saving the language file prop.put("status", "2");//error saving the language file

@ -0,0 +1,53 @@
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" >
<head>
<title>YaCy '#[clientname]#': Translation Editor</title>
#%env/templates/metas.template%#
</head>
<body id="Translator">
#%env/templates/header.template%#
#%env/templates/submenuDesign.template%#
<h2>Translation Editor</h2>
<p>Translate untranslated text of the user interface (current language). The modified translation file is stored in DATA/LOCALE directory</p>
<fieldset>
<legend>
<label>UI Translation</label>
</legend>
<p>Target Language: <b>#[targetlang]#</b></p><p class="error">#[errmsg]#</p>
<form id="Translation" method="post" action="Translator_p.html" enctype="multipart/form-data" accept-charset="UTF-8">
<label for="sourcefile">Source File</label>
<select name="sourcefile" onchange="submit();">
#{filelist}#
<option value="#[filename]#" #(selected)#::selected="selected"#(/selected)#>#[filename]#</option>
#{/filelist}#
</select> <a href="#[sourcefile]#" target="sourcefilewindow" >view it</a>
<label for="filteruntranslated">filter untranslated</label>
<input type="checkbox" name="filteruntranslated" id="filteruntranslated" value="true" onclick="submit();" #(filter.checked)#::checked="checked"#(/filter.checked)# />
<table>
<tr><th style="border-bottom: solid gray; border-bottom-width: 1px;">Source Text</th>
<th style="border-bottom: solid gray; border-bottom-width: 1px;">Translated Text ( #[targetlang]# )</th></tr>
#{textlist}#
<tr>
<td style="border-bottom: solid gray; border-bottom-width: 1px;" valign="top">
<!--<input type="text" name="sourcetxt" id="sourcetxt#[tokenid]#" size="80" disabled="true" value="#[sourcetxt]#"/>-->
<label for="targettxt#[tokenid]#" >#[sourcetxt]#</label>
</td>
<td style="border-bottom: solid gray; border-bottom-width: 1px;" valign="top" nowrap>
<input type="text" name="targettxt#[tokenid]#" id="targettxt#[tokenid]#" size="80" value="#[targettxt]#"/>#(filteruntranslated)#::<button name="approve" type="submit" value="#[tokenid]#" class="btn btn-sm"><span class="glyphicon glyphicon-ok-sign" style="color: green"/></button>#(/filteruntranslated)#
</td>
</tr>
#{/textlist}#
</table>
<input type="submit" name="savetranslationlist" value="Save translation" class="btn btn-primary"/>
</form>
</fieldset>
#%env/templates/footer.template%#
</body>
</html>

@ -0,0 +1,127 @@
/**
* Translator_p
* Copyright 2012 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany
* First released 14.09.2011 at http://yacy.net
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program in the file lgpl21.txt
* If not, see <http://www.gnu.org/licenses/>.
*/
import java.io.File;
import java.io.IOException;
import java.util.Iterator;
import java.util.Map;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.cora.util.ConcurrentLog;
import net.yacy.search.Switchboard;
import net.yacy.server.serverObjects;
import net.yacy.server.serverSwitch;
import net.yacy.server.servletProperties;
import net.yacy.utils.translation.CreateTranslationMasters;
public class Translator_p {
public static servletProperties respond(@SuppressWarnings("unused") final RequestHeader requestHeader, @SuppressWarnings("unused") final serverObjects post, @SuppressWarnings("unused") final serverSwitch env) {
try {
final servletProperties prop = new servletProperties();
final Switchboard sb = (Switchboard) env;
String langcfg = env.getConfig("locale.language", "default");
prop.put("targetlang", langcfg);
if ("default".equals(langcfg)) {
prop.put("errmsg", "activate a different language");
return prop;
} else {
prop.put("errmsg", "");
}
File lngfile = new File("locales", langcfg + ".lng");
CreateTranslationMasters ctm = new CreateTranslationMasters(/*new File ("locales","master.lng.xlf")*/);
File masterxlf = new File("locales", "master.lng.xlf");
if (!masterxlf.exists()) ctm.createMasterTranslationLists(masterxlf);
Map<String, Map<String, String>> origTrans = ctm.joinMasterTranslationLists(masterxlf, lngfile);
int i = 0;
if (origTrans.size() > 0) {
String filename = origTrans.keySet().iterator().next();
if (post != null && post.containsKey("sourcefile")) {
filename = post.get("sourcefile", filename);
}
Iterator<String> filenameit = origTrans.keySet().iterator();
while (filenameit.hasNext()) {
String tmp = filenameit.next();
prop.put("filelist_" + i + "_filename", tmp);
prop.put("filelist_" + i + "_selected", tmp.equals(filename));
i++;
}
prop.put("filelist", i);
prop.add("sourcefile", filename);
Map<String, String> origTextList = origTrans.get(filename);
i = 0;
boolean filteruntranslated = false;
int textlistid = -1;
if (post != null) {
filteruntranslated = post.getBoolean("filteruntranslated");
if (filteruntranslated) {
prop.put("filter.checked", 1);
} else {
prop.put("filter.checked", 0);
}
textlistid = post.getInt("approve", -1);
}
boolean changed = false;
for (String sourcetext : origTextList.keySet()) {
String targettxt = origTextList.get(sourcetext);
if (targettxt == null || targettxt.isEmpty()) {
prop.put("textlist_" + i + "_filteruntranslated", true);
} else if (filteruntranslated) {
continue;
}
if (i == textlistid && post != null) {
String t = post.get("targettxt" + Integer.toString(textlistid));
// correct common partial html markup (part of text identification for words also used as html parameter)
if (!t.isEmpty()) {
if (sourcetext.startsWith(">") && !t.startsWith(">")) t=">"+t;
if (sourcetext.endsWith("<") && !t.endsWith("<")) t=t+"<";
}
targettxt = t;
origTextList.replace(sourcetext, targettxt);
changed = true;
}
prop.putHTML("textlist_" + i + "_sourcetxt", sourcetext);
prop.putHTML("textlist_" + i + "_targettxt", targettxt);
prop.put("textlist_" + i + "_tokenid", Integer.toString(i));
prop.put("textlist_" + i + "_filteruntranslated_tokenid", Integer.toString(i));
//prop.put("textlist_" + i +"_filteruntranslated", filteruntranslated);
i++;
}
if (post != null && post.containsKey("savetranslationlist")) {
changed = true;
}
if (changed) {
ctm.saveAsLngFile(langcfg, ctm.getScratchFile(lngfile), origTrans);
}
}
prop.put("textlist", i);
return prop;
} catch (IOException ex) {
ConcurrentLog.logException(ex);
}
return null;
}
}

Binary file not shown.

Binary file not shown.

@ -36,7 +36,7 @@
<!-- the Solr version used in dependency section for all related dependencies --> <!-- the Solr version used in dependency section for all related dependencies -->
<solr.version>5.5.1</solr.version> <solr.version>5.5.1</solr.version>
<!-- the Jetty version used in dependency section for all related dependencies --> <!-- the Jetty version used in dependency section for all related dependencies -->
<jetty.version>9.2.16.v20160414</jetty.version> <jetty.version>9.2.17.v20160517</jetty.version>
<!-- properties used for filtering yacyBuildProperties.java --> <!-- properties used for filtering yacyBuildProperties.java -->
<REPL_DATE>${DSTAMP}</REPL_DATE> <REPL_DATE>${DSTAMP}</REPL_DATE>
@ -332,7 +332,8 @@
<dependency> <dependency>
<groupId>commons-fileupload</groupId> <groupId>commons-fileupload</groupId>
<artifactId>commons-fileupload</artifactId> <artifactId>commons-fileupload</artifactId>
<version>1.3.1</version> <version>1.3.2</version>
<type>jar</type>
</dependency> </dependency>
<dependency> <dependency>
<groupId>commons-io</groupId> <groupId>commons-io</groupId>

@ -101,8 +101,6 @@ public class Domains {
private static Set<InetAddress> publicIPv4HostAddresses = new HashSet<InetAddress>(); // subset of myHostAddresses private static Set<InetAddress> publicIPv4HostAddresses = new HashSet<InetAddress>(); // subset of myHostAddresses
private static Set<InetAddress> publicIPv6HostAddresses = new HashSet<InetAddress>(); // subset of myHostAddresses private static Set<InetAddress> publicIPv6HostAddresses = new HashSet<InetAddress>(); // subset of myHostAddresses
private static Set<String> localHostNames = new HashSet<String>(); // subset of myHostNames private static Set<String> localHostNames = new HashSet<String>(); // subset of myHostNames
private static Set<String> publicIPv4HostNames = new HashSet<String>(); // subset of myHostNames
private static Set<String> publicIPv6HostNames = new HashSet<String>(); // subset of myHostNames
static { static {
localHostNames.add(LOCALHOST); localHostNames.add(LOCALHOST);
try { try {
@ -179,10 +177,8 @@ public class Domains {
ConcurrentLog.info("Domain Init", "local host address: " + hostaddress + " (public)"); ConcurrentLog.info("Domain Init", "local host address: " + hostaddress + " (public)");
if (a instanceof Inet4Address) { if (a instanceof Inet4Address) {
publicIPv4HostAddresses.add(a); publicIPv4HostAddresses.add(a);
if (hostname != null) {publicIPv4HostNames.add(hostname); publicIPv4HostNames.add(hostaddress);}
} else { } else {
publicIPv6HostAddresses.add(a); publicIPv6HostAddresses.add(a);
if (hostname != null) {publicIPv6HostNames.add(hostname); publicIPv6HostNames.add(hostaddress);}
} }
} }
} }

@ -521,8 +521,8 @@ public final class CrawlSwitchboard {
CrawlProfile.getRecrawlDate(CRAWL_PROFILE_SNIPPET_LOCAL_MEDIA_RECRAWL_CYCLE), CrawlProfile.getRecrawlDate(CRAWL_PROFILE_SNIPPET_LOCAL_MEDIA_RECRAWL_CYCLE),
-1, -1,
true, true, true, false, // crawlingQ, followFrames, obeyHtmlRobotsNoindex, obeyHtmlRobotsNofollow, true, true, true, false, // crawlingQ, followFrames, obeyHtmlRobotsNoindex, obeyHtmlRobotsNofollow,
false, false, // indexText
false, false, // indexMedia
true, true,
false, false,
-1, false, true, CrawlProfile.MATCH_NEVER_STRING, -1, false, true, CrawlProfile.MATCH_NEVER_STRING,

@ -107,7 +107,7 @@ public class Translator {
* @param translationFile the File, which contains the Lists * @param translationFile the File, which contains the Lists
* @return a HashMap, which contains for each File a HashMap with translations. * @return a HashMap, which contains for each File a HashMap with translations.
*/ */
public static Map<String, Map<String, String>> loadTranslationsLists(final File translationFile) { public Map<String, Map<String, String>> loadTranslationsLists(final File translationFile) {
final Map<String, Map<String, String>> lists = new HashMap<String, Map<String, String>>(); //list of translationLists for different files. final Map<String, Map<String, String>> lists = new HashMap<String, Map<String, String>>(); //list of translationLists for different files.
Map<String, String> translationList = new LinkedHashMap<String, String>(); //current Translation Table (maintaining input order) Map<String, String> translationList = new LinkedHashMap<String, String>(); //current Translation Table (maintaining input order)
@ -186,7 +186,7 @@ public class Translator {
return true; return true;
} }
public static boolean translateFiles(final File sourceDir, final File destDir, final File baseDir, final File translationFile, final String extensions){ public boolean translateFiles(final File sourceDir, final File destDir, final File baseDir, final File translationFile, final String extensions){
return translateFiles(sourceDir, destDir, baseDir, loadTranslationsLists(translationFile), extensions); return translateFiles(sourceDir, destDir, baseDir, loadTranslationsLists(translationFile), extensions);
} }
@ -219,7 +219,7 @@ public class Translator {
return true; return true;
} }
public static boolean translateFilesRecursive(final File sourceDir, final File destDir, final File translationFile, final String extensions, final String notdir){ public boolean translateFilesRecursive(final File sourceDir, final File destDir, final File translationFile, final String extensions, final String notdir){
final List<File> dirList=FileUtils.getDirsRecursive(sourceDir, notdir); final List<File> dirList=FileUtils.getDirsRecursive(sourceDir, notdir);
dirList.add(sourceDir); dirList.add(sourceDir);
for (final File file : dirList) { for (final File file : dirList) {
@ -248,7 +248,7 @@ public class Translator {
return map; return map;
} }
public static boolean changeLang(final serverSwitch env, final File langPath, final String lang) { public boolean changeLang(final serverSwitch env, final File langPath, final String lang) {
boolean ret = false; boolean ret = false;
if ("default".equals(lang) || "default.lng".equals(lang)) { if ("default".equals(lang) || "default.lng".equals(lang)) {
@ -257,14 +257,10 @@ public class Translator {
} else { } else {
final String htRootPath = env.getConfig(SwitchboardConstants.HTROOT_PATH, SwitchboardConstants.HTROOT_PATH_DEFAULT); final String htRootPath = env.getConfig(SwitchboardConstants.HTROOT_PATH, SwitchboardConstants.HTROOT_PATH_DEFAULT);
final File sourceDir = new File(env.getAppPath(), htRootPath); final File sourceDir = new File(env.getAppPath(), htRootPath);
final File destDir = new File(env.getDataPath("locale.translated_html", "DATA/LOCALE/htroot"), lang.substring(0, lang.length() - 4));// cut final File destDir = new File(env.getDataPath("locale.translated_html", "DATA/LOCALE/htroot"), lang.substring(0, lang.length() - 4));// cut .lng
// .lng
//File destDir = new File(env.getRootPath(), htRootPath + "/locale/" + lang.substring(0, lang.length() - 4));// cut
// .lng
final File translationFile = new File(langPath, lang); final File translationFile = new File(langPath, lang);
//if (translator.translateFiles(sourceDir, destDir, translationFile, "html")) { if (translateFilesRecursive(sourceDir, destDir, translationFile, "html,template,inc", "locale")) {
if (Translator.translateFilesRecursive(sourceDir, destDir, translationFile, "html,template,inc", "locale")) {
env.setConfig("locale.language", lang.substring(0, lang.length() - 4)); env.setConfig("locale.language", lang.substring(0, lang.length() - 4));
Formatter.setLocale(env.getConfig("locale.language", "en")); Formatter.setLocale(env.getConfig("locale.language", "en"));
try { try {

@ -289,7 +289,7 @@ public class AutoSearch extends AbstractBusyThread {
urls.add(new DigestURL(entry, (byte[]) null)); urls.add(new DigestURL(entry, (byte[]) null));
this.gotresults++; this.gotresults++;
} }
sb.addToIndex(urls, null, "AutoSearch", null, true); sb.addToCrawler(urls, false);
ConcurrentLog.info(AutoSearch.class.getName(), "added " + urls.size() + " results from " + seed.getName() + " to index for query=" + currentQuery); ConcurrentLog.info(AutoSearch.class.getName(), "added " + urls.size() + " results from " + seed.getName() + " to index for query=" + currentQuery);
} catch (IOException ex) { } catch (IOException ex) {
ConcurrentLog.info(AutoSearch.class.getName(), "no answer from " + seed.getName()); ConcurrentLog.info(AutoSearch.class.getName(), "no answer from " + seed.getName());

@ -3292,6 +3292,8 @@ public final class Switchboard extends serverSwitch {
/** /**
* load the content of a URL, parse the content and add the content to the index This process is started * load the content of a URL, parse the content and add the content to the index This process is started
* concurrently. The method returns immediately after the call. * concurrently. The method returns immediately after the call.
* Loaded/indexed pages are added to the given SearchEvent. If this is not required prefer addToCrawler
* to spare concurrent processes, bandwidth and intransparent crawl/load activity
* *
* @param url the url that shall be indexed * @param url the url that shall be indexed
* @param searchEvent (optional) a search event that shall get results from the indexed pages directly * @param searchEvent (optional) a search event that shall get results from the indexed pages directly
@ -3328,7 +3330,7 @@ public final class Switchboard extends serverSwitch {
continue; continue;
} }
requests.add(request); requests.add(request);
} }
new Thread() { new Thread() {
@Override @Override

@ -89,7 +89,7 @@ public class CreateTranslationMasters extends TranslatorXliff {
public void createMasterTranslationLists(File masterOutputFile) throws IOException { public void createMasterTranslationLists(File masterOutputFile) throws IOException {
Map<String, Map<String, String>> xliffTrans; Map<String, Map<String, String>> xliffTrans;
if (masterOutputFile.exists()) // if file exists, conserve existing master content (may be updated by external tool) if (masterOutputFile.exists()) // if file exists, conserve existing master content (may be updated by external tool)
xliffTrans = TranslatorXliff.loadTranslationsListsFromXliff(masterOutputFile); xliffTrans = loadTranslationsListsFromXliff(masterOutputFile);
else else
xliffTrans = new TreeMap<String, Map<String, String>>(); xliffTrans = new TreeMap<String, Map<String, String>>();
@ -97,7 +97,7 @@ public class CreateTranslationMasters extends TranslatorXliff {
for (String filename : lngFiles) { for (String filename : lngFiles) {
// load translation list // load translation list
ConcurrentLog.info("TRANSLATOR", "include translation file " + filename); ConcurrentLog.info("TRANSLATOR", "include translation file " + filename);
Map<String, Map<String, String>> origTrans = Translator.loadTranslationsLists(new File("locales", filename)); Map<String, Map<String, String>> origTrans = loadTranslationsLists(new File("locales", filename));
for (String transfilename : origTrans.keySet()) { // get translation filename for (String transfilename : origTrans.keySet()) { // get translation filename
File checkfile = new File("htroot", transfilename); File checkfile = new File("htroot", transfilename);
@ -154,10 +154,10 @@ public class CreateTranslationMasters extends TranslatorXliff {
public Map<String, Map<String, String>> joinMasterTranslationLists(File xlifmaster, File lngfile) throws IOException { public Map<String, Map<String, String>> joinMasterTranslationLists(File xlifmaster, File lngfile) throws IOException {
final String filename = lngfile.getName(); final String filename = lngfile.getName();
Map<String, Map<String, String>> xliffTrans = TranslatorXliff.loadTranslationsListsFromXliff(xlifmaster); Map<String, Map<String, String>> xliffTrans = loadTranslationsListsFromXliff(xlifmaster);
// load translation list // load translation list
System.out.println("join into master translation file " + filename); System.out.println("join into master translation file " + filename);
Map<String, Map<String, String>> origTrans = Translator.loadTranslationsLists(lngfile); Map<String, Map<String, String>> origTrans = loadTranslationsLists(lngfile);
for (String transfilename : origTrans.keySet()) { // get translation filename for (String transfilename : origTrans.keySet()) { // get translation filename
// compare translation list // compare translation list

@ -87,7 +87,7 @@ public class ListNonTranslatedFiles extends TranslatorUtil {
+ translationFile); + translationFile);
try { try {
Set<String> translatedRelativePaths = Translator.loadTranslationsLists(translationFile).keySet(); Set<String> translatedRelativePaths = new Translator().loadTranslationsLists(translationFile).keySet();
List<File> srcFiles = FileUtils.getFilesRecursive(sourceDir, excludedDir, fileFilter); List<File> srcFiles = FileUtils.getFilesRecursive(sourceDir, excludedDir, fileFilter);

@ -86,7 +86,7 @@ public class TranslateAll extends TranslatorUtil {
File localeDestDir = new File(destDir, localeCode); File localeDestDir = new File(destDir, localeCode);
localeDestDir.mkdirs(); localeDestDir.mkdirs();
Translator.translateFilesRecursive(sourceDir, localeDestDir, new Translator().translateFilesRecursive(sourceDir, localeDestDir,
translationFile, extensions, "locale"); translationFile, extensions, "locale");
} }
} }

@ -66,7 +66,7 @@ public class TranslateAllToOneLanguage extends TranslatorUtil {
+ translationFile); + translationFile);
try { try {
Translator.translateFilesRecursive(sourceDir, destDir, new Translator().translateFilesRecursive(sourceDir, destDir,
translationFile, extensions, "locale"); translationFile, extensions, "locale");
} finally { } finally {
ConcurrentLog.shutdown(); ConcurrentLog.shutdown();

@ -44,6 +44,7 @@ import javax.xml.bind.Unmarshaller;
import net.yacy.cora.util.ConcurrentLog; import net.yacy.cora.util.ConcurrentLog;
import net.yacy.data.Translator; import net.yacy.data.Translator;
import net.yacy.search.Switchboard;
import org.oasis.xliff.core_12.Body; import org.oasis.xliff.core_12.Body;
import org.oasis.xliff.core_12.Target; import org.oasis.xliff.core_12.Target;
@ -66,7 +67,7 @@ public class TranslatorXliff extends Translator {
* @return a HashMap, which contains for each File a HashMap with * @return a HashMap, which contains for each File a HashMap with
* translations. * translations.
*/ */
public static Map<String, Map<String, String>> loadTranslationsListsFromXliff(final File xliffFile) { public Map<String, Map<String, String>> loadTranslationsListsFromXliff(final File xliffFile) {
final Map<String, Map<String, String>> lngLists = new TreeMap<String, Map<String, String>>(); //list of translationLists for different files. final Map<String, Map<String, String>> lngLists = new TreeMap<String, Map<String, String>>(); //list of translationLists for different files.
/** /**
@ -144,11 +145,13 @@ public class TranslatorXliff extends Translator {
* @param xliffFile * @param xliffFile
* @return translatio map * @return translatio map
*/ */
public static Map<String, Map<String, String>> loadTranslationsLists(final File xliffFile) { @Override
public Map<String, Map<String, String>> loadTranslationsLists(final File xliffFile) {
File locallng = getScratchFile(xliffFile);
if (xliffFile.getName().toLowerCase().endsWith(".xlf") || xliffFile.getName().toLowerCase().endsWith(".xliff")) { if (xliffFile.getName().toLowerCase().endsWith(".xlf") || xliffFile.getName().toLowerCase().endsWith(".xliff")) {
return loadTranslationsListsFromXliff(xliffFile); return locallng.exists() ? loadTranslationsListsFromXliff(locallng) : loadTranslationsListsFromXliff(xliffFile);
} else { } else {
return Translator.loadTranslationsLists(xliffFile); return locallng.exists() ? super.loadTranslationsLists(locallng) : super.loadTranslationsLists(xliffFile);
} }
} }
@ -317,4 +320,21 @@ public class TranslatorXliff extends Translator {
} }
return s; return s;
} }
/**
* Get the path to a work/scratch file in the DATA/LOCALE directory with the
* same name as given in the langPath
*
* @param langFile the path with filename to the language file
* @return a path to DATA/LOCALE/langFile.filename()
*/
public File getScratchFile(final File langFile) {
if (Switchboard.getSwitchboard() != null) { // for debug and testing were switchboard is null
File f = Switchboard.getSwitchboard().getDataPath("locale.translated_html", "DATA/LOCALE");
f = new File(f.getParentFile(), langFile.getName());
return f;
} else {
return langFile;
}
}
} }

@ -46,7 +46,6 @@ import net.yacy.cora.protocol.TimeoutRequest;
import net.yacy.cora.protocol.http.HTTPClient; import net.yacy.cora.protocol.http.HTTPClient;
import net.yacy.cora.sorting.Array; import net.yacy.cora.sorting.Array;
import net.yacy.cora.util.ConcurrentLog; import net.yacy.cora.util.ConcurrentLog;
import net.yacy.data.Translator;
import net.yacy.gui.YaCyApp; import net.yacy.gui.YaCyApp;
import net.yacy.gui.framework.Browser; import net.yacy.gui.framework.Browser;
import net.yacy.http.Jetty9HttpServerImpl; import net.yacy.http.Jetty9HttpServerImpl;
@ -67,6 +66,7 @@ import net.yacy.cora.protocol.ConnectionInfo;
import net.yacy.crawler.retrieval.Response; import net.yacy.crawler.retrieval.Response;
import net.yacy.peers.Seed; import net.yacy.peers.Seed;
import net.yacy.server.serverSwitch; import net.yacy.server.serverSwitch;
import net.yacy.utils.translation.TranslatorXliff;
/** /**
@ -362,7 +362,7 @@ public final class yacy {
if (currentRev == null || !currentRev.equals(sb.getConfig(Seed.VERSION, ""))) try { //is this another version?! if (currentRev == null || !currentRev.equals(sb.getConfig(Seed.VERSION, ""))) try { //is this another version?!
final File sourceDir = new File(sb.getConfig(SwitchboardConstants.HTROOT_PATH, SwitchboardConstants.HTROOT_PATH_DEFAULT)); final File sourceDir = new File(sb.getConfig(SwitchboardConstants.HTROOT_PATH, SwitchboardConstants.HTROOT_PATH_DEFAULT));
final File destDir = new File(sb.getDataPath("locale.translated_html", "DATA/LOCALE/htroot"), lang); final File destDir = new File(sb.getDataPath("locale.translated_html", "DATA/LOCALE/htroot"), lang);
if (Translator.translateFilesRecursive(sourceDir, destDir, new File(locale_source, lang + ".lng"), "html,template,inc", "locale")){ //translate it if (new TranslatorXliff().translateFilesRecursive(sourceDir, destDir, new File(locale_source, lang + ".lng"), "html,template,inc", "locale")){ //translate it
//write the new Versionnumber //write the new Versionnumber
final BufferedWriter bw = new BufferedWriter(new PrintWriter(new FileWriter(new File(destDir, "version")))); final BufferedWriter bw = new BufferedWriter(new PrintWriter(new FileWriter(new File(destDir, "version"))));
bw.write(sb.getConfig(Seed.VERSION, "Error getting Version")); bw.write(sb.getConfig(Seed.VERSION, "Error getting Version"));

@ -31,7 +31,7 @@ public class TranslatorXliffTest {
for (String filename : lngFiles) { for (String filename : lngFiles) {
// load translation list // load translation list
System.out.println("Test translation file " + filename); System.out.println("Test translation file " + filename);
Map<String, Map<String, String>> origTrans = Translator.loadTranslationsLists(new File("locales", filename)); Map<String, Map<String, String>> origTrans = new Translator().loadTranslationsLists(new File("locales", filename));
TranslatorXliff txlif = new TranslatorXliff(); TranslatorXliff txlif = new TranslatorXliff();
// save as xliff file // save as xliff file

Loading…
Cancel
Save