boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

Ensure lower case conversion consistency with any default locale.

Browse Source 
Especially for Turkish speaking users using "tr" as their system default
locale : strings for technical stuff (URLs, tag names, constants...)
must not be lower cased with the default locale, as 'I' doesn't becomes
'i' like in other locales such as "en", but becomes 'ı'.
pull/127/head
luccioman 8 years ago
parent 286f3018bd
commit 8da3174867
33 changed files with 95 additions and 60 deletions
Show Stats Download Patch File Download Diff File

3
htroot/ConfigHeuristics_p.java
Unescape View File

@ -39,6 +39,7 @@ import net.yacy.search.Switchboard;
import java.io.IOException;
import java.net.MalformedURLException;
import java.util.Iterator;
import java.util.Locale;
import net.yacy.cora.document.id.MultiProtocolURL;
import net.yacy.cora.federate.FederateSearchManager;
@ -101,7 +102,7 @@ public class ConfigHeuristics_p {
// add new entry to config file
final String tmpname = post.get("ossys_newtitle");
if (tmpname != null && tmpurl !=null) {
if (!tmpname.isEmpty() && !tmpurl.isEmpty() && tmpurl.toLowerCase().contains("{searchterms}")) {
if (!tmpname.isEmpty() && !tmpurl.isEmpty() && tmpurl.toLowerCase(Locale.ROOT).contains("{searchterms}")) {
/* Check eventual robots.txt policy */
RobotsTxtEntry robotsEntry = null;
try {

5
htroot/Crawler_p.java
Unescape View File

@ -29,6 +29,7 @@ import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.regex.Pattern;
@ -683,8 +684,8 @@ public class Crawler_p {
try {
wantedPPM = post.getInt("customPPM", wantedPPM);
} catch (final NumberFormatException e) {}
if ("minimum".equals(crawlingPerformance.toLowerCase())) wantedPPM = 10;
if ("maximum".equals(crawlingPerformance.toLowerCase())) wantedPPM = 30000;
if ("minimum".equals(crawlingPerformance.toLowerCase(Locale.ROOT))) wantedPPM = 10;
if ("maximum".equals(crawlingPerformance.toLowerCase(Locale.ROOT))) wantedPPM = 30000;
int wPPM = wantedPPM;
if ( wPPM <= 0 ) {

4
htroot/api/blacklists/add_entry_p.java
Unescape View File

@ -1,3 +1,5 @@
import java.util.Locale;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.data.ListManager;
import net.yacy.data.WorkTables;
@ -25,7 +27,7 @@ public class add_entry_p {
ListManager.switchboard.tables.recordAPICall(
post,
"add_entry_p." + header.fileType().toString().toLowerCase(),
"add_entry_p." + header.fileType().toString().toLowerCase(Locale.ROOT),
WorkTables.TABLE_API_TYPE_CONFIGURATION,
"add to blacklist '" + blacklistToUse + "': " + entry);

4
htroot/api/blacklists/delete_entry_p.java
Unescape View File

@ -1,3 +1,5 @@
import java.util.Locale;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.data.ListManager;
import net.yacy.data.WorkTables;
@ -26,7 +28,7 @@ public class delete_entry_p {
// store this call as api call
ListManager.switchboard.tables.recordAPICall(
post,
"delete_entry_p." + header.fileType().toString().toLowerCase(),
"delete_entry_p." + header.fileType().toString().toLowerCase(Locale.ROOT),
WorkTables.TABLE_API_TYPE_CONFIGURATION,
"delete from blacklist '" + blacklistToUse + "': " + entry);

3
htroot/api/getpageinfo_p.java
Unescape View File

@ -30,6 +30,7 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.Iterator;
import java.util.List;
import java.util.Locale;
import java.util.Set;
import javax.xml.parsers.DocumentBuilder;
@ -116,7 +117,7 @@ public class getpageinfo_p {
if (post.containsKey("actions"))
actions=post.get("actions");
String url=post.get("url");
if (url.toLowerCase().startsWith("ftp://")) {
if (url.toLowerCase(Locale.ROOT).startsWith("ftp://")) {
prop.put("robots-allowed", "1"); // ok to crawl
prop.put("robotsInfo", "ftp does not follow robots.txt");
prop.putXML("title", "FTP: " + url);

3
htroot/api/ymarks/add_ymark.java
Unescape View File

@ -1,4 +1,5 @@
import java.io.IOException;
import java.util.Locale;
import net.yacy.cora.document.id.DigestURL;
import net.yacy.cora.protocol.ClientIdentification;
@ -51,7 +52,7 @@ public class add_ymark {
String url = post.get(YMarkEntry.BOOKMARK.URL.key(),YMarkEntry.BOOKMARK.URL.deflt());
boolean hasProtocol = false;
for (final YMarkTables.PROTOCOLS p : YMarkTables.PROTOCOLS.values()) {
if(url.toLowerCase().startsWith(p.protocol())) {
if(url.toLowerCase(Locale.ROOT).startsWith(p.protocol())) {
hasProtocol = true;
break;
}

2
source/net/yacy/cora/protocol/RequestHeader.java
Unescape View File

@ -129,7 +129,7 @@ public class RequestHeader extends HeaderFramework implements HttpServletRequest
public FileType fileType() {
String path = this.getPathInfo();
if (path == null) return FileType.HTML;
path = path.toLowerCase();
path = path.toLowerCase(Locale.ROOT);
if (path.endsWith(".json")) return FileType.JSON;
if (path.endsWith(".xml")) return FileType.XML;
if (path.endsWith(".rdf")) return FileType.XML;

3
source/net/yacy/cora/protocol/http/HTTPClient.java
Unescape View File

@ -34,6 +34,7 @@ import java.security.KeyManagementException;
import java.security.NoSuchAlgorithmException;
import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;
import java.util.Locale;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
@ -664,7 +665,7 @@ public class HTTPClient {
mimeType = contentType.getValue();
if (mimeType != null) {
mimeType = mimeType.trim().toLowerCase();
mimeType = mimeType.trim().toLowerCase(Locale.ROOT);
final int pos = mimeType.indexOf(';');
if(pos >= 0) {

3
source/net/yacy/crawler/retrieval/Response.java
Unescape View File

@ -28,6 +28,7 @@ package net.yacy.crawler.retrieval;
import java.nio.charset.StandardCharsets;
import java.util.Date;
import java.util.Locale;
import net.yacy.cora.document.analysis.Classification;
import net.yacy.cora.document.encoding.ASCII;
@ -787,7 +788,7 @@ public class Response {
String mimeType = this.responseHeader.getContentType();
if (mimeType != null) {
mimeType = mimeType.trim().toLowerCase();
mimeType = mimeType.trim().toLowerCase(Locale.ROOT);
final int pos = mimeType.indexOf(';');
return ((pos < 0) ? mimeType : mimeType.substring(0, pos));

5
source/net/yacy/data/wiki/WikiCode.java
Unescape View File

@ -30,6 +30,7 @@ import java.util.Arrays;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.regex.Pattern;
@ -1069,7 +1070,7 @@ public class WikiCode extends AbstractWikiParser implements WikiParser {
}
if(closeIndex > 0) {
final String content = processedLine.substring(openIndex + LEN_WIKI_OPEN_METADATA, closeIndex);
if (content.toLowerCase().startsWith("coordinate")) {
if (content.toLowerCase(Locale.ROOT).startsWith("coordinate")) {
// parse Geographical Coordinates as described in
// http://en.wikipedia.org/wiki/Wikipedia:Manual_of_Style_%28dates_and_numbers%29#Geographical_coordinates
// looks like:
@ -1087,7 +1088,7 @@ public class WikiCode extends AbstractWikiParser implements WikiParser {
String name = "";
try {
for (final String c : b) {
if (c.toLowerCase().startsWith("name=")) {
if (c.toLowerCase(Locale.ROOT).startsWith("name=")) {
name = c.substring(5);
}
if (c.toUpperCase().startsWith("NS=")) {

21
source/net/yacy/document/Document.java
Unescape View File

@ -44,6 +44,7 @@ import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.regex.Matcher;
@ -565,8 +566,8 @@ dc_rights
continue;
}
final boolean noindex = url.getRelProperty().toLowerCase().indexOf("noindex",0) >= 0;
final boolean nofollow = url.getRelProperty().toLowerCase().indexOf("nofollow",0) >= 0;
final boolean noindex = url.getRelProperty().toLowerCase(Locale.ROOT).indexOf("noindex",0) >= 0;
final boolean nofollow = url.getRelProperty().toLowerCase(Locale.ROOT).indexOf("nofollow",0) >= 0;
if ((thishost == null && url.getHost() == null) ||
((thishost != null && url.getHost() != null) &&
(url.getHost().endsWith(thishost) ||
@ -578,9 +579,9 @@ dc_rights
extpos = u.lastIndexOf('.');
if (extpos > 0) {
if (((qpos = u.indexOf('?')) >= 0) && (qpos > extpos)) {
ext = u.substring(extpos + 1, qpos).toLowerCase();
ext = u.substring(extpos + 1, qpos).toLowerCase(Locale.ROOT);
} else {
ext = u.substring(extpos + 1).toLowerCase();
ext = u.substring(extpos + 1).toLowerCase(Locale.ROOT);
}
if (Classification.isMediaExtension(ext)) {
// this is not a normal anchor, its a media link
@ -705,10 +706,10 @@ dc_rights
u = url.toNormalform(true);
// find start of a referenced http url
if ((pos = u.toLowerCase().indexOf("http://", 7)) > 0) { // 7 = skip the protocol part of the source url
if ((pos = u.toLowerCase(Locale.ROOT).indexOf("http://", 7)) > 0) { // 7 = skip the protocol part of the source url
i.remove();
u = u.substring(pos);
while ((pos = u.toLowerCase().indexOf("http://", 7)) > 0)
while ((pos = u.toLowerCase(Locale.ROOT).indexOf("http://", 7)) > 0)
u = u.substring(pos);
url = new AnchorURL(u);
if (!(v.containsKey(url)))
@ -717,10 +718,10 @@ dc_rights
}
// find start of a referenced https url
if ((pos = u.toLowerCase().indexOf("https://", 7)) > 0) { // 7 = skip the protocol part of the source url
if ((pos = u.toLowerCase(Locale.ROOT).indexOf("https://", 7)) > 0) { // 7 = skip the protocol part of the source url
i.remove();
u = u.substring(pos);
while ((pos = u.toLowerCase().indexOf("https://", 7)) > 0)
while ((pos = u.toLowerCase(Locale.ROOT).indexOf("https://", 7)) > 0)
u = u.substring(pos);
url = new AnchorURL(u);
if (!(v.containsKey(url)))
@ -728,10 +729,10 @@ dc_rights
continue loop;
}
if ((pos = u.toLowerCase().indexOf("/www.", 11)) > 0) { // 11 = skip protocol part + www of source url "http://www."
if ((pos = u.toLowerCase(Locale.ROOT).indexOf("/www.", 11)) > 0) { // 11 = skip protocol part + www of source url "http://www."
i.remove();
u = url.getProtocol()+":/" + u.substring(pos);
while ((pos = u.toLowerCase().indexOf("/www.", 11)) > 0)
while ((pos = u.toLowerCase(Locale.ROOT).indexOf("/www.", 11)) > 0)
u = url.getProtocol()+":/" + u.substring(pos);
AnchorURL addurl = new AnchorURL(u);

5
source/net/yacy/document/content/SurrogateReader.java
Unescape View File

@ -32,6 +32,7 @@ import java.io.PushbackInputStream;
import java.io.StringReader;
import java.net.MalformedURLException;
import java.nio.charset.StandardCharsets;
import java.util.Locale;
import java.util.Map;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
@ -220,7 +221,7 @@ public class SurrogateReader extends DefaultHandler implements Runnable {
@Override
public void startElement(final String uri, final String name, String tag, final Attributes atts) throws SAXException {
if (tag == null) return;
tag = tag.toLowerCase();
tag = tag.toLowerCase(Locale.ROOT);
if ("record".equals(tag) || "document".equals(tag) || "doc".equals(tag)) {
this.dcEntry = new DCEntry();
} else if ("element".equals(tag) || "str".equals(tag) || "int".equals(tag) || "bool".equals(tag) || "long".equals(tag)) {
@ -239,7 +240,7 @@ public class SurrogateReader extends DefaultHandler implements Runnable {
@Override
public void endElement(final String uri, final String name, String tag) {
if (tag == null) return;
tag = tag.toLowerCase();
tag = tag.toLowerCase(Locale.ROOT);
if ("record".equals(tag) || "document".equals(tag) || "doc".equals(tag)) {
try {
// check if url is in accepted domain

5
source/net/yacy/document/parser/html/TransformerWriter.java
Unescape View File

@ -37,6 +37,7 @@ import java.io.OutputStreamWriter;
import java.io.Writer;
import java.nio.charset.Charset;
import java.util.Enumeration;
import java.util.Locale;
import java.util.Properties;
import java.util.Stack;
import net.yacy.document.parser.html.ContentScraper.TagName;
@ -194,7 +195,7 @@ public final class TransformerWriter extends Writer {
if (in[1] == '/') {
// a closing tag
tagend = tagEnd(in, 2);
tag = new String(in, 2, tagend - 2).toLowerCase();
tag = new String(in, 2, tagend - 2).toLowerCase(Locale.ROOT);
final char[] text = new char[in.length - tagend - 1];
System.arraycopy(in, tagend, text, 0, in.length - tagend - 1);
return filterTag(text, quotechar, tag, false);
@ -207,7 +208,7 @@ public final class TransformerWriter extends Writer {
// an opening tag
tagend = tagEnd(in, 1);
tag = new String(in, 1, tagend - 1).toLowerCase();
tag = new String(in, 1, tagend - 1).toLowerCase(Locale.ROOT);
final char[] text = new char[in.length - tagend - 1];
System.arraycopy(in, tagend, text, 0, in.length - tagend - 1);
return filterTag(text, quotechar, tag, true);

4
source/net/yacy/document/parser/rdfa/impl/RDFaTripleImpl.java
Unescape View File

@ -5,6 +5,8 @@ import java.io.File;
import java.io.IOException;
import java.io.Reader;
import java.util.ArrayList;
import java.util.Locale;
import javax.xml.transform.Templates;
import javax.xml.transform.Transformer;
import javax.xml.transform.TransformerConfigurationException;
@ -38,7 +40,7 @@ public class RDFaTripleImpl{
BufferedReader bufReader = new BufferedReader(in);
bufReader.mark(2048); // mark position for following reset
String readLine = bufReader.readLine();
if (!readLine.toLowerCase().contains("<!doctype")){
if (!readLine.toLowerCase(Locale.ROOT).contains("<!doctype")){
bufReader.reset();
}

3
source/net/yacy/gui/framework/Browser.java
Unescape View File

@ -30,6 +30,7 @@ import java.io.IOException;
import java.io.InputStreamReader;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.Locale;
import java.util.Properties;
import net.yacy.cora.util.ConcurrentLog;
@ -61,7 +62,7 @@ public class Browser {
static {
// check operation system type
final Properties sysprop = System.getProperties();
final String sysname = sysprop.getProperty("os.name","").toLowerCase();
final String sysname = sysprop.getProperty("os.name","").toLowerCase(Locale.ROOT);
if (sysname.startsWith("mac os x")) {
systemOS = systemMacOSX;
} else if (sysname.startsWith("mac os")) {

3
source/net/yacy/http/AbstractRemoteHandler.java
Unescape View File

@ -27,6 +27,7 @@ package net.yacy.http;
import java.io.IOException;
import java.net.InetAddress;
import java.util.HashSet;
import java.util.Locale;
import java.util.Set;
import java.util.StringTokenizer;
@ -143,7 +144,7 @@ abstract public class AbstractRemoteHandler extends ConnectHandler implements Ha
}
// check the blacklist
if (Switchboard.urlBlacklist.isListed(BlacklistType.PROXY, hostOnly.toLowerCase(), request.getPathInfo())) {
if (Switchboard.urlBlacklist.isListed(BlacklistType.PROXY, hostOnly.toLowerCase(Locale.ROOT), request.getPathInfo())) {
response.sendError(HttpServletResponse.SC_FORBIDDEN,
"URL '" + hostOnly + "' blocked by yacy proxy (blacklisted)");
baseRequest.setHandled(true);

5
source/net/yacy/http/servlets/YaCyDefaultServlet.java
Unescape View File

@ -39,6 +39,7 @@ import java.util.AbstractMap;
import java.util.Enumeration;
import java.util.Iterator;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.Callable;
@ -705,7 +706,7 @@ public class YaCyDefaultServlet extends HttpServlet {
/* Let's check this header has a valid value */
if("http".equals(protocolHeader) || "https".equals(protocolHeader)) {
protocol = protocolHeader.toLowerCase();
protocol = protocolHeader.toLowerCase(Locale.ROOT);
} else if(protocolHeader != null && !protocolHeader.isEmpty()) {
ConcurrentLog.warn("FILEHANDLER","YaCyDefaultServlet: illegal protocol scheme header value : " + protocolHeader);
}
@ -713,7 +714,7 @@ public class YaCyDefaultServlet extends HttpServlet {
/* This peer can also be behind a reverse proxy requested using https, even if the request coming to this YaCy peer is http only
* Possible scenario (happens for example when YaCy is deployed on Heroku Platform) : User browser -> https://reverseProxy/yacyURL -> http://yacypeer/yacyURL
* In that case, absolute URLs rendered by this peer (in rss feeds for example) must effectively start with the https scheme */
protocolHeader = header.get(HttpHeaders.X_FORWARDED_PROTO.toString(), "").toLowerCase();
protocolHeader = header.get(HttpHeaders.X_FORWARDED_PROTO.toString(), "").toLowerCase(Locale.ROOT);
/* Here we only allow an upgrade from HTTP to HTTPS, not the reverse (we don't want a forged HTTP header by an eventual attacker to force fallback to HTTP) */
if("https".equals(protocolHeader)) {

5
source/net/yacy/kelondro/data/meta/URIMetadataNode.java
Unescape View File

@ -34,6 +34,7 @@ import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Locale;
import java.util.Properties;
import java.util.Set;
import java.util.regex.Pattern;
@ -264,7 +265,7 @@ public class URIMetadataNode extends SolrDocument /* implements Comparable<URIMe
public boolean matches(Pattern pattern) {
return pattern.matcher(this.url.toNormalform(true)).matches();
//CharacterRunAutomaton automaton = new CharacterRunAutomaton(matcher);
//boolean match = automaton.run(this.url.toNormalform(true).toLowerCase());
//boolean match = automaton.run(this.url.toNormalform(true).toLowerCase(Locale.ROOT));
//return match;
}
@ -930,7 +931,7 @@ public class URIMetadataNode extends SolrDocument /* implements Comparable<URIMe
if (this.alternative_urlstring != null) return this.alternative_urlstring;
if (!pdfParser.individualPages) return this.url().toNormalform(true);
if (!"pdf".equals(MultiProtocolURL.getFileExtension(this.url().getFileName()).toLowerCase())) return this.url().toNormalform(true);
if (!"pdf".equals(MultiProtocolURL.getFileExtension(this.url().getFileName()).toLowerCase(Locale.ROOT))) return this.url().toNormalform(true);
// for pdf links we rewrite the url
// this is a special treatment of pdf files which can be splitted into subpages
String pageprop = pdfParser.individualPagePropertyname;

2
source/net/yacy/kelondro/util/Formatter.java
Unescape View File

@ -77,7 +77,7 @@ public final class Formatter {
* @param lang an ISO 639 language code which is used to generate a {@link Locale}
*/
public static void setLocale(final String lang) {
final String l = (lang.equalsIgnoreCase("default") ? "en" : lang.toLowerCase());
final String l = (lang.equalsIgnoreCase("default") ? "en" : lang.toLowerCase(Locale.ROOT));
setLocale(l.equals("none") ? null : new Locale(l));
}

7
source/net/yacy/kelondro/util/ISO639.java
Unescape View File

@ -26,6 +26,7 @@
package net.yacy.kelondro.util;
import java.util.Locale;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
@ -183,7 +184,7 @@ public class ISO639 {
* @return the name of the country
*/
public static final String country(String code) {
return mapping.get(code.toLowerCase());
return mapping.get(code.toLowerCase(Locale.ROOT));
}
/**
@ -192,7 +193,7 @@ public class ISO639 {
* @return true if the code exists
*/
public static final boolean exists(String code) {
return mapping.containsKey(code.toLowerCase());
return mapping.containsKey(code.toLowerCase(Locale.ROOT));
}
/**
@ -202,7 +203,7 @@ public class ISO639 {
*/
public static final String userAgentLanguageDetection(String userAgent) {
if (userAgent == null || userAgent.length() < 2) return null;
userAgent = userAgent.toLowerCase();
userAgent = userAgent.toLowerCase(Locale.ROOT);
if (mapping.containsKey(userAgent.substring(0, 2))) return userAgent.substring(0, 2);
if (userAgent.length() == 2 && mapping.containsKey(userAgent)) return userAgent;
if (userAgent.length() == 5 && mapping.containsKey(userAgent.substring(0, 2))) return userAgent.substring(0, 2);

3
source/net/yacy/kelondro/util/OS.java
Unescape View File

@ -29,6 +29,7 @@ import java.lang.management.ManagementFactory;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Properties;
@ -71,7 +72,7 @@ public final class OS {
static {
// check operation system type
final Properties sysprop = java.lang.System.getProperties();
final String sysname = sysprop.getProperty("os.name","").toLowerCase();
final String sysname = sysprop.getProperty("os.name","").toLowerCase(Locale.ROOT);
if (sysname.startsWith("mac os x")) systemOS = System.MacOSX;
else if (sysname.startsWith("mac os")) systemOS = System.MacOSC;
else if (sysname.startsWith("windows")) systemOS = System.Windows;

3
source/net/yacy/peers/Network.java
Unescape View File

@ -45,6 +45,7 @@ import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
@ -601,7 +602,7 @@ public class Network
if ( seedURLStr.isEmpty() ) {
throw new MalformedURLException("The seed-file url must not be empty.");
}
if ( !(seedURLStr.toLowerCase().startsWith("http://") || seedURLStr.toLowerCase().startsWith(
if ( !(seedURLStr.toLowerCase(Locale.ROOT).startsWith("http://") || seedURLStr.toLowerCase(Locale.ROOT).startsWith(
"https://")) ) {
throw new MalformedURLException("Unsupported protocol.");
}

3
source/net/yacy/peers/operation/yacyRelease.java
Unescape View File

@ -39,6 +39,7 @@ import java.security.SignatureException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.SortedSet;
import java.util.TreeSet;
@ -322,7 +323,7 @@ public final class yacyRelease extends yacyVersion {
}
final ResponseHeader header = response.getResponseHeader();
final boolean unzipped = header.gzip() && (header.mime().toLowerCase().equals("application/x-tar")); // if true, then the httpc has unzipped the file
final boolean unzipped = header.gzip() && (header.mime().toLowerCase(Locale.ROOT).equals("application/x-tar")); // if true, then the httpc has unzipped the file
if (unzipped && name.endsWith(".tar.gz")) {
download = new File(storagePath, name.substring(0, name.length() - 3));
} else {

5
source/net/yacy/search/index/Segment.java
Unescape View File

@ -33,6 +33,7 @@ import java.util.Collection;
import java.util.Date;
import java.util.Iterator;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ArrayBlockingQueue;
@ -504,10 +505,10 @@ public class Segment {
// two possible results: compare and report conflicts
if (!language.equals(url.language())) {
// see if we have a hint in the url that the statistic was right
final String u = urlNormalform.toLowerCase();
final String u = urlNormalform.toLowerCase(Locale.ROOT);
String ISO639_country = ISO639.country(language);
if (u.contains("/" + language + "/") ||
(ISO639_country != null && u.contains("/" + ISO639.country(language).toLowerCase() + "/"))) {
(ISO639_country != null && u.contains("/" + ISO639.country(language).toLowerCase(Locale.ROOT) + "/"))) {
// this is a strong hint that the statistics was in fact correct
} else {
// no confirmation using the url, use the TLD

3
source/net/yacy/search/query/QueryModifier.java
Unescape View File

@ -23,6 +23,7 @@ package net.yacy.search.query;
import java.net.MalformedURLException;
import java.util.ArrayList;
import java.util.Date;
import java.util.Locale;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.MultiMapSolrParams;
@ -200,7 +201,7 @@ public class QueryModifier {
this.language = querystring.substring(langi + 10, langi + 12);
querystring = querystring.replace("/language/" + this.language, "");
if (this.language.length() == 2 && ISO639.exists(this.language)) { // only 2-digit codes valid
this.language = this.language.toLowerCase();
this.language = this.language.toLowerCase(Locale.ROOT);
add("/language/" + this.language);
} else {
this.language = null;

3
source/net/yacy/search/query/QueryParams.java
Unescape View File

@ -36,6 +36,7 @@ import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.SortedSet;
@ -700,7 +701,7 @@ public final class QueryParams {
StringBuilder sb = new StringBuilder(120);
sb.append("yacysearch.");
sb.append(ext.name().toLowerCase());
sb.append(ext.name().toLowerCase(Locale.ROOT));
sb.append("?query=");
sb.append(theQuery.getQueryGoal().getQueryString(true));

7
source/net/yacy/search/schema/CollectionConfiguration.java
Unescape View File

@ -40,6 +40,7 @@ import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
@ -302,7 +303,7 @@ public class CollectionConfiguration extends SchemaConfiguration implements Seri
// path elements of link
String filename = digestURL.getFileName();
String extension = MultiProtocolURL.getFileExtension(filename);
String filenameStub = filename.toLowerCase().endsWith("." + extension) ? filename.substring(0, filename.length() - extension.length() - 1) : filename;
String filenameStub = filename.toLowerCase(Locale.ROOT).endsWith("." + extension) ? filename.substring(0, filename.length() - extension.length() - 1) : filename;
// remove possible jsession (or other url parm like "img.jpg;jsession=123")
// TODO: consider to implement ";jsession=123" check in getFileExtension()
if (extension.indexOf(';') >= 0) extension = extension.substring(0,extension.indexOf(';'));
@ -632,7 +633,7 @@ public class CollectionConfiguration extends SchemaConfiguration implements Seri
String robots_meta = html.getMetas().get("robots");
// this tag may have values: all, index, noindex, nofollow; see http://www.robotstxt.org/meta.html
if (robots_meta != null) {
robots_meta = robots_meta.toLowerCase();
robots_meta = robots_meta.toLowerCase(Locale.ROOT);
if (robots_meta.indexOf("all",0) >= 0) b += 1; // set bit 0
if (robots_meta.indexOf("index",0) == 0 || robots_meta.indexOf(" index",0) >= 0 || robots_meta.indexOf(",index",0) >= 0 ) b += 2; // set bit 1
if (robots_meta.indexOf("follow",0) == 0 || robots_meta.indexOf(" follow",0) >= 0 || robots_meta.indexOf(",follow",0) >= 0 ) b += 4; // set bit 2
@ -2166,7 +2167,7 @@ public class CollectionConfiguration extends SchemaConfiguration implements Seri
List<Integer> il = new ArrayList<Integer>(rel.size());
for (final String s: rel) {
int i = 0;
final String s0 = s.toLowerCase().trim();
final String s0 = s.toLowerCase(Locale.ROOT).trim();
if ("me".equals(s0)) i += 1;
if ("nofollow".equals(s0)) i += 2;
il.add(i);

3
source/net/yacy/search/schema/CollectionSchema.java
Unescape View File

@ -22,6 +22,7 @@ package net.yacy.search.schema;
import java.util.Date;
import java.util.List;
import java.util.Locale;
import net.yacy.cora.federate.solr.SchemaDeclaration;
import net.yacy.cora.federate.solr.SolrType;
@ -322,7 +323,7 @@ public enum CollectionSchema implements SchemaDeclaration {
public final void setSolrFieldName(String theValue) {
// make sure no empty string is assigned
if ( (theValue != null) && (!theValue.isEmpty()) ) {
this.solrFieldName = theValue.toLowerCase();
this.solrFieldName = theValue.toLowerCase(Locale.ROOT);
} else {
this.solrFieldName = null;
}

3
source/net/yacy/search/schema/WebgraphSchema.java
Unescape View File

@ -22,6 +22,7 @@ package net.yacy.search.schema;
import java.util.Date;
import java.util.List;
import java.util.Locale;
import net.yacy.cora.federate.solr.SchemaDeclaration;
import net.yacy.cora.federate.solr.SolrType;
@ -155,7 +156,7 @@ public enum WebgraphSchema implements SchemaDeclaration {
public final void setSolrFieldName(String theValue) {
// make sure no empty string is assigned
if ( (theValue != null) && (!theValue.isEmpty()) ) {
this.solrFieldName = theValue.toLowerCase();
this.solrFieldName = theValue.toLowerCase(Locale.ROOT);
} else {
this.solrFieldName = null;
}

3
source/net/yacy/server/serverObjects.java
Unescape View File

@ -55,6 +55,7 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.regex.Pattern;
@ -480,7 +481,7 @@ public class serverObjects implements Serializable, Cloneable {
public boolean getBoolean(final String key) {
String s = removeByteOrderMark(get(key));
if (s == null) return false;
s = s.toLowerCase();
s = s.toLowerCase(Locale.ROOT);
return s.equals("true") || s.equals("on") || s.equals("1");
}

3
source/net/yacy/utils/translation/TranslatorXliff.java
Unescape View File

@ -34,6 +34,7 @@ import java.io.IOException;
import java.io.OutputStreamWriter;
import java.nio.charset.StandardCharsets;
import java.util.LinkedHashMap;
import java.util.Locale;
import java.util.Map;
import java.util.TreeMap;
import javax.xml.stream.XMLInputFactory;
@ -163,7 +164,7 @@ public class TranslatorXliff extends Translator {
@Override
public Map<String, Map<String, String>> loadTranslationsLists(final File xliffFile) {
File locallng = getScratchFile(xliffFile);
if (xliffFile.getName().toLowerCase().endsWith(".xlf") || xliffFile.getName().toLowerCase().endsWith(".xliff")) {
if (xliffFile.getName().toLowerCase(Locale.ROOT).endsWith(".xlf") || xliffFile.getName().toLowerCase(Locale.ROOT).endsWith(".xliff")) {
if (locallng.exists()) {
Map<String, Map<String, String>> mergedList = loadTranslationsListsFromXliff(xliffFile);
Map<String, Map<String, String>> tmplist = loadTranslationsListsFromXliff(locallng);

15
source/net/yacy/yacy.java
Unescape View File

@ -40,6 +40,7 @@ import java.nio.channels.FileLock;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.Semaphore;
@ -780,7 +781,7 @@ public final class yacy {
// exception : if the -gui option is used then do not go into headless mode since that uses a gui
boolean headless = true;
if (OS.isWindows) headless = false;
if (args.length >= 1 && args[0].toLowerCase().equals("-gui")) headless = false;
if (args.length >= 1 && args[0].toLowerCase(Locale.ROOT).equals("-gui")) headless = false;
System.setProperty("java.awt.headless", headless ? "true" : "false");
String s = ""; for (final String a: args) s += a + " ";
@ -790,32 +791,32 @@ public final class yacy {
File dataRoot = applicationRoot;
//System.out.println("args.length=" + args.length);
//System.out.print("args=["); for (int i = 0; i < args.length; i++) System.out.print(args[i] + ", "); System.out.println("]");
if ((args.length >= 1) && (args[0].toLowerCase().equals("-startup") || args[0].equals("-start"))) {
if ((args.length >= 1) && (args[0].toLowerCase(Locale.ROOT).equals("-startup") || args[0].equals("-start"))) {
// normal start-up of yacy
if (args.length > 1) {
dataRoot = new File(System.getProperty("user.home").replace('\\', '/'), args[1]);
}
preReadSavedConfigandInit(dataRoot);
startup(dataRoot, applicationRoot, startupMemFree, startupMemTotal, false);
} else if (args.length >= 1 && args[0].toLowerCase().equals("-gui")) {
} else if (args.length >= 1 && args[0].toLowerCase(Locale.ROOT).equals("-gui")) {
// start-up of yacy with gui
if (args.length > 1) {
dataRoot = new File(System.getProperty("user.home").replace('\\', '/'), args[1]);
}
preReadSavedConfigandInit(dataRoot);
startup(dataRoot, applicationRoot, startupMemFree, startupMemTotal, true);
} else if ((args.length >= 1) && ((args[0].toLowerCase().equals("-shutdown")) || (args[0].equals("-stop")))) {
} else if ((args.length >= 1) && ((args[0].toLowerCase(Locale.ROOT).equals("-shutdown")) || (args[0].equals("-stop")))) {
// normal shutdown of yacy
if (args.length == 2) applicationRoot= new File(args[1]);
shutdown(applicationRoot);
} else if ((args.length >= 1) && (args[0].toLowerCase().equals("-update"))) {
} else if ((args.length >= 1) && (args[0].toLowerCase(Locale.ROOT).equals("-update"))) {
// aut-update yacy
if (args.length == 2) applicationRoot= new File(args[1]);
update(applicationRoot);
} else if ((args.length >= 1) && (args[0].toLowerCase().equals("-version"))) {
} else if ((args.length >= 1) && (args[0].toLowerCase(Locale.ROOT).equals("-version"))) {
// show yacy version
System.out.println(copyright);
} else if ((args.length > 1) && (args[0].toLowerCase().equals("-config"))) {
} else if ((args.length > 1) && (args[0].toLowerCase(Locale.ROOT).equals("-config"))) {
// set config parameter. Special handling of adminAccount=user:pwd (generates md5 encoded password)
// on Windows parameter should be enclosed in doublequotes to accept = sign (e.g. -config "port=8090" "port.ssl=8043")
File f = new File (dataRoot,"DATA/SETTINGS/");

6
test/java/net/yacy/document/parser/htmlParserTest.java
Unescape View File

@ -8,6 +8,8 @@ import java.net.MalformedURLException;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.Locale;
import junit.framework.TestCase;
import net.yacy.cora.document.id.AnchorURL;
import net.yacy.document.Document;
@ -40,13 +42,13 @@ public class htmlParserTest extends TestCase {
for (int i=0; i < testStrings.length; i++) {
// desired conversion result
String shouldBe = testStrings[i][1];
shouldBe = shouldBe!=null ? shouldBe.toLowerCase() : null;
shouldBe = shouldBe!=null ? shouldBe.toLowerCase(Locale.ROOT) : null;
// conversion result
String charset = htmlParser.patchCharsetEncoding(testStrings[i][0]);
// test if equal
assertEquals(shouldBe, charset!=null ? charset.toLowerCase() : null);
assertEquals(shouldBe, charset!=null ? charset.toLowerCase(Locale.ROOT) : null);
System.out.println("testGetRealCharsetEncoding: " + (testStrings[i][0]!=null?testStrings[i][0]:"null") + " -> " + (charset!=null?charset:"null") + " | Supported: " + (charset!=null?Charset.isSupported(charset):false));
}

Write Preview
Loading…
Cancel
Save