diff --git a/htroot/ConfigAppearance_p.java b/htroot/ConfigAppearance_p.java index 3791f9e8b..362012421 100644 --- a/htroot/ConfigAppearance_p.java +++ b/htroot/ConfigAppearance_p.java @@ -102,7 +102,7 @@ public class ConfigAppearance_p { final Iterator it; try { final DigestURL u = new DigestURL(url); - it = FileUtils.strings(u.get(ClientIdentification.yacyInternetCrawlerAgent)); + it = FileUtils.strings(u.get(ClientIdentification.yacyInternetCrawlerAgent, sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, ""))); } catch (final IOException e) { prop.put("status", "1");// unable to get URL prop.put("status_url", url); diff --git a/htroot/ConfigLanguage_p.java b/htroot/ConfigLanguage_p.java index b307d2c9a..e2239c3e2 100644 --- a/htroot/ConfigLanguage_p.java +++ b/htroot/ConfigLanguage_p.java @@ -46,6 +46,7 @@ import net.yacy.data.Translator; import net.yacy.data.WorkTables; import net.yacy.kelondro.util.FileUtils; import net.yacy.search.Switchboard; +import net.yacy.search.SwitchboardConstants; import net.yacy.server.serverObjects; import net.yacy.server.serverSwitch; @@ -101,7 +102,7 @@ public class ConfigLanguage_p { Iterator it; try { final DigestURL u = new DigestURL(url); - it = FileUtils.strings(u.get(ClientIdentification.yacyInternetCrawlerAgent)); + it = FileUtils.strings(u.get(ClientIdentification.yacyInternetCrawlerAgent, sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, ""))); } catch(final IOException e) { prop.put("status", "1");//unable to get url prop.put("status_url", url); diff --git a/htroot/CrawlStartScanner_p.java b/htroot/CrawlStartScanner_p.java index 190f023c4..694686b9c 100644 --- a/htroot/CrawlStartScanner_p.java +++ b/htroot/CrawlStartScanner_p.java @@ -42,6 +42,7 @@ import net.yacy.cora.sorting.ReversibleScoreMap; import net.yacy.cora.util.ConcurrentLog; import net.yacy.data.WorkTables; import net.yacy.search.Switchboard; +import net.yacy.search.SwitchboardConstants; import net.yacy.search.query.SearchEventCache; import net.yacy.search.schema.CollectionSchema; import net.yacy.server.serverObjects; @@ -215,7 +216,8 @@ public class CrawlStartScanner_p Domains.LOCALHOST, (int) sb.getConfigLong("port", 8090), path, - pk); + pk, + sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, "")); } } } @@ -260,7 +262,8 @@ public class CrawlStartScanner_p Domains.LOCALHOST, (int) sb.getConfigLong("port", 8090), path, - u.hash()); + u.hash(), + sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, "")); } } catch (final MalformedURLException e ) { ConcurrentLog.logException(e); diff --git a/htroot/Table_API_p.java b/htroot/Table_API_p.java index d755503cd..94455df8c 100644 --- a/htroot/Table_API_p.java +++ b/htroot/Table_API_p.java @@ -206,7 +206,7 @@ public class Table_API_p { } // now call the api URLs and store the result status - final Map l = sb.tables.execAPICalls(Domains.LOCALHOST, (int) sb.getConfigLong("port", 8090), pks); + final Map l = sb.tables.execAPICalls(Domains.LOCALHOST, (int) sb.getConfigLong("port", 8090), pks, sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, "")); // construct result table prop.put("showexec", l.isEmpty() ? 0 : 1); diff --git a/htroot/sharedBlacklist_p.java b/htroot/sharedBlacklist_p.java index 0bbea85fb..4e0da60f4 100644 --- a/htroot/sharedBlacklist_p.java +++ b/htroot/sharedBlacklist_p.java @@ -51,6 +51,7 @@ import net.yacy.peers.Seed; import net.yacy.repository.Blacklist; import net.yacy.repository.Blacklist.BlacklistType; import net.yacy.search.Switchboard; +import net.yacy.search.SwitchboardConstants; import net.yacy.search.query.SearchEventCache; import net.yacy.server.serverObjects; import net.yacy.server.serverSwitch; @@ -139,7 +140,7 @@ public class sharedBlacklist_p { // get List final DigestURL u = new DigestURL(downloadURLOld); - otherBlacklist = FileUtils.strings(u.get(agent)); + otherBlacklist = FileUtils.strings(u.get(agent, sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, ""))); } catch (final Exception e) { prop.put("status", STATUS_PEER_UNKNOWN); prop.putHTML("status_name", hash); @@ -156,7 +157,7 @@ public class sharedBlacklist_p { try { final DigestURL u = new DigestURL(downloadURL); - otherBlacklist = FileUtils.strings(u.get(agent)); + otherBlacklist = FileUtils.strings(u.get(agent, sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, ""))); } catch (final Exception e) { prop.put("status", STATUS_URL_PROBLEM); prop.putHTML("status_address",downloadURL); diff --git a/source/net/yacy/contentcontrol/SMWListSyncThread.java b/source/net/yacy/contentcontrol/SMWListSyncThread.java index b4d211155..a1957cdf1 100644 --- a/source/net/yacy/contentcontrol/SMWListSyncThread.java +++ b/source/net/yacy/contentcontrol/SMWListSyncThread.java @@ -79,7 +79,7 @@ public class SMWListSyncThread { + "/limit%3D200000" + "/format%3Dystat"); - String reply = UTF8.String(new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent).GETbytes(urlCount.toString())); + String reply = UTF8.String(new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent).GETbytes(urlCount.toString(), null)); String overallcount = reply.split(",")[0]; String lastsyncstring = reply.split(",")[1]; this.currentmax = Integer.parseInt(overallcount); diff --git a/source/net/yacy/cora/document/id/MultiProtocolURL.java b/source/net/yacy/cora/document/id/MultiProtocolURL.java index b901c0d7c..da0fa29ae 100644 --- a/source/net/yacy/cora/document/id/MultiProtocolURL.java +++ b/source/net/yacy/cora/document/id/MultiProtocolURL.java @@ -2046,7 +2046,7 @@ public class MultiProtocolURL implements Serializable, Comparable execAPICalls(String host, int port, Collection pks) { + public Map execAPICalls(String host, int port, Collection pks, final String pass) { // now call the api URLs and store the result status final HTTPClient client = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent); client.setTimout(120000); @@ -238,7 +238,7 @@ public class WorkTables extends Tables { url += "&" + WorkTables.TABLE_API_COL_APICALL_PK + "=" + UTF8.String(row.getPK()); ConcurrentLog.info("WorkTables", "executing url: " + url); try { - client.GETbytes(url); + client.GETbytes(url, pass); l.put(url, client.getStatusCode()); } catch (final IOException e) { ConcurrentLog.logException(e); @@ -248,14 +248,14 @@ public class WorkTables extends Tables { return l; } - public static int execAPICall(String host, int port, String path, byte[] pk) { + public static int execAPICall(String host, int port, String path, byte[] pk, final String pass) { // now call the api URLs and store the result status final HTTPClient client = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent); client.setTimout(120000); String url = "http://" + host + ":" + port + path; if (pk != null) url += "&" + WorkTables.TABLE_API_COL_APICALL_PK + "=" + UTF8.String(pk); try { - client.GETbytes(url); + client.GETbytes(url, pass); return client.getStatusCode(); } catch (final IOException e) { ConcurrentLog.logException(e); @@ -271,10 +271,10 @@ public class WorkTables extends Tables { * @param realm authentification realm * @return the http status code of the api call or -1 if any other IOException occurred */ - public int execAPICall(String pk, String host, int port) { + public int execAPICall(String pk, String host, int port, final String pass) { ArrayList pks = new ArrayList(); pks.add(pk); - Map m = execAPICalls(host, port, pks); + Map m = execAPICalls(host, port, pks, pass); if (m.isEmpty()) return -1; return m.values().iterator().next().intValue(); } diff --git a/source/net/yacy/data/ymark/YMarkCrawlStart.java b/source/net/yacy/data/ymark/YMarkCrawlStart.java index 2c2dc3681..3951bd632 100644 --- a/source/net/yacy/data/ymark/YMarkCrawlStart.java +++ b/source/net/yacy/data/ymark/YMarkCrawlStart.java @@ -110,8 +110,8 @@ public class YMarkCrawlStart extends HashMap{ } } - public int exec(final String host, final int port) { - return this.worktables.execAPICall(this.apicall_pk, host, port); + public int exec(final String host, final int port, final String pass) { + return this.worktables.execAPICall(this.apicall_pk, host, port, pass); } private void load() { diff --git a/source/net/yacy/document/parser/htmlParser.java b/source/net/yacy/document/parser/htmlParser.java index d844b1d58..2d9faee3e 100644 --- a/source/net/yacy/document/parser/htmlParser.java +++ b/source/net/yacy/document/parser/htmlParser.java @@ -305,7 +305,7 @@ public class htmlParser extends AbstractParser implements Parser { AnchorURL url; try { url = new AnchorURL(args[0]); - final byte[] content = url.get(ClientIdentification.yacyInternetCrawlerAgent); + final byte[] content = url.get(ClientIdentification.yacyInternetCrawlerAgent, null); final Document[] document = new htmlParser().parse(url, "text/html", null, new ByteArrayInputStream(content)); final String title = document[0].dc_title(); System.out.println(title); diff --git a/source/net/yacy/http/YaCyLegacyCredential.java b/source/net/yacy/http/YaCyLegacyCredential.java index 11a133b52..3e139a4ae 100644 --- a/source/net/yacy/http/YaCyLegacyCredential.java +++ b/source/net/yacy/http/YaCyLegacyCredential.java @@ -48,7 +48,7 @@ public class YaCyLegacyCredential extends Credential { * @param clear password * @return hash string */ - private static String calcHash(String pw) { + public static String calcHash(String pw) { return Digest.encodeMD5Hex(Base64Order.standardCoder.encodeString(pw)); } diff --git a/source/net/yacy/peers/SeedDB.java b/source/net/yacy/peers/SeedDB.java index 2714a7356..d64dcc8ca 100644 --- a/source/net/yacy/peers/SeedDB.java +++ b/source/net/yacy/peers/SeedDB.java @@ -816,7 +816,7 @@ public final class SeedDB implements AlternativeDomainNames { byte[] content = null; try { // send request - content = client.GETbytes(seedURL); + content = client.GETbytes(seedURL, null); } catch (final Exception e) { throw new IOException("Unable to download seed file '" + seedURL + "'. " + e.getMessage()); } diff --git a/source/net/yacy/peers/operation/yacyRelease.java b/source/net/yacy/peers/operation/yacyRelease.java index 95f0c4a35..fae58ba2c 100644 --- a/source/net/yacy/peers/operation/yacyRelease.java +++ b/source/net/yacy/peers/operation/yacyRelease.java @@ -294,7 +294,7 @@ public final class yacyRelease extends yacyVersion { // download signature first, if public key is available try { if (this.publicKey != null) { - final byte[] signatureData = client.GETbytes(getUrl().toString() + ".sig"); + final byte[] signatureData = client.GETbytes(getUrl().toString() + ".sig", null); if (signatureData == null) { ConcurrentLog.warn("yacyVersion", "download of signature " + getUrl().toString() + " failed. ignoring signature file."); } diff --git a/source/net/yacy/search/Switchboard.java b/source/net/yacy/search/Switchboard.java index 6d32f401f..da879767b 100644 --- a/source/net/yacy/search/Switchboard.java +++ b/source/net/yacy/search/Switchboard.java @@ -210,6 +210,7 @@ import net.yacy.utils.crypt; import com.google.common.io.Files; import net.yacy.http.YaCyHttpServer; +import net.yacy.http.YaCyLegacyCredential; public final class Switchboard extends serverSwitch { @@ -2457,7 +2458,7 @@ public final class Switchboard extends serverSwitch { startupAction = false; // execute api calls - final Map callResult = this.tables.execAPICalls("localhost", (int) getConfigLong("port", 8090), pks); + final Map callResult = this.tables.execAPICalls("localhost", (int) getConfigLong("port", 8090), pks, getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, "")); for ( final Map.Entry call : callResult.entrySet() ) { this.log.info("Scheduler executed api call, response " + call.getValue() + ": " + call.getKey()); } @@ -3235,6 +3236,7 @@ public final class Switchboard extends serverSwitch { public int adminAuthenticated(final RequestHeader requestHeader) { // authorization in case that there is no account stored + final String adminAccountUserName = getConfig(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME, "admin"); final String adminAccountBase64MD5 = getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, ""); if ( adminAccountBase64MD5.isEmpty() ) { adminAuthenticationLastAccess = System.currentTimeMillis(); @@ -3258,7 +3260,8 @@ public final class Switchboard extends serverSwitch { } // authorization by encoded password, only for localhost access - if ( accessFromLocalhost && (adminAccountBase64MD5.equals(realmValue)) ) { + String pass = Base64Order.standardCoder.encodeString(adminAccountUserName + ":" + adminAccountBase64MD5); + if ( accessFromLocalhost && (pass.equals(realmValue)) ) { adminAuthenticationLastAccess = System.currentTimeMillis(); return 3; // soft-authenticated for localhost } @@ -3734,7 +3737,7 @@ public final class Switchboard extends serverSwitch { } } scc.incrementAndGet(); - final byte[] content = client.GETbytes(url); + final byte[] content = client.GETbytes(url, null); Iterator enu = FileUtils.strings(content); int lc = 0; while ( enu.hasNext() ) { diff --git a/source/net/yacy/search/index/DocumentIndex.java b/source/net/yacy/search/index/DocumentIndex.java index f30cfc26b..d402b7ca8 100644 --- a/source/net/yacy/search/index/DocumentIndex.java +++ b/source/net/yacy/search/index/DocumentIndex.java @@ -153,7 +153,7 @@ public class DocumentIndex extends Segment { length = -1; } try { - documents = TextParser.parseSource(url, null, null, length, url.getInputStream(ClientIdentification.yacyInternetCrawlerAgent)); + documents = TextParser.parseSource(url, null, null, length, url.getInputStream(ClientIdentification.yacyInternetCrawlerAgent, null)); } catch (final Exception e ) { throw new IOException("cannot parse " + url.toString() + ": " + e.getMessage()); } diff --git a/source/net/yacy/server/serverSwitch.java b/source/net/yacy/server/serverSwitch.java index 1f4208e69..d6acf5e6a 100644 --- a/source/net/yacy/server/serverSwitch.java +++ b/source/net/yacy/server/serverSwitch.java @@ -50,6 +50,7 @@ import net.yacy.http.YaCyHttpServer; import net.yacy.kelondro.util.FileUtils; import net.yacy.kelondro.workflow.BusyThread; import net.yacy.kelondro.workflow.WorkflowThread; +import net.yacy.search.SwitchboardConstants; public class serverSwitch { @@ -536,7 +537,7 @@ public class serverSwitch reqHeader.put(HeaderFramework.USER_AGENT, ClientIdentification.yacyInternetCrawlerAgent.userAgent); final HTTPClient client = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent); client.setHeader(reqHeader.entrySet()); - byte[] data = client.GETbytes(uri); + byte[] data = client.GETbytes(uri, getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, "")); if ( data == null || data.length == 0 ) { continue; } diff --git a/source/net/yacy/utils/loaderThreads.java b/source/net/yacy/utils/loaderThreads.java index 0deaa9960..de547e76f 100644 --- a/source/net/yacy/utils/loaderThreads.java +++ b/source/net/yacy/utils/loaderThreads.java @@ -122,7 +122,7 @@ public class loaderThreads { @Override public void run() { try { - this.page = this.url.get(this.agent); + this.page = this.url.get(this.agent, null); this.loaded = true; this.process.feed(this.page); if (this.process.status() == loaderCore.STATUS_FAILED) { diff --git a/source/net/yacy/yacy.java b/source/net/yacy/yacy.java index efcf95a7b..5da965ef9 100644 --- a/source/net/yacy/yacy.java +++ b/source/net/yacy/yacy.java @@ -549,38 +549,20 @@ public final class yacy { // send 'wget' to web interface final RequestHeader requestHeader = new RequestHeader(); - requestHeader.put(RequestHeader.AUTHORIZATION, "Basic " + encodedPassword); // for http-authentify -// final Client con = new Client(10000, requestHeader); final HTTPClient con = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent); con.setHeader(requestHeader.entrySet()); -// ResponseContainer res = null; try { -// res = con.GET("http://localhost:"+ port +"/" + path); - con.GETbytes("http://localhost:"+ port +"/" + path); - - // read response -// if (res.getStatusLine().startsWith("2")) { + con.GETbytes("http://localhost:"+ port +"/" + path, encodedPassword); if (con.getStatusCode() > 199 && con.getStatusCode() < 300) { ConcurrentLog.config("COMMAND-STEERING", "YACY accepted steering command: " + processdescription); -// final ByteArrayOutputStream bos = new ByteArrayOutputStream(); //This is stream is not used??? -// try { -// FileUtils.copyToStream(new BufferedInputStream(res.getDataAsStream()), new BufferedOutputStream(bos)); -// } finally { -// res.closeStream(); -// } + } else { -// Log.logSevere("COMMAND-STEERING", "error response from YACY socket: " + res.getStatusLine()); ConcurrentLog.severe("COMMAND-STEERING", "error response from YACY socket: " + con.getHttpResponse().getStatusLine()); System.exit(-1); } } catch (final IOException e) { ConcurrentLog.severe("COMMAND-STEERING", "could not establish connection to YACY socket: " + e.getMessage()); System.exit(-1); -// } finally { -// // release connection -// if(res != null) { -// res.closeStream(); -// } } try {