From 6932aa4d7adbbf11219b7a7a3a2d86f8f9fd5257 Mon Sep 17 00:00:00 2001 From: reger Date: Tue, 7 Jan 2014 21:26:50 +0100 Subject: [PATCH] use configured admin-username for api calls - the admin user name can be configured, in apiExec calls the default "admin" username is used. TODO: the bin/apicall.sh script should likely take that into account. --- htroot/ConfigAppearance_p.java | 2 +- htroot/ConfigLanguage_p.java | 2 +- htroot/CrawlStartScanner_p.java | 2 ++ htroot/Table_API_p.java | 2 +- htroot/sharedBlacklist_p.java | 4 ++-- .../yacy/contentcontrol/SMWListSyncThread.java | 2 +- .../cora/document/id/MultiProtocolURL.java | 8 ++++---- .../yacy/cora/federate/yacy/api/Network.java | 2 +- .../yacy/cora/protocol/http/HTTPClient.java | 18 +++++++++--------- .../net/yacy/crawler/retrieval/FileLoader.java | 2 +- .../net/yacy/crawler/retrieval/HTTPLoader.java | 4 ++-- .../net/yacy/crawler/retrieval/SMBLoader.java | 2 +- source/net/yacy/data/WorkTables.java | 12 ++++++------ .../net/yacy/data/ymark/YMarkCrawlStart.java | 4 ++-- .../net/yacy/document/parser/htmlParser.java | 2 +- source/net/yacy/peers/SeedDB.java | 2 +- .../net/yacy/peers/operation/yacyRelease.java | 2 +- source/net/yacy/search/Switchboard.java | 4 ++-- .../net/yacy/search/index/DocumentIndex.java | 2 +- source/net/yacy/server/serverSwitch.java | 2 +- source/net/yacy/utils/loaderThreads.java | 2 +- source/net/yacy/yacy.java | 2 +- 22 files changed, 43 insertions(+), 41 deletions(-) diff --git a/htroot/ConfigAppearance_p.java b/htroot/ConfigAppearance_p.java index 362012421..e7495bb6b 100644 --- a/htroot/ConfigAppearance_p.java +++ b/htroot/ConfigAppearance_p.java @@ -102,7 +102,7 @@ public class ConfigAppearance_p { final Iterator it; try { final DigestURL u = new DigestURL(url); - it = FileUtils.strings(u.get(ClientIdentification.yacyInternetCrawlerAgent, sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, ""))); + it = FileUtils.strings(u.get(ClientIdentification.yacyInternetCrawlerAgent, sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME, "admin"), sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, ""))); } catch (final IOException e) { prop.put("status", "1");// unable to get URL prop.put("status_url", url); diff --git a/htroot/ConfigLanguage_p.java b/htroot/ConfigLanguage_p.java index e2239c3e2..65d2b076c 100644 --- a/htroot/ConfigLanguage_p.java +++ b/htroot/ConfigLanguage_p.java @@ -102,7 +102,7 @@ public class ConfigLanguage_p { Iterator it; try { final DigestURL u = new DigestURL(url); - it = FileUtils.strings(u.get(ClientIdentification.yacyInternetCrawlerAgent, sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, ""))); + it = FileUtils.strings(u.get(ClientIdentification.yacyInternetCrawlerAgent, sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME, "admin"), sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, ""))); } catch(final IOException e) { prop.put("status", "1");//unable to get url prop.put("status_url", url); diff --git a/htroot/CrawlStartScanner_p.java b/htroot/CrawlStartScanner_p.java index 694686b9c..420657343 100644 --- a/htroot/CrawlStartScanner_p.java +++ b/htroot/CrawlStartScanner_p.java @@ -217,6 +217,7 @@ public class CrawlStartScanner_p (int) sb.getConfigLong("port", 8090), path, pk, + sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME, "admin"), sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, "")); } } @@ -263,6 +264,7 @@ public class CrawlStartScanner_p (int) sb.getConfigLong("port", 8090), path, u.hash(), + sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME, "admin"), sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, "")); } } catch (final MalformedURLException e ) { diff --git a/htroot/Table_API_p.java b/htroot/Table_API_p.java index 94455df8c..8749d0fd9 100644 --- a/htroot/Table_API_p.java +++ b/htroot/Table_API_p.java @@ -206,7 +206,7 @@ public class Table_API_p { } // now call the api URLs and store the result status - final Map l = sb.tables.execAPICalls(Domains.LOCALHOST, (int) sb.getConfigLong("port", 8090), pks, sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, "")); + final Map l = sb.tables.execAPICalls(Domains.LOCALHOST, (int) sb.getConfigLong("port", 8090), pks, sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME, "admin"), sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, "")); // construct result table prop.put("showexec", l.isEmpty() ? 0 : 1); diff --git a/htroot/sharedBlacklist_p.java b/htroot/sharedBlacklist_p.java index 4e0da60f4..cffeed8f2 100644 --- a/htroot/sharedBlacklist_p.java +++ b/htroot/sharedBlacklist_p.java @@ -140,7 +140,7 @@ public class sharedBlacklist_p { // get List final DigestURL u = new DigestURL(downloadURLOld); - otherBlacklist = FileUtils.strings(u.get(agent, sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, ""))); + otherBlacklist = FileUtils.strings(u.get(agent, sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME, "admin"), sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, ""))); } catch (final Exception e) { prop.put("status", STATUS_PEER_UNKNOWN); prop.putHTML("status_name", hash); @@ -157,7 +157,7 @@ public class sharedBlacklist_p { try { final DigestURL u = new DigestURL(downloadURL); - otherBlacklist = FileUtils.strings(u.get(agent, sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, ""))); + otherBlacklist = FileUtils.strings(u.get(agent, sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME, "admin"), sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, ""))); } catch (final Exception e) { prop.put("status", STATUS_URL_PROBLEM); prop.putHTML("status_address",downloadURL); diff --git a/source/net/yacy/contentcontrol/SMWListSyncThread.java b/source/net/yacy/contentcontrol/SMWListSyncThread.java index a1957cdf1..6d611f14a 100644 --- a/source/net/yacy/contentcontrol/SMWListSyncThread.java +++ b/source/net/yacy/contentcontrol/SMWListSyncThread.java @@ -79,7 +79,7 @@ public class SMWListSyncThread { + "/limit%3D200000" + "/format%3Dystat"); - String reply = UTF8.String(new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent).GETbytes(urlCount.toString(), null)); + String reply = UTF8.String(new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent).GETbytes(urlCount.toString(), null, null)); String overallcount = reply.split(",")[0]; String lastsyncstring = reply.split(",")[1]; this.currentmax = Integer.parseInt(overallcount); diff --git a/source/net/yacy/cora/document/id/MultiProtocolURL.java b/source/net/yacy/cora/document/id/MultiProtocolURL.java index da0fa29ae..eb3dd1068 100644 --- a/source/net/yacy/cora/document/id/MultiProtocolURL.java +++ b/source/net/yacy/cora/document/id/MultiProtocolURL.java @@ -2046,7 +2046,7 @@ public class MultiProtocolURL implements Serializable, Comparable execAPICalls(String host, int port, Collection pks, final String pass) { + public Map execAPICalls(String host, int port, Collection pks, final String username, final String pass) { // now call the api URLs and store the result status final HTTPClient client = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent); client.setTimout(120000); @@ -238,7 +238,7 @@ public class WorkTables extends Tables { url += "&" + WorkTables.TABLE_API_COL_APICALL_PK + "=" + UTF8.String(row.getPK()); ConcurrentLog.info("WorkTables", "executing url: " + url); try { - client.GETbytes(url, pass); + client.GETbytes(url, username, pass); l.put(url, client.getStatusCode()); } catch (final IOException e) { ConcurrentLog.logException(e); @@ -248,14 +248,14 @@ public class WorkTables extends Tables { return l; } - public static int execAPICall(String host, int port, String path, byte[] pk, final String pass) { + public static int execAPICall(String host, int port, String path, byte[] pk, final String username, final String pass) { // now call the api URLs and store the result status final HTTPClient client = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent); client.setTimout(120000); String url = "http://" + host + ":" + port + path; if (pk != null) url += "&" + WorkTables.TABLE_API_COL_APICALL_PK + "=" + UTF8.String(pk); try { - client.GETbytes(url, pass); + client.GETbytes(url, username, pass); return client.getStatusCode(); } catch (final IOException e) { ConcurrentLog.logException(e); @@ -271,10 +271,10 @@ public class WorkTables extends Tables { * @param realm authentification realm * @return the http status code of the api call or -1 if any other IOException occurred */ - public int execAPICall(String pk, String host, int port, final String pass) { + public int execAPICall(String pk, String host, int port, final String username, final String pass) { ArrayList pks = new ArrayList(); pks.add(pk); - Map m = execAPICalls(host, port, pks, pass); + Map m = execAPICalls(host, port, pks, username, pass); if (m.isEmpty()) return -1; return m.values().iterator().next().intValue(); } diff --git a/source/net/yacy/data/ymark/YMarkCrawlStart.java b/source/net/yacy/data/ymark/YMarkCrawlStart.java index 3951bd632..753c77ff6 100644 --- a/source/net/yacy/data/ymark/YMarkCrawlStart.java +++ b/source/net/yacy/data/ymark/YMarkCrawlStart.java @@ -110,8 +110,8 @@ public class YMarkCrawlStart extends HashMap{ } } - public int exec(final String host, final int port, final String pass) { - return this.worktables.execAPICall(this.apicall_pk, host, port, pass); + public int exec(final String host, final int port, final String username, final String pass) { + return this.worktables.execAPICall(this.apicall_pk, host, port, username, pass); } private void load() { diff --git a/source/net/yacy/document/parser/htmlParser.java b/source/net/yacy/document/parser/htmlParser.java index 2d9faee3e..1241c7188 100644 --- a/source/net/yacy/document/parser/htmlParser.java +++ b/source/net/yacy/document/parser/htmlParser.java @@ -305,7 +305,7 @@ public class htmlParser extends AbstractParser implements Parser { AnchorURL url; try { url = new AnchorURL(args[0]); - final byte[] content = url.get(ClientIdentification.yacyInternetCrawlerAgent, null); + final byte[] content = url.get(ClientIdentification.yacyInternetCrawlerAgent, null, null); final Document[] document = new htmlParser().parse(url, "text/html", null, new ByteArrayInputStream(content)); final String title = document[0].dc_title(); System.out.println(title); diff --git a/source/net/yacy/peers/SeedDB.java b/source/net/yacy/peers/SeedDB.java index d64dcc8ca..b86d5c5d5 100644 --- a/source/net/yacy/peers/SeedDB.java +++ b/source/net/yacy/peers/SeedDB.java @@ -816,7 +816,7 @@ public final class SeedDB implements AlternativeDomainNames { byte[] content = null; try { // send request - content = client.GETbytes(seedURL, null); + content = client.GETbytes(seedURL, null, null); } catch (final Exception e) { throw new IOException("Unable to download seed file '" + seedURL + "'. " + e.getMessage()); } diff --git a/source/net/yacy/peers/operation/yacyRelease.java b/source/net/yacy/peers/operation/yacyRelease.java index fae58ba2c..dc5aedf32 100644 --- a/source/net/yacy/peers/operation/yacyRelease.java +++ b/source/net/yacy/peers/operation/yacyRelease.java @@ -294,7 +294,7 @@ public final class yacyRelease extends yacyVersion { // download signature first, if public key is available try { if (this.publicKey != null) { - final byte[] signatureData = client.GETbytes(getUrl().toString() + ".sig", null); + final byte[] signatureData = client.GETbytes(getUrl().toString() + ".sig", null, null); if (signatureData == null) { ConcurrentLog.warn("yacyVersion", "download of signature " + getUrl().toString() + " failed. ignoring signature file."); } diff --git a/source/net/yacy/search/Switchboard.java b/source/net/yacy/search/Switchboard.java index a364165de..8e15ab0a8 100644 --- a/source/net/yacy/search/Switchboard.java +++ b/source/net/yacy/search/Switchboard.java @@ -2460,7 +2460,7 @@ public final class Switchboard extends serverSwitch { startupAction = false; // execute api calls - final Map callResult = this.tables.execAPICalls("localhost", (int) getConfigLong("port", 8090), pks, getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, "")); + final Map callResult = this.tables.execAPICalls("localhost", (int) getConfigLong("port", 8090), pks, getConfig(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME, "admin"), getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, "")); for ( final Map.Entry call : callResult.entrySet() ) { this.log.info("Scheduler executed api call, response " + call.getValue() + ": " + call.getKey()); } @@ -3739,7 +3739,7 @@ public final class Switchboard extends serverSwitch { } } scc.incrementAndGet(); - final byte[] content = client.GETbytes(url, null); + final byte[] content = client.GETbytes(url, null, null); Iterator enu = FileUtils.strings(content); int lc = 0; while ( enu.hasNext() ) { diff --git a/source/net/yacy/search/index/DocumentIndex.java b/source/net/yacy/search/index/DocumentIndex.java index 5cb53e546..46b8a5960 100644 --- a/source/net/yacy/search/index/DocumentIndex.java +++ b/source/net/yacy/search/index/DocumentIndex.java @@ -153,7 +153,7 @@ public class DocumentIndex extends Segment { length = -1; } try { - documents = TextParser.parseSource(url, null, null, length, url.getInputStream(ClientIdentification.yacyInternetCrawlerAgent, null)); + documents = TextParser.parseSource(url, null, null, length, url.getInputStream(ClientIdentification.yacyInternetCrawlerAgent, null, null)); } catch (final Exception e ) { throw new IOException("cannot parse " + url.toString() + ": " + e.getMessage()); } diff --git a/source/net/yacy/server/serverSwitch.java b/source/net/yacy/server/serverSwitch.java index d6acf5e6a..2ed89ecce 100644 --- a/source/net/yacy/server/serverSwitch.java +++ b/source/net/yacy/server/serverSwitch.java @@ -537,7 +537,7 @@ public class serverSwitch reqHeader.put(HeaderFramework.USER_AGENT, ClientIdentification.yacyInternetCrawlerAgent.userAgent); final HTTPClient client = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent); client.setHeader(reqHeader.entrySet()); - byte[] data = client.GETbytes(uri, getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, "")); + byte[] data = client.GETbytes(uri, getConfig(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME, "admin"),getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, "")); if ( data == null || data.length == 0 ) { continue; } diff --git a/source/net/yacy/utils/loaderThreads.java b/source/net/yacy/utils/loaderThreads.java index de547e76f..56c965390 100644 --- a/source/net/yacy/utils/loaderThreads.java +++ b/source/net/yacy/utils/loaderThreads.java @@ -122,7 +122,7 @@ public class loaderThreads { @Override public void run() { try { - this.page = this.url.get(this.agent, null); + this.page = this.url.get(this.agent, null, null); this.loaded = true; this.process.feed(this.page); if (this.process.status() == loaderCore.STATUS_FAILED) { diff --git a/source/net/yacy/yacy.java b/source/net/yacy/yacy.java index e1beb3a03..6fef1bfd8 100644 --- a/source/net/yacy/yacy.java +++ b/source/net/yacy/yacy.java @@ -523,7 +523,7 @@ public final class yacy { final HTTPClient con = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent); con.setHeader(requestHeader.entrySet()); try { - con.GETbytes("http://localhost:"+ port +"/" + path, encodedPassword); + con.GETbytes("http://localhost:"+ port +"/" + path, sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME,"admin"), encodedPassword); if (con.getStatusCode() > 199 && con.getStatusCode() < 300) { ConcurrentLog.config("COMMAND-STEERING", "YACY accepted steering command: " + processdescription);