boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

fixed shutdown authenticaton problem

Browse Source
pull/1/head
orbiter 11 years ago
parent ed06b5b94b
commit 3cb6c7861f
23 changed files with 56 additions and 67 deletions
Show Stats Download Patch File Download Diff File

2
htroot/ConfigAppearance_p.java
Unescape View File

@ -102,7 +102,7 @@ public class ConfigAppearance_p {
final Iterator<String> it;
try {
final DigestURL u = new DigestURL(url);
it = FileUtils.strings(u.get(ClientIdentification.yacyInternetCrawlerAgent));
it = FileUtils.strings(u.get(ClientIdentification.yacyInternetCrawlerAgent, sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, "")));
} catch (final IOException e) {
prop.put("status", "1");// unable to get URL
prop.put("status_url", url);

3
htroot/ConfigLanguage_p.java
Unescape View File

@ -46,6 +46,7 @@ import net.yacy.data.Translator;
import net.yacy.data.WorkTables;
import net.yacy.kelondro.util.FileUtils;
import net.yacy.search.Switchboard;
import net.yacy.search.SwitchboardConstants;
import net.yacy.server.serverObjects;
import net.yacy.server.serverSwitch;
@ -101,7 +102,7 @@ public class ConfigLanguage_p {
Iterator<String> it;
try {
final DigestURL u = new DigestURL(url);
it = FileUtils.strings(u.get(ClientIdentification.yacyInternetCrawlerAgent));
it = FileUtils.strings(u.get(ClientIdentification.yacyInternetCrawlerAgent, sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, "")));
} catch(final IOException e) {
prop.put("status", "1");//unable to get url
prop.put("status_url", url);

7
htroot/CrawlStartScanner_p.java
Unescape View File

@ -42,6 +42,7 @@ import net.yacy.cora.sorting.ReversibleScoreMap;
import net.yacy.cora.util.ConcurrentLog;
import net.yacy.data.WorkTables;
import net.yacy.search.Switchboard;
import net.yacy.search.SwitchboardConstants;
import net.yacy.search.query.SearchEventCache;
import net.yacy.search.schema.CollectionSchema;
import net.yacy.server.serverObjects;
@ -215,7 +216,8 @@ public class CrawlStartScanner_p
Domains.LOCALHOST,
(int) sb.getConfigLong("port", 8090),
path,
pk);
pk,
sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, ""));
}
}
}
@ -260,7 +262,8 @@ public class CrawlStartScanner_p
Domains.LOCALHOST,
(int) sb.getConfigLong("port", 8090),
path,
u.hash());
u.hash(),
sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, ""));
}
} catch (final MalformedURLException e ) {
ConcurrentLog.logException(e);

2
htroot/Table_API_p.java
Unescape View File

@ -206,7 +206,7 @@ public class Table_API_p {
}
// now call the api URLs and store the result status
final Map<String, Integer> l = sb.tables.execAPICalls(Domains.LOCALHOST, (int) sb.getConfigLong("port", 8090), pks);
final Map<String, Integer> l = sb.tables.execAPICalls(Domains.LOCALHOST, (int) sb.getConfigLong("port", 8090), pks, sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, ""));
// construct result table
prop.put("showexec", l.isEmpty() ? 0 : 1);

5
htroot/sharedBlacklist_p.java
Unescape View File

@ -51,6 +51,7 @@ import net.yacy.peers.Seed;
import net.yacy.repository.Blacklist;
import net.yacy.repository.Blacklist.BlacklistType;
import net.yacy.search.Switchboard;
import net.yacy.search.SwitchboardConstants;
import net.yacy.search.query.SearchEventCache;
import net.yacy.server.serverObjects;
import net.yacy.server.serverSwitch;
@ -139,7 +140,7 @@ public class sharedBlacklist_p {
// get List
final DigestURL u = new DigestURL(downloadURLOld);
otherBlacklist = FileUtils.strings(u.get(agent));
otherBlacklist = FileUtils.strings(u.get(agent, sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, "")));
} catch (final Exception e) {
prop.put("status", STATUS_PEER_UNKNOWN);
prop.putHTML("status_name", hash);
@ -156,7 +157,7 @@ public class sharedBlacklist_p {
try {
final DigestURL u = new DigestURL(downloadURL);
otherBlacklist = FileUtils.strings(u.get(agent));
otherBlacklist = FileUtils.strings(u.get(agent, sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, "")));
} catch (final Exception e) {
prop.put("status", STATUS_URL_PROBLEM);
prop.putHTML("status_address",downloadURL);

2
source/net/yacy/contentcontrol/SMWListSyncThread.java
Unescape View File

@ -79,7 +79,7 @@ public class SMWListSyncThread {
+ "/limit%3D200000"
+ "/format%3Dystat");
String reply = UTF8.String(new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent).GETbytes(urlCount.toString()));
String reply = UTF8.String(new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent).GETbytes(urlCount.toString(), null));
String overallcount = reply.split(",")[0];
String lastsyncstring = reply.split(",")[1];
this.currentmax = Integer.parseInt(overallcount);

8
source/net/yacy/cora/document/id/MultiProtocolURL.java
Unescape View File

@ -2046,7 +2046,7 @@ public class MultiProtocolURL implements Serializable, Comparable<MultiProtocolU
return null;
}
public InputStream getInputStream(final ClientIdentification.Agent agent) throws IOException {
public InputStream getInputStream(final ClientIdentification.Agent agent, final String pass) throws IOException {
if (isFile()) return new BufferedInputStream(new FileInputStream(getFSFile()));
if (isSMB()) return new BufferedInputStream(new SmbFileInputStream(getSmbFile()));
if (isFTP()) {
@ -2059,13 +2059,13 @@ public class MultiProtocolURL implements Serializable, Comparable<MultiProtocolU
if (isHTTP() || isHTTPS()) {
final HTTPClient client = new HTTPClient(agent);
client.setHost(getHost());
return new ByteArrayInputStream(client.GETbytes(this));
return new ByteArrayInputStream(client.GETbytes(this, pass));
}
return null;
}
public byte[] get(final ClientIdentification.Agent agent) throws IOException {
public byte[] get(final ClientIdentification.Agent agent, final String pass) throws IOException {
if (isFile()) return read(new FileInputStream(getFSFile()));
if (isSMB()) return read(new SmbFileInputStream(getSmbFile()));
if (isFTP()) {
@ -2078,7 +2078,7 @@ public class MultiProtocolURL implements Serializable, Comparable<MultiProtocolU
if (isHTTP() || isHTTPS()) {
final HTTPClient client = new HTTPClient(agent);
client.setHost(getHost());
return client.GETbytes(this);
return client.GETbytes(this, pass);
}
return null;

2
source/net/yacy/cora/federate/yacy/api/Network.java
Unescape View File

@ -50,7 +50,7 @@ public class Network {
public static Peers getNetwork(final String address) throws IOException {
Peers peers = new Peers();
final HTTPClient httpclient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent);
final byte[] content = httpclient.GETbytes("http://" + address + "/Network.xml?page=1&maxCount=1000&ip=");
final byte[] content = httpclient.GETbytes("http://" + address + "/Network.xml?page=1&maxCount=1000&ip=", null);
ByteArrayInputStream bais = new ByteArrayInputStream(content);
Document doc = null;
try {

22
source/net/yacy/cora/protocol/http/HTTPClient.java
Unescape View File

@ -49,8 +49,6 @@ import net.yacy.cora.protocol.ClientIdentification;
import net.yacy.cora.protocol.ConnectionInfo;
import net.yacy.cora.protocol.Domains;
import net.yacy.cora.protocol.HeaderFramework;
import net.yacy.search.Switchboard;
import net.yacy.search.SwitchboardConstants;
import org.apache.http.Header;
import org.apache.http.HttpEntity;
@ -309,8 +307,8 @@ public class HTTPClient {
* @return content bytes
* @throws IOException
*/
public byte[] GETbytes(final String uri) throws IOException {
return GETbytes(uri, Integer.MAX_VALUE);
public byte[] GETbytes(final String uri, final String pass) throws IOException {
return GETbytes(uri, pass, Integer.MAX_VALUE);
}
/**
@ -320,8 +318,8 @@ public class HTTPClient {
* @return content bytes
* @throws IOException
*/
public byte[] GETbytes(final MultiProtocolURL url) throws IOException {
return GETbytes(url, Integer.MAX_VALUE);
public byte[] GETbytes(final MultiProtocolURL url, final String pass) throws IOException {
return GETbytes(url, pass, Integer.MAX_VALUE);
}
/**
@ -332,8 +330,8 @@ public class HTTPClient {
* @return content bytes
* @throws IOException
*/
public byte[] GETbytes(final String uri, final int maxBytes) throws IOException {
return GETbytes(new MultiProtocolURL(uri), maxBytes);
public byte[] GETbytes(final String uri, final String pass, final int maxBytes) throws IOException {
return GETbytes(new MultiProtocolURL(uri), pass, maxBytes);
}
@ -345,7 +343,7 @@ public class HTTPClient {
* @return content bytes
* @throws IOException
*/
public byte[] GETbytes(final MultiProtocolURL url, final int maxBytes) throws IOException {
public byte[] GETbytes(final MultiProtocolURL url, final String pass, final int maxBytes) throws IOException {
final boolean localhost = Domains.isLocalhost(url.getHost());
final String urix = url.toNormalform(true);
HttpGet httpGet = null;
@ -355,11 +353,11 @@ public class HTTPClient {
throw new IOException(e.getMessage()); // can be caused at java.net.URI.create()
}
if (!localhost) setHost(url.getHost()); // overwrite resolved IP, needed for shared web hosting DO NOT REMOVE, see http://en.wikipedia.org/wiki/Shared_web_hosting_service
if (localhost) {
if (localhost && pass != null) {
CredentialsProvider credsProvider = new BasicCredentialsProvider();
credsProvider.setCredentials(
AuthScope.ANY, // thats ok since we tested for localhost!
new UsernamePasswordCredentials("admin", Switchboard.getSwitchboard().getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, "")));
new UsernamePasswordCredentials("admin", pass));
CloseableHttpClient httpclient = HttpClients.custom().setDefaultCredentialsProvider(credsProvider).build();
byte[] content = null;
try {
@ -790,7 +788,7 @@ public class HTTPClient {
url = "http://" + url;
}
try {
System.out.println(UTF8.String(client.GETbytes(url)));
System.out.println(UTF8.String(client.GETbytes(url, null)));
} catch (final IOException e) {
e.printStackTrace();
}

2
source/net/yacy/crawler/retrieval/FileLoader.java
Unescape View File

@ -135,7 +135,7 @@ public class FileLoader {
}
// load the resource
InputStream is = url.getInputStream(ClientIdentification.yacyInternetCrawlerAgent);
InputStream is = url.getInputStream(ClientIdentification.yacyInternetCrawlerAgent, null);
byte[] b = FileUtils.read(is);
is.close();

4
source/net/yacy/crawler/retrieval/HTTPLoader.java
Unescape View File

@ -128,7 +128,7 @@ public final class HTTPLoader {
client.setHeader(requestHeader.entrySet());
// send request
final byte[] responseBody = client.GETbytes(url, maxFileSize);
final byte[] responseBody = client.GETbytes(url, sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, ""), maxFileSize);
final int statusCode = client.getHttpResponse().getStatusLine().getStatusCode();
final ResponseHeader responseHeader = new ResponseHeader(statusCode, client.getHttpResponse().getAllHeaders());
String requestURLString = request.url().toNormalform(true);
@ -243,7 +243,7 @@ public final class HTTPLoader {
final HTTPClient client = new HTTPClient(agent);
client.setTimout(20000);
client.setHeader(requestHeader.entrySet());
final byte[] responseBody = client.GETbytes(request.url());
final byte[] responseBody = client.GETbytes(request.url(), null);
final int code = client.getHttpResponse().getStatusLine().getStatusCode();
final ResponseHeader header = new ResponseHeader(code, client.getHttpResponse().getAllHeaders());
// FIXME: 30*-handling (bottom) is never reached

2
source/net/yacy/crawler/retrieval/SMBLoader.java
Unescape View File

@ -153,7 +153,7 @@ public class SMBLoader {
}
// load the resource
InputStream is = url.getInputStream(ClientIdentification.yacyInternetCrawlerAgent);
InputStream is = url.getInputStream(ClientIdentification.yacyInternetCrawlerAgent, null);
byte[] b = FileUtils.read(is);
is.close();

12
source/net/yacy/data/WorkTables.java
Unescape View File

@ -217,7 +217,7 @@ public class WorkTables extends Tables {
* @param port the port on the host
* @return a map of the called urls and the http status code of the api call or -1 if any other IOException occurred
*/
public Map<String, Integer> execAPICalls(String host, int port, Collection<String> pks) {
public Map<String, Integer> execAPICalls(String host, int port, Collection<String> pks, final String pass) {
// now call the api URLs and store the result status
final HTTPClient client = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent);
client.setTimout(120000);
@ -238,7 +238,7 @@ public class WorkTables extends Tables {
url += "&" + WorkTables.TABLE_API_COL_APICALL_PK + "=" + UTF8.String(row.getPK());
ConcurrentLog.info("WorkTables", "executing url: " + url);
try {
client.GETbytes(url);
client.GETbytes(url, pass);
l.put(url, client.getStatusCode());
} catch (final IOException e) {
ConcurrentLog.logException(e);
@ -248,14 +248,14 @@ public class WorkTables extends Tables {
return l;
}
public static int execAPICall(String host, int port, String path, byte[] pk) {
public static int execAPICall(String host, int port, String path, byte[] pk, final String pass) {
// now call the api URLs and store the result status
final HTTPClient client = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent);
client.setTimout(120000);
String url = "http://" + host + ":" + port + path;
if (pk != null) url += "&" + WorkTables.TABLE_API_COL_APICALL_PK + "=" + UTF8.String(pk);
try {
client.GETbytes(url);
client.GETbytes(url, pass);
return client.getStatusCode();
} catch (final IOException e) {
ConcurrentLog.logException(e);
@ -271,10 +271,10 @@ public class WorkTables extends Tables {
* @param realm authentification realm
* @return the http status code of the api call or -1 if any other IOException occurred
*/
public int execAPICall(String pk, String host, int port) {
public int execAPICall(String pk, String host, int port, final String pass) {
ArrayList<String> pks = new ArrayList<String>();
pks.add(pk);
Map<String, Integer> m = execAPICalls(host, port, pks);
Map<String, Integer> m = execAPICalls(host, port, pks, pass);
if (m.isEmpty()) return -1;
return m.values().iterator().next().intValue();
}

4
source/net/yacy/data/ymark/YMarkCrawlStart.java
Unescape View File

@ -110,8 +110,8 @@ public class YMarkCrawlStart extends HashMap<String,String>{
}
}
public int exec(final String host, final int port) {
return this.worktables.execAPICall(this.apicall_pk, host, port);
public int exec(final String host, final int port, final String pass) {
return this.worktables.execAPICall(this.apicall_pk, host, port, pass);
}
private void load() {

2
source/net/yacy/document/parser/htmlParser.java
Unescape View File

@ -305,7 +305,7 @@ public class htmlParser extends AbstractParser implements Parser {
AnchorURL url;
try {
url = new AnchorURL(args[0]);
final byte[] content = url.get(ClientIdentification.yacyInternetCrawlerAgent);
final byte[] content = url.get(ClientIdentification.yacyInternetCrawlerAgent, null);
final Document[] document = new htmlParser().parse(url, "text/html", null, new ByteArrayInputStream(content));
final String title = document[0].dc_title();
System.out.println(title);

2
source/net/yacy/http/YaCyLegacyCredential.java
Unescape View File

@ -48,7 +48,7 @@ public class YaCyLegacyCredential extends Credential {
* @param clear password
* @return hash string
*/
private static String calcHash(String pw) {
public static String calcHash(String pw) {
return Digest.encodeMD5Hex(Base64Order.standardCoder.encodeString(pw));
}

2
source/net/yacy/peers/SeedDB.java
Unescape View File

@ -816,7 +816,7 @@ public final class SeedDB implements AlternativeDomainNames {
byte[] content = null;
try {
// send request
content = client.GETbytes(seedURL);
content = client.GETbytes(seedURL, null);
} catch (final Exception e) {
throw new IOException("Unable to download seed file '" + seedURL + "'. " + e.getMessage());
}

2
source/net/yacy/peers/operation/yacyRelease.java
Unescape View File

@ -294,7 +294,7 @@ public final class yacyRelease extends yacyVersion {
// download signature first, if public key is available
try {
if (this.publicKey != null) {
final byte[] signatureData = client.GETbytes(getUrl().toString() + ".sig");
final byte[] signatureData = client.GETbytes(getUrl().toString() + ".sig", null);
if (signatureData == null) {
ConcurrentLog.warn("yacyVersion", "download of signature " + getUrl().toString() + " failed. ignoring signature file.");
}

9
source/net/yacy/search/Switchboard.java
Unescape View File

@ -210,6 +210,7 @@ import net.yacy.utils.crypt;
import com.google.common.io.Files;
import net.yacy.http.YaCyHttpServer;
import net.yacy.http.YaCyLegacyCredential;
public final class Switchboard extends serverSwitch {
@ -2457,7 +2458,7 @@ public final class Switchboard extends serverSwitch {
startupAction = false;
// execute api calls
final Map<String, Integer> callResult = this.tables.execAPICalls("localhost", (int) getConfigLong("port", 8090), pks);
final Map<String, Integer> callResult = this.tables.execAPICalls("localhost", (int) getConfigLong("port", 8090), pks, getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, ""));
for ( final Map.Entry<String, Integer> call : callResult.entrySet() ) {
this.log.info("Scheduler executed api call, response " + call.getValue() + ": " + call.getKey());
}
@ -3235,6 +3236,7 @@ public final class Switchboard extends serverSwitch {
public int adminAuthenticated(final RequestHeader requestHeader) {
// authorization in case that there is no account stored
final String adminAccountUserName = getConfig(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME, "admin");
final String adminAccountBase64MD5 = getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, "");
if ( adminAccountBase64MD5.isEmpty() ) {
adminAuthenticationLastAccess = System.currentTimeMillis();
@ -3258,7 +3260,8 @@ public final class Switchboard extends serverSwitch {
}
// authorization by encoded password, only for localhost access
if ( accessFromLocalhost && (adminAccountBase64MD5.equals(realmValue)) ) {
String pass = Base64Order.standardCoder.encodeString(adminAccountUserName + ":" + adminAccountBase64MD5);
if ( accessFromLocalhost && (pass.equals(realmValue)) ) {
adminAuthenticationLastAccess = System.currentTimeMillis();
return 3; // soft-authenticated for localhost
}
@ -3734,7 +3737,7 @@ public final class Switchboard extends serverSwitch {
}
}
scc.incrementAndGet();
final byte[] content = client.GETbytes(url);
final byte[] content = client.GETbytes(url, null);
Iterator<String> enu = FileUtils.strings(content);
int lc = 0;
while ( enu.hasNext() ) {

2
source/net/yacy/search/index/DocumentIndex.java
Unescape View File

@ -153,7 +153,7 @@ public class DocumentIndex extends Segment {
length = -1;
}
try {
documents = TextParser.parseSource(url, null, null, length, url.getInputStream(ClientIdentification.yacyInternetCrawlerAgent));
documents = TextParser.parseSource(url, null, null, length, url.getInputStream(ClientIdentification.yacyInternetCrawlerAgent, null));
} catch (final Exception e ) {
throw new IOException("cannot parse " + url.toString() + ": " + e.getMessage());
}

3
source/net/yacy/server/serverSwitch.java
Unescape View File

@ -50,6 +50,7 @@ import net.yacy.http.YaCyHttpServer;
import net.yacy.kelondro.util.FileUtils;
import net.yacy.kelondro.workflow.BusyThread;
import net.yacy.kelondro.workflow.WorkflowThread;
import net.yacy.search.SwitchboardConstants;
public class serverSwitch
{
@ -536,7 +537,7 @@ public class serverSwitch
reqHeader.put(HeaderFramework.USER_AGENT, ClientIdentification.yacyInternetCrawlerAgent.userAgent);
final HTTPClient client = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent);
client.setHeader(reqHeader.entrySet());
byte[] data = client.GETbytes(uri);
byte[] data = client.GETbytes(uri, getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, ""));
if ( data == null || data.length == 0 ) {
continue;
}

2
source/net/yacy/utils/loaderThreads.java
Unescape View File

@ -122,7 +122,7 @@ public class loaderThreads {
@Override
public void run() {
try {
this.page = this.url.get(this.agent);
this.page = this.url.get(this.agent, null);
this.loaded = true;
this.process.feed(this.page);
if (this.process.status() == loaderCore.STATUS_FAILED) {

22
source/net/yacy/yacy.java
Unescape View File

@ -549,38 +549,20 @@ public final class yacy {
// send 'wget' to web interface
final RequestHeader requestHeader = new RequestHeader();
requestHeader.put(RequestHeader.AUTHORIZATION, "Basic " + encodedPassword); // for http-authentify
// final Client con = new Client(10000, requestHeader);
final HTTPClient con = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent);
con.setHeader(requestHeader.entrySet());
// ResponseContainer res = null;
try {
// res = con.GET("http://localhost:"+ port +"/" + path);
con.GETbytes("http://localhost:"+ port +"/" + path);
// read response
// if (res.getStatusLine().startsWith("2")) {
con.GETbytes("http://localhost:"+ port +"/" + path, encodedPassword);
if (con.getStatusCode() > 199 && con.getStatusCode() < 300) {
ConcurrentLog.config("COMMAND-STEERING", "YACY accepted steering command: " + processdescription);
// final ByteArrayOutputStream bos = new ByteArrayOutputStream(); //This is stream is not used???
// try {
// FileUtils.copyToStream(new BufferedInputStream(res.getDataAsStream()), new BufferedOutputStream(bos));
// } finally {
// res.closeStream();
// }
} else {
// Log.logSevere("COMMAND-STEERING", "error response from YACY socket: " + res.getStatusLine());
ConcurrentLog.severe("COMMAND-STEERING", "error response from YACY socket: " + con.getHttpResponse().getStatusLine());
System.exit(-1);
}
} catch (final IOException e) {
ConcurrentLog.severe("COMMAND-STEERING", "could not establish connection to YACY socket: " + e.getMessage());
System.exit(-1);
// } finally {
// // release connection
// if(res != null) {
// res.closeStream();
// }
}
try {

Write Preview
Loading…
Cancel
Save