boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

Merge branch 'master' of ssh://git@gitorious.org/yacy/rc1.git

Browse Source
pull/1/head
Michael Peter Christen 11 years ago
parent 644573cfc4 6932aa4d7a
commit ce4d42d77c
28 changed files with 105 additions and 80 deletions
Show Stats Download Patch File Download Diff File

3
defaults/yacy.init
Unescape View File

@ -287,6 +287,9 @@ promoteSearchPageGreeting.smallImage = /env/grafics/YaCyLogo_60ppi.png
# when the secondary path should be equal to the primary, it must be declared empty
indexPrimaryPath=DATA/INDEX
# the path to index archive dumps
indexArchivePath=DATA/ARCHIVE
# the path to the LISTS files. Most lists are used to filter web content
listsPath=DATA/LISTS

51
htroot/ConfigAccounts_p.java
Unescape View File

@ -58,35 +58,46 @@ public class ConfigAccounts_p {
boolean localhostAccess = sb.getConfigBool(SwitchboardConstants.ADMIN_ACCOUNT_FOR_LOCALHOST, false);
if (post != null && post.containsKey("setAdmin")) {
localhostAccess = Domains.isLocalhost(post.get("access", ""));
final String user = (post == null) ? "" : post.get("adminuser", "");
final String pw1 = (post == null) ? "" : post.get("adminpw1", "");
final String pw2 = (post == null) ? "" : post.get("adminpw2", "");
final String user = post.get("adminuser", "");
final String pw1 = post.get("adminpw1", "");
final String pw2 = post.get("adminpw2", "");
int inputerror=0;
// may be overwritten if new password is given
if (user.length() > 0 && pw1.length() > 3 && pw1.equals(pw2)) {
// check passed. set account:
env.setConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, Digest.encodeMD5Hex(Base64Order.standardCoder.encodeString(user + ":" + pw1)));
env.setConfig(SwitchboardConstants.ADMIN_ACCOUNT, "");
env.setConfig(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME,user);
env.setConfig(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME,user);
} else {
if (!localhostAccess) {
if (user.isEmpty()) {
inputerror = 3;
} else {
inputerror = 2; // password match error
}
prop.put("error", inputerror);
}
}
if (localhostAccess) {
if (inputerror == 0) {
if (localhostAccess) {
sb.setConfig(SwitchboardConstants.ADMIN_ACCOUNT_FOR_LOCALHOST, true);
// if an localhost access is configured, check if a local password is given
// if not, set a random password
if (env.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, "").isEmpty()) {
// make a 'random' password
env.setConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, "0000" + sb.genRandomPassword());
env.setConfig(SwitchboardConstants.ADMIN_ACCOUNT, "");
}
} else {
sb.setConfig(SwitchboardConstants.ADMIN_ACCOUNT_FOR_LOCALHOST, false);
if (env.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, "").startsWith("0000")) {
// make shure that the user can still use the interface after a random password was set
env.setConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, "");
sb.setConfig(SwitchboardConstants.ADMIN_ACCOUNT_FOR_LOCALHOST, true);
// if an localhost access is configured, check if a local password is given
// if not, set a random password
if (env.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, "").isEmpty()) {
// make a 'random' password
env.setConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, "0000" + sb.genRandomPassword());
env.setConfig(SwitchboardConstants.ADMIN_ACCOUNT, "");
}
} else {
sb.setConfig(SwitchboardConstants.ADMIN_ACCOUNT_FOR_LOCALHOST, false);
if (env.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, "").startsWith("0000")) {
// make shure that the user can still use the interface after a random password was set
env.setConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, "");
}
}
}
}
}
if (env.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, "").isEmpty() && !env.getConfigBool(SwitchboardConstants.ADMIN_ACCOUNT_FOR_LOCALHOST, false)) {

2
htroot/ConfigAppearance_p.java
Unescape View File

@ -102,7 +102,7 @@ public class ConfigAppearance_p {
final Iterator<String> it;
try {
final DigestURL u = new DigestURL(url);
it = FileUtils.strings(u.get(ClientIdentification.yacyInternetCrawlerAgent, sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, "")));
it = FileUtils.strings(u.get(ClientIdentification.yacyInternetCrawlerAgent, sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME, "admin"), sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, "")));
} catch (final IOException e) {
prop.put("status", "1");// unable to get URL
prop.put("status_url", url);

2
htroot/ConfigLanguage_p.java
Unescape View File

@ -102,7 +102,7 @@ public class ConfigLanguage_p {
Iterator<String> it;
try {
final DigestURL u = new DigestURL(url);
it = FileUtils.strings(u.get(ClientIdentification.yacyInternetCrawlerAgent, sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, "")));
it = FileUtils.strings(u.get(ClientIdentification.yacyInternetCrawlerAgent, sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME, "admin"), sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, "")));
} catch(final IOException e) {
prop.put("status", "1");//unable to get url
prop.put("status_url", url);

2
htroot/CrawlStartScanner_p.java
Unescape View File

@ -217,6 +217,7 @@ public class CrawlStartScanner_p
(int) sb.getConfigLong("port", 8090),
path,
pk,
sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME, "admin"),
sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, ""));
}
}
@ -263,6 +264,7 @@ public class CrawlStartScanner_p
(int) sb.getConfigLong("port", 8090),
path,
u.hash(),
sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME, "admin"),
sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, ""));
}
} catch (final MalformedURLException e ) {

2
htroot/IndexControlURLs_p.java
Unescape View File

@ -278,6 +278,8 @@ public class IndexControlURLs_p {
final File dump = segment.fulltext().dumpSolr();
prop.put("indexdump", 1);
prop.put("indexdump_dumpfile", dump.getAbsolutePath());
dumpFiles = segment.fulltext().dumpFiles();
prop.put("dumprestore_dumpfile", dumpFiles.size() == 0 ? "" : dumpFiles.get(dumpFiles.size() - 1).getAbsolutePath());
sb.tables.recordAPICall(post, "IndexControlURLs_p.html", WorkTables.TABLE_API_TYPE_STEERING, "solr dump generation");
}

2
htroot/Table_API_p.java
Unescape View File

@ -206,7 +206,7 @@ public class Table_API_p {
}
// now call the api URLs and store the result status
final Map<String, Integer> l = sb.tables.execAPICalls(Domains.LOCALHOST, (int) sb.getConfigLong("port", 8090), pks, sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, ""));
final Map<String, Integer> l = sb.tables.execAPICalls(Domains.LOCALHOST, (int) sb.getConfigLong("port", 8090), pks, sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME, "admin"), sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, ""));
// construct result table
prop.put("showexec", l.isEmpty() ? 0 : 1);

4
htroot/sharedBlacklist_p.java
Unescape View File

@ -140,7 +140,7 @@ public class sharedBlacklist_p {
// get List
final DigestURL u = new DigestURL(downloadURLOld);
otherBlacklist = FileUtils.strings(u.get(agent, sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, "")));
otherBlacklist = FileUtils.strings(u.get(agent, sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME, "admin"), sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, "")));
} catch (final Exception e) {
prop.put("status", STATUS_PEER_UNKNOWN);
prop.putHTML("status_name", hash);
@ -157,7 +157,7 @@ public class sharedBlacklist_p {
try {
final DigestURL u = new DigestURL(downloadURL);
otherBlacklist = FileUtils.strings(u.get(agent, sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, "")));
otherBlacklist = FileUtils.strings(u.get(agent, sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME, "admin"), sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, "")));
} catch (final Exception e) {
prop.put("status", STATUS_URL_PROBLEM);
prop.putHTML("status_address",downloadURL);

2
source/net/yacy/contentcontrol/SMWListSyncThread.java
Unescape View File

@ -79,7 +79,7 @@ public class SMWListSyncThread {
+ "/limit%3D200000"
+ "/format%3Dystat");
String reply = UTF8.String(new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent).GETbytes(urlCount.toString(), null));
String reply = UTF8.String(new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent).GETbytes(urlCount.toString(), null, null));
String overallcount = reply.split(",")[0];
String lastsyncstring = reply.split(",")[1];
this.currentmax = Integer.parseInt(overallcount);

8
source/net/yacy/cora/document/id/MultiProtocolURL.java
Unescape View File

@ -2046,7 +2046,7 @@ public class MultiProtocolURL implements Serializable, Comparable<MultiProtocolU
return null;
}
public InputStream getInputStream(final ClientIdentification.Agent agent, final String pass) throws IOException {
public InputStream getInputStream(final ClientIdentification.Agent agent, final String username, final String pass) throws IOException {
if (isFile()) return new BufferedInputStream(new FileInputStream(getFSFile()));
if (isSMB()) return new BufferedInputStream(new SmbFileInputStream(getSmbFile()));
if (isFTP()) {
@ -2059,13 +2059,13 @@ public class MultiProtocolURL implements Serializable, Comparable<MultiProtocolU
if (isHTTP() || isHTTPS()) {
final HTTPClient client = new HTTPClient(agent);
client.setHost(getHost());
return new ByteArrayInputStream(client.GETbytes(this, pass));
return new ByteArrayInputStream(client.GETbytes(this, username, pass));
}
return null;
}
public byte[] get(final ClientIdentification.Agent agent, final String pass) throws IOException {
public byte[] get(final ClientIdentification.Agent agent, final String username, final String pass) throws IOException {
if (isFile()) return read(new FileInputStream(getFSFile()));
if (isSMB()) return read(new SmbFileInputStream(getSmbFile()));
if (isFTP()) {
@ -2078,7 +2078,7 @@ public class MultiProtocolURL implements Serializable, Comparable<MultiProtocolU
if (isHTTP() || isHTTPS()) {
final HTTPClient client = new HTTPClient(agent);
client.setHost(getHost());
return client.GETbytes(this, pass);
return client.GETbytes(this, username, pass);
}
return null;

2
source/net/yacy/cora/federate/yacy/api/Network.java
Unescape View File

@ -50,7 +50,7 @@ public class Network {
public static Peers getNetwork(final String address) throws IOException {
Peers peers = new Peers();
final HTTPClient httpclient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent);
final byte[] content = httpclient.GETbytes("http://" + address + "/Network.xml?page=1&maxCount=1000&ip=", null);
final byte[] content = httpclient.GETbytes("http://" + address + "/Network.xml?page=1&maxCount=1000&ip=", null, null);
ByteArrayInputStream bais = new ByteArrayInputStream(content);
Document doc = null;
try {

18
source/net/yacy/cora/protocol/http/HTTPClient.java
Unescape View File

@ -307,8 +307,8 @@ public class HTTPClient {
* @return content bytes
* @throws IOException
*/
public byte[] GETbytes(final String uri, final String pass) throws IOException {
return GETbytes(uri, pass, Integer.MAX_VALUE);
public byte[] GETbytes(final String uri, final String username, final String pass) throws IOException {
return GETbytes(uri, username, pass, Integer.MAX_VALUE);
}
/**
@ -318,8 +318,8 @@ public class HTTPClient {
* @return content bytes
* @throws IOException
*/
public byte[] GETbytes(final MultiProtocolURL url, final String pass) throws IOException {
return GETbytes(url, pass, Integer.MAX_VALUE);
public byte[] GETbytes(final MultiProtocolURL url, final String username, final String pass) throws IOException {
return GETbytes(url, username, pass, Integer.MAX_VALUE);
}
/**
@ -330,8 +330,8 @@ public class HTTPClient {
* @return content bytes
* @throws IOException
*/
public byte[] GETbytes(final String uri, final String pass, final int maxBytes) throws IOException {
return GETbytes(new MultiProtocolURL(uri), pass, maxBytes);
public byte[] GETbytes(final String uri, final String username, final String pass, final int maxBytes) throws IOException {
return GETbytes(new MultiProtocolURL(uri), username, pass, maxBytes);
}
@ -343,7 +343,7 @@ public class HTTPClient {
* @return content bytes
* @throws IOException
*/
public byte[] GETbytes(final MultiProtocolURL url, final String pass, final int maxBytes) throws IOException {
public byte[] GETbytes(final MultiProtocolURL url, final String username, final String pass, final int maxBytes) throws IOException {
final boolean localhost = Domains.isLocalhost(url.getHost());
final String urix = url.toNormalform(true);
HttpGet httpGet = null;
@ -357,7 +357,7 @@ public class HTTPClient {
CredentialsProvider credsProvider = new BasicCredentialsProvider();
credsProvider.setCredentials(
AuthScope.ANY, // thats ok since we tested for localhost!
new UsernamePasswordCredentials("admin", pass));
new UsernamePasswordCredentials(username, pass));
CloseableHttpClient httpclient = HttpClients.custom().setDefaultCredentialsProvider(credsProvider).build();
byte[] content = null;
try {
@ -822,7 +822,7 @@ public class HTTPClient {
url = "http://" + url;
}
try {
System.out.println(UTF8.String(client.GETbytes(url, null)));
System.out.println(UTF8.String(client.GETbytes(url, null, null)));
} catch (final IOException e) {
e.printStackTrace();
}

2
source/net/yacy/crawler/retrieval/FileLoader.java
Unescape View File

@ -135,7 +135,7 @@ public class FileLoader {
}
// load the resource
InputStream is = url.getInputStream(ClientIdentification.yacyInternetCrawlerAgent, null);
InputStream is = url.getInputStream(ClientIdentification.yacyInternetCrawlerAgent, null, null);
byte[] b = FileUtils.read(is);
is.close();

4
source/net/yacy/crawler/retrieval/HTTPLoader.java
Unescape View File

@ -128,7 +128,7 @@ public final class HTTPLoader {
client.setHeader(requestHeader.entrySet());
// send request
final byte[] responseBody = client.GETbytes(url, sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, ""), maxFileSize);
final byte[] responseBody = client.GETbytes(url, sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME, "admin"), sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, ""), maxFileSize);
final int statusCode = client.getHttpResponse().getStatusLine().getStatusCode();
final ResponseHeader responseHeader = new ResponseHeader(statusCode, client.getHttpResponse().getAllHeaders());
String requestURLString = request.url().toNormalform(true);
@ -243,7 +243,7 @@ public final class HTTPLoader {
final HTTPClient client = new HTTPClient(agent);
client.setTimout(20000);
client.setHeader(requestHeader.entrySet());
final byte[] responseBody = client.GETbytes(request.url(), null);
final byte[] responseBody = client.GETbytes(request.url(), null, null);
final int code = client.getHttpResponse().getStatusLine().getStatusCode();
final ResponseHeader header = new ResponseHeader(code, client.getHttpResponse().getAllHeaders());
// FIXME: 30*-handling (bottom) is never reached

2
source/net/yacy/crawler/retrieval/SMBLoader.java
Unescape View File

@ -153,7 +153,7 @@ public class SMBLoader {
}
// load the resource
InputStream is = url.getInputStream(ClientIdentification.yacyInternetCrawlerAgent, null);
InputStream is = url.getInputStream(ClientIdentification.yacyInternetCrawlerAgent, null, null);
byte[] b = FileUtils.read(is);
is.close();

12
source/net/yacy/data/WorkTables.java
Unescape View File

@ -217,7 +217,7 @@ public class WorkTables extends Tables {
* @param port the port on the host
* @return a map of the called urls and the http status code of the api call or -1 if any other IOException occurred
*/
public Map<String, Integer> execAPICalls(String host, int port, Collection<String> pks, final String pass) {
public Map<String, Integer> execAPICalls(String host, int port, Collection<String> pks, final String username, final String pass) {
// now call the api URLs and store the result status
final HTTPClient client = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent);
client.setTimout(120000);
@ -238,7 +238,7 @@ public class WorkTables extends Tables {
url += "&" + WorkTables.TABLE_API_COL_APICALL_PK + "=" + UTF8.String(row.getPK());
ConcurrentLog.info("WorkTables", "executing url: " + url);
try {
client.GETbytes(url, pass);
client.GETbytes(url, username, pass);
l.put(url, client.getStatusCode());
} catch (final IOException e) {
ConcurrentLog.logException(e);
@ -248,14 +248,14 @@ public class WorkTables extends Tables {
return l;
}
public static int execAPICall(String host, int port, String path, byte[] pk, final String pass) {
public static int execAPICall(String host, int port, String path, byte[] pk, final String username, final String pass) {
// now call the api URLs and store the result status
final HTTPClient client = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent);
client.setTimout(120000);
String url = "http://" + host + ":" + port + path;
if (pk != null) url += "&" + WorkTables.TABLE_API_COL_APICALL_PK + "=" + UTF8.String(pk);
try {
client.GETbytes(url, pass);
client.GETbytes(url, username, pass);
return client.getStatusCode();
} catch (final IOException e) {
ConcurrentLog.logException(e);
@ -271,10 +271,10 @@ public class WorkTables extends Tables {
* @param realm authentification realm
* @return the http status code of the api call or -1 if any other IOException occurred
*/
public int execAPICall(String pk, String host, int port, final String pass) {
public int execAPICall(String pk, String host, int port, final String username, final String pass) {
ArrayList<String> pks = new ArrayList<String>();
pks.add(pk);
Map<String, Integer> m = execAPICalls(host, port, pks, pass);
Map<String, Integer> m = execAPICalls(host, port, pks, username, pass);
if (m.isEmpty()) return -1;
return m.values().iterator().next().intValue();
}

4
source/net/yacy/data/ymark/YMarkCrawlStart.java
Unescape View File

@ -110,8 +110,8 @@ public class YMarkCrawlStart extends HashMap<String,String>{
}
}
public int exec(final String host, final int port, final String pass) {
return this.worktables.execAPICall(this.apicall_pk, host, port, pass);
public int exec(final String host, final int port, final String username, final String pass) {
return this.worktables.execAPICall(this.apicall_pk, host, port, username, pass);
}
private void load() {

2
source/net/yacy/document/parser/htmlParser.java
Unescape View File

@ -305,7 +305,7 @@ public class htmlParser extends AbstractParser implements Parser {
AnchorURL url;
try {
url = new AnchorURL(args[0]);
final byte[] content = url.get(ClientIdentification.yacyInternetCrawlerAgent, null);
final byte[] content = url.get(ClientIdentification.yacyInternetCrawlerAgent, null, null);
final Document[] document = new htmlParser().parse(url, "text/html", null, new ByteArrayInputStream(content));
final String title = document[0].dc_title();
System.out.println(title);

2
source/net/yacy/peers/SeedDB.java
Unescape View File

@ -816,7 +816,7 @@ public final class SeedDB implements AlternativeDomainNames {
byte[] content = null;
try {
// send request
content = client.GETbytes(seedURL, null);
content = client.GETbytes(seedURL, null, null);
} catch (final Exception e) {
throw new IOException("Unable to download seed file '" + seedURL + "'. " + e.getMessage());
}

2
source/net/yacy/peers/operation/yacyRelease.java
Unescape View File

@ -294,7 +294,7 @@ public final class yacyRelease extends yacyVersion {
// download signature first, if public key is available
try {
if (this.publicKey != null) {
final byte[] signatureData = client.GETbytes(getUrl().toString() + ".sig", null);
final byte[] signatureData = client.GETbytes(getUrl().toString() + ".sig", null, null);
if (signatureData == null) {
ConcurrentLog.warn("yacyVersion", "download of signature " + getUrl().toString() + " failed. ignoring signature file.");
}

16
source/net/yacy/search/Switchboard.java
Unescape View File

@ -210,7 +210,6 @@ import net.yacy.utils.crypt;
import com.google.common.io.Files;
import net.yacy.http.YaCyHttpServer;
import net.yacy.http.YaCyLegacyCredential;
public final class Switchboard extends serverSwitch {
@ -339,9 +338,10 @@ public final class Switchboard extends serverSwitch {
}
// load values from configs
final File indexPath =
getDataPath(SwitchboardConstants.INDEX_PRIMARY_PATH, SwitchboardConstants.INDEX_PATH_DEFAULT);
final File indexPath = getDataPath(SwitchboardConstants.INDEX_PRIMARY_PATH, SwitchboardConstants.INDEX_PATH_DEFAULT);
this.log.config("Index Primary Path: " + indexPath.toString());
final File archivePath = getDataPath(SwitchboardConstants.INDEX_ARCHIVE_PATH, SwitchboardConstants.INDEX_ARCHIVE_DEFAULT);
this.log.config("Index Archive Path: " + archivePath.toString());
this.listsPath =
getDataPath(SwitchboardConstants.LISTS_PATH, SwitchboardConstants.LISTS_PATH_DEFAULT);
this.log.config("Lists Path: " + this.listsPath.toString());
@ -498,7 +498,7 @@ public final class Switchboard extends serverSwitch {
// initialize index
ReferenceContainer.maxReferences = getConfigInt("index.maxReferences", 0);
final File segmentsPath = new File(new File(indexPath, networkName), "SEGMENTS");
this.index = new Segment(this.log, segmentsPath, solrCollectionConfigurationWork, solrWebgraphConfigurationWork);
this.index = new Segment(this.log, segmentsPath, archivePath, solrCollectionConfigurationWork, solrWebgraphConfigurationWork);
if (this.getConfigBool(SwitchboardConstants.CORE_SERVICE_RWI, true)) try {
this.index.connectRWI(wordCacheMaxCount, fileSizeMax);
} catch (final IOException e) {ConcurrentLog.logException(e);}
@ -1331,7 +1331,9 @@ public final class Switchboard extends serverSwitch {
partitionExponent,
this.useTailCache,
this.exceed134217727);
this.index = new Segment(this.log, new File(new File(indexPrimaryPath, networkName), "SEGMENTS"), collectionConfiguration, webgraphConfiguration);
final File segmentsPath = new File(new File(indexPrimaryPath, networkName), "SEGMENTS");
final File archivePath = getDataPath(SwitchboardConstants.INDEX_ARCHIVE_PATH, SwitchboardConstants.INDEX_ARCHIVE_DEFAULT);
this.index = new Segment(this.log, segmentsPath, archivePath, collectionConfiguration, webgraphConfiguration);
if (this.getConfigBool(SwitchboardConstants.CORE_SERVICE_RWI, true)) this.index.connectRWI(wordCacheMaxCount, fileSizeMax);
if (this.getConfigBool(SwitchboardConstants.CORE_SERVICE_CITATION, true)) this.index.connectCitation(wordCacheMaxCount, fileSizeMax);
if (this.getConfigBool(SwitchboardConstants.CORE_SERVICE_FULLTEXT, true)) {
@ -2458,7 +2460,7 @@ public final class Switchboard extends serverSwitch {
startupAction = false;
// execute api calls
final Map<String, Integer> callResult = this.tables.execAPICalls("localhost", (int) getConfigLong("port", 8090), pks, getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, ""));
final Map<String, Integer> callResult = this.tables.execAPICalls("localhost", (int) getConfigLong("port", 8090), pks, getConfig(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME, "admin"), getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, ""));
for ( final Map.Entry<String, Integer> call : callResult.entrySet() ) {
this.log.info("Scheduler executed api call, response " + call.getValue() + ": " + call.getKey());
}
@ -3737,7 +3739,7 @@ public final class Switchboard extends serverSwitch {
}
}
scc.incrementAndGet();
final byte[] content = client.GETbytes(url, null);
final byte[] content = client.GETbytes(url, null, null);
Iterator<String> enu = FileUtils.strings(content);
int lc = 0;
while ( enu.hasNext() ) {

4
source/net/yacy/search/SwitchboardConstants.java
Unescape View File

@ -406,8 +406,10 @@ public final class SwitchboardConstants {
* <p>Name of the setting specifying the folder beginning from the YaCy-installation's top-folder, where the
* whole database of known RWIs and URLs as well as dumps of the DHT-In and DHT-Out caches are stored</p>
*/
public static final String INDEX_PRIMARY_PATH = "indexPrimaryPath"; // this is a relative path to the data root
public static final String INDEX_PRIMARY_PATH = "indexPrimaryPath"; // this is a relative path to the application root or an absolute path
public static final String INDEX_PATH_DEFAULT = "DATA/INDEX";
public static final String INDEX_ARCHIVE_PATH = "indexArchivePath"; // this is a relative path to the application root or an absolute path
public static final String INDEX_ARCHIVE_DEFAULT = "DATA/ARCHIVE";
/**
* <p><code>public static final String <strong>LISTS_PATH</strong> = "listsPath"</code></p>
* <p>Name of the setting specifying the folder beginning from the YaCy-installation's top-folder, where all

6
source/net/yacy/search/index/DocumentIndex.java
Unescape View File

@ -66,9 +66,9 @@ public class DocumentIndex extends Segment {
static final ThreadGroup workerThreadGroup = new ThreadGroup("workerThreadGroup");
public DocumentIndex(final File segmentPath, final File collectionConfigurationPath, final File webgraphConfigurationPath, final CallbackListener callback, final int cachesize)
public DocumentIndex(final File segmentPath, final File archivePath, final File collectionConfigurationPath, final File webgraphConfigurationPath, final CallbackListener callback, final int cachesize)
throws IOException {
super(new ConcurrentLog("DocumentIndex"), segmentPath,
super(new ConcurrentLog("DocumentIndex"), segmentPath, archivePath,
collectionConfigurationPath == null ? null : new CollectionConfiguration(collectionConfigurationPath, true),
webgraphConfigurationPath == null ? null : new WebgraphConfiguration(webgraphConfigurationPath, true)
);
@ -153,7 +153,7 @@ public class DocumentIndex extends Segment {
length = -1;
}
try {
documents = TextParser.parseSource(url, null, null, length, url.getInputStream(ClientIdentification.yacyInternetCrawlerAgent, null));
documents = TextParser.parseSource(url, null, null, length, url.getInputStream(ClientIdentification.yacyInternetCrawlerAgent, null, null));
} catch (final Exception e ) {
throw new IOException("cannot parse " + url.toString() + ": " + e.getMessage());
}

18
source/net/yacy/search/index/Fulltext.java
Unescape View File

@ -88,7 +88,8 @@ public final class Fulltext {
private static final String SOLR_OLD_PATH[] = new String[]{"solr_36", "solr_40", "solr_44", "solr_45"};
// class objects
private final File segmentPath;
private final File segmentPath;
private final File archivePath;
private Index urlIndexFile;
private Export exportthread; // will have a export thread assigned if exporter is running
private String tablename;
@ -98,8 +99,10 @@ public final class Fulltext {
private final WebgraphConfiguration webgraphConfiguration;
private boolean writeWebgraph;
protected Fulltext(final File segmentPath, final CollectionConfiguration collectionConfiguration, final WebgraphConfiguration webgraphConfiguration) {
protected Fulltext(final File segmentPath, final File archivePath,
final CollectionConfiguration collectionConfiguration, final WebgraphConfiguration webgraphConfiguration) {
this.segmentPath = segmentPath;
this.archivePath = archivePath;
this.tablename = null;
this.urlIndexFile = null;
this.exportthread = null; // will have a export thread assigned if exporter is running
@ -661,14 +664,13 @@ public final class Fulltext {
ConcurrentLog.warn("Fulltext", "HOT DUMP selected solr0.getStoragePath() == NULL, no dump list!");
return zips;
}
File storagePath = esc.getContainerPath().getParentFile();
if (storagePath == null) {
if (this.archivePath == null) {
ConcurrentLog.warn("Fulltext", "HOT DUMP selected esc.getStoragePath().getParentFile() == NULL, no dump list!");
return zips;
}
ConcurrentLog.info("Fulltext", "HOT DUMP dump path = " + storagePath.toString());
for (String p: storagePath.list()) {
if (p.endsWith("zip")) zips.add(new File(storagePath, p));
ConcurrentLog.info("Fulltext", "HOT DUMP dump path = " + this.archivePath.toString());
for (String p: this.archivePath.list()) {
if (p.endsWith("zip")) zips.add(new File(this.archivePath, p));
}
return zips;
}
@ -680,7 +682,7 @@ public final class Fulltext {
public File dumpSolr() {
EmbeddedInstance esc = this.solrInstances.getSolr0();
File storagePath = esc.getContainerPath();
File zipOut = new File(storagePath.toString() + "_" + GenericFormatter.SHORT_DAY_FORMATTER.format() + ".zip");
File zipOut = new File(this.archivePath, storagePath.getName() + "_" + GenericFormatter.SHORT_DAY_FORMATTER.format() + ".zip");
synchronized (this.solrInstances) {
this.disconnectLocalSolr();
this.solrInstances.close();

7
source/net/yacy/search/index/Segment.java
Unescape View File

@ -126,14 +126,15 @@ public class Segment {
* @param segmentPath that should be the path ponting to the directory "SEGMENT"
* @param collectionSchema
*/
public Segment(final ConcurrentLog log, final File segmentPath,
public Segment(final ConcurrentLog log, final File segmentPath, final File archivePath,
final CollectionConfiguration collectionConfiguration, final WebgraphConfiguration webgraphConfiguration) {
log.info("Initializing Segment '" + segmentPath + ".");
this.log = log;
this.segmentPath = segmentPath;
archivePath.mkdirs();
// create LURL-db
this.fulltext = new Fulltext(segmentPath, collectionConfiguration, webgraphConfiguration);
this.fulltext = new Fulltext(segmentPath, archivePath, collectionConfiguration, webgraphConfiguration);
this.termIndex = null;
this.urlCitationIndex = null;

2
source/net/yacy/server/serverSwitch.java
Unescape View File

@ -537,7 +537,7 @@ public class serverSwitch
reqHeader.put(HeaderFramework.USER_AGENT, ClientIdentification.yacyInternetCrawlerAgent.userAgent);
final HTTPClient client = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent);
client.setHeader(reqHeader.entrySet());
byte[] data = client.GETbytes(uri, getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, ""));
byte[] data = client.GETbytes(uri, getConfig(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME, "admin"),getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, ""));
if ( data == null || data.length == 0 ) {
continue;
}

2
source/net/yacy/utils/loaderThreads.java
Unescape View File

@ -122,7 +122,7 @@ public class loaderThreads {
@Override
public void run() {
try {
this.page = this.url.get(this.agent, null);
this.page = this.url.get(this.agent, null, null);
this.loaded = true;
this.process.feed(this.page);
if (this.process.status() == loaderCore.STATUS_FAILED) {

2
source/net/yacy/yacy.java
Unescape View File

@ -523,7 +523,7 @@ public final class yacy {
final HTTPClient con = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent);
con.setHeader(requestHeader.entrySet());
try {
con.GETbytes("http://localhost:"+ port +"/" + path, encodedPassword);
con.GETbytes("http://localhost:"+ port +"/" + path, sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME,"admin"), encodedPassword);
if (con.getStatusCode() > 199 && con.getStatusCode() < 300) {
ConcurrentLog.config("COMMAND-STEERING", "YACY accepted steering command: " + processdescription);

Write Preview
Loading…
Cancel
Save