diff --git a/defaults/yacy.init b/defaults/yacy.init index 2b83fa660..6e4d490d1 100644 --- a/defaults/yacy.init +++ b/defaults/yacy.init @@ -287,6 +287,9 @@ promoteSearchPageGreeting.smallImage = /env/grafics/YaCyLogo_60ppi.png # when the secondary path should be equal to the primary, it must be declared empty indexPrimaryPath=DATA/INDEX +# the path to index archive dumps +indexArchivePath=DATA/ARCHIVE + # the path to the LISTS files. Most lists are used to filter web content listsPath=DATA/LISTS diff --git a/htroot/ConfigAccounts_p.java b/htroot/ConfigAccounts_p.java index b9401f451..e64ecc2d6 100644 --- a/htroot/ConfigAccounts_p.java +++ b/htroot/ConfigAccounts_p.java @@ -58,35 +58,46 @@ public class ConfigAccounts_p { boolean localhostAccess = sb.getConfigBool(SwitchboardConstants.ADMIN_ACCOUNT_FOR_LOCALHOST, false); if (post != null && post.containsKey("setAdmin")) { localhostAccess = Domains.isLocalhost(post.get("access", "")); - final String user = (post == null) ? "" : post.get("adminuser", ""); - final String pw1 = (post == null) ? "" : post.get("adminpw1", ""); - final String pw2 = (post == null) ? "" : post.get("adminpw2", ""); - + final String user = post.get("adminuser", ""); + final String pw1 = post.get("adminpw1", ""); + final String pw2 = post.get("adminpw2", ""); + int inputerror=0; // may be overwritten if new password is given if (user.length() > 0 && pw1.length() > 3 && pw1.equals(pw2)) { // check passed. set account: env.setConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, Digest.encodeMD5Hex(Base64Order.standardCoder.encodeString(user + ":" + pw1))); env.setConfig(SwitchboardConstants.ADMIN_ACCOUNT, ""); - env.setConfig(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME,user); + env.setConfig(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME,user); + } else { + if (!localhostAccess) { + if (user.isEmpty()) { + inputerror = 3; + } else { + inputerror = 2; // password match error + } + prop.put("error", inputerror); + } } - if (localhostAccess) { + if (inputerror == 0) { + if (localhostAccess) { - sb.setConfig(SwitchboardConstants.ADMIN_ACCOUNT_FOR_LOCALHOST, true); - // if an localhost access is configured, check if a local password is given - // if not, set a random password - if (env.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, "").isEmpty()) { - // make a 'random' password - env.setConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, "0000" + sb.genRandomPassword()); - env.setConfig(SwitchboardConstants.ADMIN_ACCOUNT, ""); - } - } else { - sb.setConfig(SwitchboardConstants.ADMIN_ACCOUNT_FOR_LOCALHOST, false); - if (env.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, "").startsWith("0000")) { - // make shure that the user can still use the interface after a random password was set - env.setConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, ""); + sb.setConfig(SwitchboardConstants.ADMIN_ACCOUNT_FOR_LOCALHOST, true); + // if an localhost access is configured, check if a local password is given + // if not, set a random password + if (env.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, "").isEmpty()) { + // make a 'random' password + env.setConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, "0000" + sb.genRandomPassword()); + env.setConfig(SwitchboardConstants.ADMIN_ACCOUNT, ""); + } + } else { + sb.setConfig(SwitchboardConstants.ADMIN_ACCOUNT_FOR_LOCALHOST, false); + if (env.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, "").startsWith("0000")) { + // make shure that the user can still use the interface after a random password was set + env.setConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, ""); + } } - } + } } if (env.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, "").isEmpty() && !env.getConfigBool(SwitchboardConstants.ADMIN_ACCOUNT_FOR_LOCALHOST, false)) { diff --git a/htroot/ConfigAppearance_p.java b/htroot/ConfigAppearance_p.java index 362012421..e7495bb6b 100644 --- a/htroot/ConfigAppearance_p.java +++ b/htroot/ConfigAppearance_p.java @@ -102,7 +102,7 @@ public class ConfigAppearance_p { final Iterator it; try { final DigestURL u = new DigestURL(url); - it = FileUtils.strings(u.get(ClientIdentification.yacyInternetCrawlerAgent, sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, ""))); + it = FileUtils.strings(u.get(ClientIdentification.yacyInternetCrawlerAgent, sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME, "admin"), sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, ""))); } catch (final IOException e) { prop.put("status", "1");// unable to get URL prop.put("status_url", url); diff --git a/htroot/ConfigLanguage_p.java b/htroot/ConfigLanguage_p.java index e2239c3e2..65d2b076c 100644 --- a/htroot/ConfigLanguage_p.java +++ b/htroot/ConfigLanguage_p.java @@ -102,7 +102,7 @@ public class ConfigLanguage_p { Iterator it; try { final DigestURL u = new DigestURL(url); - it = FileUtils.strings(u.get(ClientIdentification.yacyInternetCrawlerAgent, sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, ""))); + it = FileUtils.strings(u.get(ClientIdentification.yacyInternetCrawlerAgent, sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME, "admin"), sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, ""))); } catch(final IOException e) { prop.put("status", "1");//unable to get url prop.put("status_url", url); diff --git a/htroot/CrawlStartScanner_p.java b/htroot/CrawlStartScanner_p.java index 694686b9c..420657343 100644 --- a/htroot/CrawlStartScanner_p.java +++ b/htroot/CrawlStartScanner_p.java @@ -217,6 +217,7 @@ public class CrawlStartScanner_p (int) sb.getConfigLong("port", 8090), path, pk, + sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME, "admin"), sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, "")); } } @@ -263,6 +264,7 @@ public class CrawlStartScanner_p (int) sb.getConfigLong("port", 8090), path, u.hash(), + sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME, "admin"), sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, "")); } } catch (final MalformedURLException e ) { diff --git a/htroot/IndexControlURLs_p.java b/htroot/IndexControlURLs_p.java index c1e4ca30b..6d8e1237c 100644 --- a/htroot/IndexControlURLs_p.java +++ b/htroot/IndexControlURLs_p.java @@ -278,6 +278,8 @@ public class IndexControlURLs_p { final File dump = segment.fulltext().dumpSolr(); prop.put("indexdump", 1); prop.put("indexdump_dumpfile", dump.getAbsolutePath()); + dumpFiles = segment.fulltext().dumpFiles(); + prop.put("dumprestore_dumpfile", dumpFiles.size() == 0 ? "" : dumpFiles.get(dumpFiles.size() - 1).getAbsolutePath()); sb.tables.recordAPICall(post, "IndexControlURLs_p.html", WorkTables.TABLE_API_TYPE_STEERING, "solr dump generation"); } diff --git a/htroot/Table_API_p.java b/htroot/Table_API_p.java index 94455df8c..8749d0fd9 100644 --- a/htroot/Table_API_p.java +++ b/htroot/Table_API_p.java @@ -206,7 +206,7 @@ public class Table_API_p { } // now call the api URLs and store the result status - final Map l = sb.tables.execAPICalls(Domains.LOCALHOST, (int) sb.getConfigLong("port", 8090), pks, sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, "")); + final Map l = sb.tables.execAPICalls(Domains.LOCALHOST, (int) sb.getConfigLong("port", 8090), pks, sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME, "admin"), sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, "")); // construct result table prop.put("showexec", l.isEmpty() ? 0 : 1); diff --git a/htroot/sharedBlacklist_p.java b/htroot/sharedBlacklist_p.java index 4e0da60f4..cffeed8f2 100644 --- a/htroot/sharedBlacklist_p.java +++ b/htroot/sharedBlacklist_p.java @@ -140,7 +140,7 @@ public class sharedBlacklist_p { // get List final DigestURL u = new DigestURL(downloadURLOld); - otherBlacklist = FileUtils.strings(u.get(agent, sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, ""))); + otherBlacklist = FileUtils.strings(u.get(agent, sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME, "admin"), sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, ""))); } catch (final Exception e) { prop.put("status", STATUS_PEER_UNKNOWN); prop.putHTML("status_name", hash); @@ -157,7 +157,7 @@ public class sharedBlacklist_p { try { final DigestURL u = new DigestURL(downloadURL); - otherBlacklist = FileUtils.strings(u.get(agent, sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, ""))); + otherBlacklist = FileUtils.strings(u.get(agent, sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME, "admin"), sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, ""))); } catch (final Exception e) { prop.put("status", STATUS_URL_PROBLEM); prop.putHTML("status_address",downloadURL); diff --git a/source/net/yacy/contentcontrol/SMWListSyncThread.java b/source/net/yacy/contentcontrol/SMWListSyncThread.java index a1957cdf1..6d611f14a 100644 --- a/source/net/yacy/contentcontrol/SMWListSyncThread.java +++ b/source/net/yacy/contentcontrol/SMWListSyncThread.java @@ -79,7 +79,7 @@ public class SMWListSyncThread { + "/limit%3D200000" + "/format%3Dystat"); - String reply = UTF8.String(new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent).GETbytes(urlCount.toString(), null)); + String reply = UTF8.String(new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent).GETbytes(urlCount.toString(), null, null)); String overallcount = reply.split(",")[0]; String lastsyncstring = reply.split(",")[1]; this.currentmax = Integer.parseInt(overallcount); diff --git a/source/net/yacy/cora/document/id/MultiProtocolURL.java b/source/net/yacy/cora/document/id/MultiProtocolURL.java index da0fa29ae..eb3dd1068 100644 --- a/source/net/yacy/cora/document/id/MultiProtocolURL.java +++ b/source/net/yacy/cora/document/id/MultiProtocolURL.java @@ -2046,7 +2046,7 @@ public class MultiProtocolURL implements Serializable, Comparable execAPICalls(String host, int port, Collection pks, final String pass) { + public Map execAPICalls(String host, int port, Collection pks, final String username, final String pass) { // now call the api URLs and store the result status final HTTPClient client = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent); client.setTimout(120000); @@ -238,7 +238,7 @@ public class WorkTables extends Tables { url += "&" + WorkTables.TABLE_API_COL_APICALL_PK + "=" + UTF8.String(row.getPK()); ConcurrentLog.info("WorkTables", "executing url: " + url); try { - client.GETbytes(url, pass); + client.GETbytes(url, username, pass); l.put(url, client.getStatusCode()); } catch (final IOException e) { ConcurrentLog.logException(e); @@ -248,14 +248,14 @@ public class WorkTables extends Tables { return l; } - public static int execAPICall(String host, int port, String path, byte[] pk, final String pass) { + public static int execAPICall(String host, int port, String path, byte[] pk, final String username, final String pass) { // now call the api URLs and store the result status final HTTPClient client = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent); client.setTimout(120000); String url = "http://" + host + ":" + port + path; if (pk != null) url += "&" + WorkTables.TABLE_API_COL_APICALL_PK + "=" + UTF8.String(pk); try { - client.GETbytes(url, pass); + client.GETbytes(url, username, pass); return client.getStatusCode(); } catch (final IOException e) { ConcurrentLog.logException(e); @@ -271,10 +271,10 @@ public class WorkTables extends Tables { * @param realm authentification realm * @return the http status code of the api call or -1 if any other IOException occurred */ - public int execAPICall(String pk, String host, int port, final String pass) { + public int execAPICall(String pk, String host, int port, final String username, final String pass) { ArrayList pks = new ArrayList(); pks.add(pk); - Map m = execAPICalls(host, port, pks, pass); + Map m = execAPICalls(host, port, pks, username, pass); if (m.isEmpty()) return -1; return m.values().iterator().next().intValue(); } diff --git a/source/net/yacy/data/ymark/YMarkCrawlStart.java b/source/net/yacy/data/ymark/YMarkCrawlStart.java index 3951bd632..753c77ff6 100644 --- a/source/net/yacy/data/ymark/YMarkCrawlStart.java +++ b/source/net/yacy/data/ymark/YMarkCrawlStart.java @@ -110,8 +110,8 @@ public class YMarkCrawlStart extends HashMap{ } } - public int exec(final String host, final int port, final String pass) { - return this.worktables.execAPICall(this.apicall_pk, host, port, pass); + public int exec(final String host, final int port, final String username, final String pass) { + return this.worktables.execAPICall(this.apicall_pk, host, port, username, pass); } private void load() { diff --git a/source/net/yacy/document/parser/htmlParser.java b/source/net/yacy/document/parser/htmlParser.java index 2d9faee3e..1241c7188 100644 --- a/source/net/yacy/document/parser/htmlParser.java +++ b/source/net/yacy/document/parser/htmlParser.java @@ -305,7 +305,7 @@ public class htmlParser extends AbstractParser implements Parser { AnchorURL url; try { url = new AnchorURL(args[0]); - final byte[] content = url.get(ClientIdentification.yacyInternetCrawlerAgent, null); + final byte[] content = url.get(ClientIdentification.yacyInternetCrawlerAgent, null, null); final Document[] document = new htmlParser().parse(url, "text/html", null, new ByteArrayInputStream(content)); final String title = document[0].dc_title(); System.out.println(title); diff --git a/source/net/yacy/peers/SeedDB.java b/source/net/yacy/peers/SeedDB.java index d64dcc8ca..b86d5c5d5 100644 --- a/source/net/yacy/peers/SeedDB.java +++ b/source/net/yacy/peers/SeedDB.java @@ -816,7 +816,7 @@ public final class SeedDB implements AlternativeDomainNames { byte[] content = null; try { // send request - content = client.GETbytes(seedURL, null); + content = client.GETbytes(seedURL, null, null); } catch (final Exception e) { throw new IOException("Unable to download seed file '" + seedURL + "'. " + e.getMessage()); } diff --git a/source/net/yacy/peers/operation/yacyRelease.java b/source/net/yacy/peers/operation/yacyRelease.java index fae58ba2c..dc5aedf32 100644 --- a/source/net/yacy/peers/operation/yacyRelease.java +++ b/source/net/yacy/peers/operation/yacyRelease.java @@ -294,7 +294,7 @@ public final class yacyRelease extends yacyVersion { // download signature first, if public key is available try { if (this.publicKey != null) { - final byte[] signatureData = client.GETbytes(getUrl().toString() + ".sig", null); + final byte[] signatureData = client.GETbytes(getUrl().toString() + ".sig", null, null); if (signatureData == null) { ConcurrentLog.warn("yacyVersion", "download of signature " + getUrl().toString() + " failed. ignoring signature file."); } diff --git a/source/net/yacy/search/Switchboard.java b/source/net/yacy/search/Switchboard.java index da879767b..8e15ab0a8 100644 --- a/source/net/yacy/search/Switchboard.java +++ b/source/net/yacy/search/Switchboard.java @@ -210,7 +210,6 @@ import net.yacy.utils.crypt; import com.google.common.io.Files; import net.yacy.http.YaCyHttpServer; -import net.yacy.http.YaCyLegacyCredential; public final class Switchboard extends serverSwitch { @@ -339,9 +338,10 @@ public final class Switchboard extends serverSwitch { } // load values from configs - final File indexPath = - getDataPath(SwitchboardConstants.INDEX_PRIMARY_PATH, SwitchboardConstants.INDEX_PATH_DEFAULT); + final File indexPath = getDataPath(SwitchboardConstants.INDEX_PRIMARY_PATH, SwitchboardConstants.INDEX_PATH_DEFAULT); this.log.config("Index Primary Path: " + indexPath.toString()); + final File archivePath = getDataPath(SwitchboardConstants.INDEX_ARCHIVE_PATH, SwitchboardConstants.INDEX_ARCHIVE_DEFAULT); + this.log.config("Index Archive Path: " + archivePath.toString()); this.listsPath = getDataPath(SwitchboardConstants.LISTS_PATH, SwitchboardConstants.LISTS_PATH_DEFAULT); this.log.config("Lists Path: " + this.listsPath.toString()); @@ -498,7 +498,7 @@ public final class Switchboard extends serverSwitch { // initialize index ReferenceContainer.maxReferences = getConfigInt("index.maxReferences", 0); final File segmentsPath = new File(new File(indexPath, networkName), "SEGMENTS"); - this.index = new Segment(this.log, segmentsPath, solrCollectionConfigurationWork, solrWebgraphConfigurationWork); + this.index = new Segment(this.log, segmentsPath, archivePath, solrCollectionConfigurationWork, solrWebgraphConfigurationWork); if (this.getConfigBool(SwitchboardConstants.CORE_SERVICE_RWI, true)) try { this.index.connectRWI(wordCacheMaxCount, fileSizeMax); } catch (final IOException e) {ConcurrentLog.logException(e);} @@ -1331,7 +1331,9 @@ public final class Switchboard extends serverSwitch { partitionExponent, this.useTailCache, this.exceed134217727); - this.index = new Segment(this.log, new File(new File(indexPrimaryPath, networkName), "SEGMENTS"), collectionConfiguration, webgraphConfiguration); + final File segmentsPath = new File(new File(indexPrimaryPath, networkName), "SEGMENTS"); + final File archivePath = getDataPath(SwitchboardConstants.INDEX_ARCHIVE_PATH, SwitchboardConstants.INDEX_ARCHIVE_DEFAULT); + this.index = new Segment(this.log, segmentsPath, archivePath, collectionConfiguration, webgraphConfiguration); if (this.getConfigBool(SwitchboardConstants.CORE_SERVICE_RWI, true)) this.index.connectRWI(wordCacheMaxCount, fileSizeMax); if (this.getConfigBool(SwitchboardConstants.CORE_SERVICE_CITATION, true)) this.index.connectCitation(wordCacheMaxCount, fileSizeMax); if (this.getConfigBool(SwitchboardConstants.CORE_SERVICE_FULLTEXT, true)) { @@ -2458,7 +2460,7 @@ public final class Switchboard extends serverSwitch { startupAction = false; // execute api calls - final Map callResult = this.tables.execAPICalls("localhost", (int) getConfigLong("port", 8090), pks, getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, "")); + final Map callResult = this.tables.execAPICalls("localhost", (int) getConfigLong("port", 8090), pks, getConfig(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME, "admin"), getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, "")); for ( final Map.Entry call : callResult.entrySet() ) { this.log.info("Scheduler executed api call, response " + call.getValue() + ": " + call.getKey()); } @@ -3737,7 +3739,7 @@ public final class Switchboard extends serverSwitch { } } scc.incrementAndGet(); - final byte[] content = client.GETbytes(url, null); + final byte[] content = client.GETbytes(url, null, null); Iterator enu = FileUtils.strings(content); int lc = 0; while ( enu.hasNext() ) { diff --git a/source/net/yacy/search/SwitchboardConstants.java b/source/net/yacy/search/SwitchboardConstants.java index 9e574a80f..031470014 100644 --- a/source/net/yacy/search/SwitchboardConstants.java +++ b/source/net/yacy/search/SwitchboardConstants.java @@ -406,8 +406,10 @@ public final class SwitchboardConstants { *

Name of the setting specifying the folder beginning from the YaCy-installation's top-folder, where the * whole database of known RWIs and URLs as well as dumps of the DHT-In and DHT-Out caches are stored

*/ - public static final String INDEX_PRIMARY_PATH = "indexPrimaryPath"; // this is a relative path to the data root + public static final String INDEX_PRIMARY_PATH = "indexPrimaryPath"; // this is a relative path to the application root or an absolute path public static final String INDEX_PATH_DEFAULT = "DATA/INDEX"; + public static final String INDEX_ARCHIVE_PATH = "indexArchivePath"; // this is a relative path to the application root or an absolute path + public static final String INDEX_ARCHIVE_DEFAULT = "DATA/ARCHIVE"; /** *

public static final String LISTS_PATH = "listsPath"

*

Name of the setting specifying the folder beginning from the YaCy-installation's top-folder, where all diff --git a/source/net/yacy/search/index/DocumentIndex.java b/source/net/yacy/search/index/DocumentIndex.java index d402b7ca8..46b8a5960 100644 --- a/source/net/yacy/search/index/DocumentIndex.java +++ b/source/net/yacy/search/index/DocumentIndex.java @@ -66,9 +66,9 @@ public class DocumentIndex extends Segment { static final ThreadGroup workerThreadGroup = new ThreadGroup("workerThreadGroup"); - public DocumentIndex(final File segmentPath, final File collectionConfigurationPath, final File webgraphConfigurationPath, final CallbackListener callback, final int cachesize) + public DocumentIndex(final File segmentPath, final File archivePath, final File collectionConfigurationPath, final File webgraphConfigurationPath, final CallbackListener callback, final int cachesize) throws IOException { - super(new ConcurrentLog("DocumentIndex"), segmentPath, + super(new ConcurrentLog("DocumentIndex"), segmentPath, archivePath, collectionConfigurationPath == null ? null : new CollectionConfiguration(collectionConfigurationPath, true), webgraphConfigurationPath == null ? null : new WebgraphConfiguration(webgraphConfigurationPath, true) ); @@ -153,7 +153,7 @@ public class DocumentIndex extends Segment { length = -1; } try { - documents = TextParser.parseSource(url, null, null, length, url.getInputStream(ClientIdentification.yacyInternetCrawlerAgent, null)); + documents = TextParser.parseSource(url, null, null, length, url.getInputStream(ClientIdentification.yacyInternetCrawlerAgent, null, null)); } catch (final Exception e ) { throw new IOException("cannot parse " + url.toString() + ": " + e.getMessage()); } diff --git a/source/net/yacy/search/index/Fulltext.java b/source/net/yacy/search/index/Fulltext.java index e8c5c5b58..0eb84686a 100644 --- a/source/net/yacy/search/index/Fulltext.java +++ b/source/net/yacy/search/index/Fulltext.java @@ -88,7 +88,8 @@ public final class Fulltext { private static final String SOLR_OLD_PATH[] = new String[]{"solr_36", "solr_40", "solr_44", "solr_45"}; // class objects - private final File segmentPath; + private final File segmentPath; + private final File archivePath; private Index urlIndexFile; private Export exportthread; // will have a export thread assigned if exporter is running private String tablename; @@ -98,8 +99,10 @@ public final class Fulltext { private final WebgraphConfiguration webgraphConfiguration; private boolean writeWebgraph; - protected Fulltext(final File segmentPath, final CollectionConfiguration collectionConfiguration, final WebgraphConfiguration webgraphConfiguration) { + protected Fulltext(final File segmentPath, final File archivePath, + final CollectionConfiguration collectionConfiguration, final WebgraphConfiguration webgraphConfiguration) { this.segmentPath = segmentPath; + this.archivePath = archivePath; this.tablename = null; this.urlIndexFile = null; this.exportthread = null; // will have a export thread assigned if exporter is running @@ -661,14 +664,13 @@ public final class Fulltext { ConcurrentLog.warn("Fulltext", "HOT DUMP selected solr0.getStoragePath() == NULL, no dump list!"); return zips; } - File storagePath = esc.getContainerPath().getParentFile(); - if (storagePath == null) { + if (this.archivePath == null) { ConcurrentLog.warn("Fulltext", "HOT DUMP selected esc.getStoragePath().getParentFile() == NULL, no dump list!"); return zips; } - ConcurrentLog.info("Fulltext", "HOT DUMP dump path = " + storagePath.toString()); - for (String p: storagePath.list()) { - if (p.endsWith("zip")) zips.add(new File(storagePath, p)); + ConcurrentLog.info("Fulltext", "HOT DUMP dump path = " + this.archivePath.toString()); + for (String p: this.archivePath.list()) { + if (p.endsWith("zip")) zips.add(new File(this.archivePath, p)); } return zips; } @@ -680,7 +682,7 @@ public final class Fulltext { public File dumpSolr() { EmbeddedInstance esc = this.solrInstances.getSolr0(); File storagePath = esc.getContainerPath(); - File zipOut = new File(storagePath.toString() + "_" + GenericFormatter.SHORT_DAY_FORMATTER.format() + ".zip"); + File zipOut = new File(this.archivePath, storagePath.getName() + "_" + GenericFormatter.SHORT_DAY_FORMATTER.format() + ".zip"); synchronized (this.solrInstances) { this.disconnectLocalSolr(); this.solrInstances.close(); diff --git a/source/net/yacy/search/index/Segment.java b/source/net/yacy/search/index/Segment.java index c7e0e0183..e3b53c7ad 100644 --- a/source/net/yacy/search/index/Segment.java +++ b/source/net/yacy/search/index/Segment.java @@ -126,14 +126,15 @@ public class Segment { * @param segmentPath that should be the path ponting to the directory "SEGMENT" * @param collectionSchema */ - public Segment(final ConcurrentLog log, final File segmentPath, + public Segment(final ConcurrentLog log, final File segmentPath, final File archivePath, final CollectionConfiguration collectionConfiguration, final WebgraphConfiguration webgraphConfiguration) { log.info("Initializing Segment '" + segmentPath + "."); this.log = log; this.segmentPath = segmentPath; - + archivePath.mkdirs(); + // create LURL-db - this.fulltext = new Fulltext(segmentPath, collectionConfiguration, webgraphConfiguration); + this.fulltext = new Fulltext(segmentPath, archivePath, collectionConfiguration, webgraphConfiguration); this.termIndex = null; this.urlCitationIndex = null; diff --git a/source/net/yacy/server/serverSwitch.java b/source/net/yacy/server/serverSwitch.java index d6acf5e6a..2ed89ecce 100644 --- a/source/net/yacy/server/serverSwitch.java +++ b/source/net/yacy/server/serverSwitch.java @@ -537,7 +537,7 @@ public class serverSwitch reqHeader.put(HeaderFramework.USER_AGENT, ClientIdentification.yacyInternetCrawlerAgent.userAgent); final HTTPClient client = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent); client.setHeader(reqHeader.entrySet()); - byte[] data = client.GETbytes(uri, getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, "")); + byte[] data = client.GETbytes(uri, getConfig(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME, "admin"),getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, "")); if ( data == null || data.length == 0 ) { continue; } diff --git a/source/net/yacy/utils/loaderThreads.java b/source/net/yacy/utils/loaderThreads.java index de547e76f..56c965390 100644 --- a/source/net/yacy/utils/loaderThreads.java +++ b/source/net/yacy/utils/loaderThreads.java @@ -122,7 +122,7 @@ public class loaderThreads { @Override public void run() { try { - this.page = this.url.get(this.agent, null); + this.page = this.url.get(this.agent, null, null); this.loaded = true; this.process.feed(this.page); if (this.process.status() == loaderCore.STATUS_FAILED) { diff --git a/source/net/yacy/yacy.java b/source/net/yacy/yacy.java index e1beb3a03..6fef1bfd8 100644 --- a/source/net/yacy/yacy.java +++ b/source/net/yacy/yacy.java @@ -523,7 +523,7 @@ public final class yacy { final HTTPClient con = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent); con.setHeader(requestHeader.entrySet()); try { - con.GETbytes("http://localhost:"+ port +"/" + path, encodedPassword); + con.GETbytes("http://localhost:"+ port +"/" + path, sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME,"admin"), encodedPassword); if (con.getStatusCode() > 199 && con.getStatusCode() < 300) { ConcurrentLog.config("COMMAND-STEERING", "YACY accepted steering command: " + processdescription);