diff --git a/bin/clearall.sh b/bin/clearall.sh index 17cb50e99..885379c9c 100755 --- a/bin/clearall.sh +++ b/bin/clearall.sh @@ -1,3 +1,3 @@ #!/usr/bin/env sh cd "`dirname $0`" -./apicall.sh "/IndexControlURLs_p.html?deletecomplete=&deleteIndex=on&deleteSolr=on&deleteCrawlQueues=on&deleteRobots=on&deleteSearchFl=on&deleteCache=on" > /dev/null \ No newline at end of file +./protectedPostApiCall.sh "IndexControlURLs_p.html" "deletecomplete=&deleteIndex=on&deleteSolr=on&deleteCrawlQueues=on&deleteRobots=on&deleteSearchFl=on&deleteCache=on" \ No newline at end of file diff --git a/bin/clearcache.sh b/bin/clearcache.sh index 906595bec..926d70e78 100755 --- a/bin/clearcache.sh +++ b/bin/clearcache.sh @@ -1,3 +1,3 @@ #!/usr/bin/env sh cd "`dirname $0`" -./apicall.sh "/IndexControlURLs_p.html?deleteIndex=off&deleteSolr=off&deleteCache=on&deleteCrawlQueues=off&deleteRobots=on&deleteSearchFl=on&deletecomplete=" > /dev/null +./protectedPostApiCall.sh "IndexControlURLs_p.html" "deleteIndex=off&deleteSolr=off&deleteCache=on&deleteCrawlQueues=off&deleteRobots=on&deleteSearchFl=on&deletecomplete=" diff --git a/bin/clearindex.sh b/bin/clearindex.sh index c32abf94a..9c78bb997 100755 --- a/bin/clearindex.sh +++ b/bin/clearindex.sh @@ -1,3 +1,3 @@ #!/usr/bin/env sh cd "`dirname $0`" -./apicall.sh "/IndexControlURLs_p.html?deletecomplete=&deleteIndex=on&deleteSolr=on&deleteCrawlQueues=on&deleteRobots=on&deleteSearchFl=on&deleteCache=off" > /dev/null +./protectedPostApiCall.sh "IndexControlURLs_p.html" "deletecomplete=&deleteIndex=on&deleteSolr=on&deleteCrawlQueues=on&deleteRobots=on&deleteSearchFl=on&deleteCache=off" diff --git a/bin/deleteurl.sh b/bin/deleteurl.sh index fd9949637..94aefcfb3 100755 --- a/bin/deleteurl.sh +++ b/bin/deleteurl.sh @@ -1,3 +1,3 @@ #!/usr/bin/env sh cd "`dirname $0`" -./apicall.sh "/IndexControlURLs_p.html?urlhashdeleteall=&urlstring=$1" > /dev/null +./protectedPostApiCall.sh "IndexControlURLs_p.html" "urlhashdeleteall=&urlstring=$1" diff --git a/bin/passwd.sh b/bin/passwd.sh index 62267fc46..45335853e 100755 --- a/bin/passwd.sh +++ b/bin/passwd.sh @@ -6,7 +6,7 @@ if [ -z "$1" ]; then exit 2 fi -(./apicall.sh "ConfigAccounts_p.html?setAdmin=&adminuser=admin&adminpw1=$1&adminpw2=$1&access=" > /dev/null && \ +(./protectedPostApiCall.sh "ConfigAccounts_p.html" "setAdmin=&adminuser=admin&adminpw1=$1&adminpw2=$1&access=" && \ echo "Password for User Name 'admin' set to '$1'") || \ (echo "Password setting failed" && \ exit 1) \ No newline at end of file diff --git a/bin/protectedPostApiCall.sh b/bin/protectedPostApiCall.sh new file mode 100755 index 000000000..652abdf35 --- /dev/null +++ b/bin/protectedPostApiCall.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env sh +# Call a YaCy HTTP POST API URL protected by HTTP authentication and transaction token validation +# $1 : API path +# $2 : POST parameters (example : "param1=value1¶m2=value2") +# +# Authentication options : +# - enable unauthenticated local access as administrator : set adminAccountForLocalhost=true in the DATA/SETTINGS/yacy.conf file +# - OR use the legacy Basic HTTP authentication mode (unsecured for remote access): set the "auth-method" to BASIC in the defaults/web.xml file +# - OR use the Digest HTTP authentication mode : set the "auth-method" to DIGEST in the defaults/web.xml file. +# With that last option, the script will run in interactive mode as default, prompting for the administrator password. +# To run in batch mode, you must first export an environment variable filled with the clear-text administrator password before using this script : +# For example with > export YACY_ADMIN_PASSWORD=your_admin_password +# + +cd "`dirname $0`" +port=$(grep ^port= ../DATA/SETTINGS/yacy.conf |cut -d= -f2) +admin=$(grep ^adminAccountUserName= ../DATA/SETTINGS/yacy.conf |cut -d= -f2) +adminAccountForLocalhost=$(grep ^adminAccountForLocalhost= ../DATA/SETTINGS/yacy.conf | cut -d= -f2) + +if grep "BASIC" ../defaults/web.xml > /dev/null; then + # When authentication method is in basic mode, use directly the password hash from the configuration file + YACY_ADMIN_PASSWORD=$(grep ^adminAccountBase64MD5= ../DATA/SETTINGS/yacy.conf |cut -d= -f2) +fi + +if which curl > /dev/null; then + if [ "$adminAccountForLocalhost" = "true" ]; then + # localhost access as administrator without authentication is enabled + + # retrieve the transaction token from the HTTP GET flavor of the URL + transactionToken=$(curl -sSfI "http://127.0.0.1:$port/$1" | grep X-YaCy-Transaction-Token: | awk {'printf $2'} | tr -d '[:space:]') + # send POST data including the transaction token + curl -sSf -d "$2&transactionToken=$transactionToken" "http://127.0.0.1:$port/$1" > /dev/null + + else + if [ -z "$YACY_ADMIN_PASSWORD" ]; then + # no password environment variable : we ask interactively for it only once (not using read -s to be POSIX compliant) + stty -echo + read -p "Enter host password for user '$admin':" YACY_ADMIN_PASSWORD + stty echo + printf "\n" + fi + + # retrieve the transaction token from the HTTP GET flavor of the URL + transactionToken=$(curl -sSfI --anyauth -u "$admin:$YACY_ADMIN_PASSWORD" "http://127.0.0.1:$port/$1" | grep X-YaCy-Transaction-Token: | awk {'printf $2'} | tr -d '[:space:]') + # send POST data including the transaction token + curl -sSf --anyauth -u "$admin:$YACY_ADMIN_PASSWORD" -d "$2&transactionToken=$transactionToken" "http://127.0.0.1:$port/$1" > /dev/null + + fi +elif which wget > /dev/null; then + + if [ "$adminAccountForLocalhost" = "true" ]; then + # localhost access as administrator without authentication is enabled + + # retrieve the transaction token from the HTTP GET flavor of the URL + transactionToken=$(wget -q -t 1 -O - --save-headers --timeout=120 "http://127.0.0.1:$port/$1" | grep X-YaCy-Transaction-Token: | awk {'printf $2'} | tr -d '[:space:]') + # send POST data including the transaction token + wget -nv -t 1 -O /dev/null --timeout=120 --post-data "$2&transactionToken=$transactionToken" "http://127.0.0.1:$port/$1" + + else + if [ -z "$YACY_ADMIN_PASSWORD" ]; then + # no password environment variable : we ask interactively for it only once (not using read -s to be POSIX compliant) + stty -echo + read -p "Enter host password for user '$admin':" YACY_ADMIN_PASSWORD + stty echo + printf "\n" + fi + + # retrieve the transaction token from the HTTP GET flavor of the URL + transactionToken=$(wget -q -t 1 -O - --http-user "$admin" --http-password "$YACY_ADMIN_PASSWORD" --save-headers --timeout=120 "http://127.0.0.1:$port/$1" | grep X-YaCy-Transaction-Token: | awk {'printf $2'} | tr -d '[:space:]') + # send POST data including the transaction token + wget -nv -t 1 -O /dev/null --timeout=120 --http-user "$admin" --http-password "$YACY_ADMIN_PASSWORD" --post-data "$2&transactionToken=$transactionToken" "http://127.0.0.1:$port/$1" + + fi +else + printf "Please install curl or wget\n" > /dev/stderr + exit 1 +fi \ No newline at end of file diff --git a/htroot/ConfigAccounts_p.html b/htroot/ConfigAccounts_p.html index 0d8ade6a9..40bd836aa 100644 --- a/htroot/ConfigAccounts_p.html +++ b/htroot/ConfigAccounts_p.html @@ -30,6 +30,7 @@
Admin Account
+
@@ -60,6 +61,7 @@
Access Rules +
Protection of all pages: if set to on, access to all pages need authorization; if off, only pages with "_p" extension are protected.
@@ -74,6 +76,7 @@
User Accounts +
Select user
:
@@ -93,6 +96,7 @@
+
Edit current user: #[username]# diff --git a/htroot/ConfigAccounts_p.java b/htroot/ConfigAccounts_p.java index d21ec6d4d..7ab11efc9 100644 --- a/htroot/ConfigAccounts_p.java +++ b/htroot/ConfigAccounts_p.java @@ -33,9 +33,11 @@ import java.util.EnumMap; import java.util.HashMap; import java.util.Iterator; import java.util.Map; + import net.yacy.cora.order.Digest; import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.util.ConcurrentLog; +import net.yacy.data.TransactionManager; import net.yacy.data.UserDB; import net.yacy.data.UserDB.AccessRight; import net.yacy.http.Jetty9HttpServerImpl; @@ -45,10 +47,14 @@ import net.yacy.server.serverObjects; import net.yacy.server.serverSwitch; public class ConfigAccounts_p { - - public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, final serverObjects post, final serverSwitch env) { + + public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) { final serverObjects prop = new serverObjects(); + + /* Acquire a transaction token for the next POST form submission */ + prop.put(TransactionManager.TRANSACTION_TOKEN_PARAM, TransactionManager.getTransactionToken(header)); + final Switchboard sb = Switchboard.getSwitchboard(); UserDB.Entry entry = null; @@ -56,10 +62,12 @@ public class ConfigAccounts_p { boolean localhostAccess = sb.getConfigBool(SwitchboardConstants.ADMIN_ACCOUNT_FOR_LOCALHOST, false); if (post != null && post.containsKey("setAccess")) { + TransactionManager.checkPostTransaction(header, post); sb.setConfig(SwitchboardConstants.ADMIN_ACCOUNT_All_PAGES, post.getBoolean(SwitchboardConstants.ADMIN_ACCOUNT_All_PAGES)); } if (post != null && post.containsKey("setAdmin")) { + TransactionManager.checkPostTransaction(header, post); localhostAccess = post.get("access", "").equals("localhost"); final String user = post.get("adminuser", ""); final String pw1 = post.get("adminpw1", ""); @@ -149,6 +157,7 @@ public class ConfigAccounts_p { //user=from userlist //current_user = edited user } else if (post.containsKey("user") && !"newuser".equals(post.get("user"))){ + TransactionManager.checkPostTransaction(header, post); if (post.containsKey("change_user")) { //defaults for newuser are set above entry = sb.userDB.getEntry(post.get("user")); @@ -174,6 +183,7 @@ public class ConfigAccounts_p { sb.userDB.removeEntry(post.get("user")); } } else if (post.containsKey("change")) { //New User / edit User + TransactionManager.checkPostTransaction(header, post); prop.put("text", "0"); prop.put("error", "0"); diff --git a/htroot/ConfigProperties_p.html b/htroot/ConfigProperties_p.html index 98699eb78..ed1bc4333 100644 --- a/htroot/ConfigProperties_p.html +++ b/htroot/ConfigProperties_p.html @@ -55,6 +55,7 @@ #{/options}#
+
:   diff --git a/htroot/ConfigProperties_p.java b/htroot/ConfigProperties_p.java index 04a900951..39e262a7b 100644 --- a/htroot/ConfigProperties_p.java +++ b/htroot/ConfigProperties_p.java @@ -34,20 +34,27 @@ import java.util.Iterator; import java.util.List; import net.yacy.cora.protocol.RequestHeader; +import net.yacy.data.TransactionManager; import net.yacy.server.serverObjects; import net.yacy.server.serverSwitch; public class ConfigProperties_p { - public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, final serverObjects post, final serverSwitch env) { + public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) { // return variable that accumulates replacements final serverObjects prop = new serverObjects(); + + /* Acquire a transaction token for the next POST form submission */ + prop.put(TransactionManager.TRANSACTION_TOKEN_PARAM, TransactionManager.getTransactionToken(header)); String key = ""; String value = ""; //change a key if (post != null && post.containsKey("key") && post.containsKey("value")) { + /* Check the transaction is valid */ + TransactionManager.checkPostTransaction(header, post); + key = post.get("key").trim(); value = post.get("value").trim(); if (key != null && !key.isEmpty()) { diff --git a/htroot/ConfigUpdate_p.html b/htroot/ConfigUpdate_p.html index 2be2f3ee6..e5228281f 100644 --- a/htroot/ConfigUpdate_p.html +++ b/htroot/ConfigUpdate_p.html @@ -1,7 +1,6 @@ - - + + - #(forwardToSteering)#::#(/forwardToSteering)# YaCy '#[clientname]#': System Update #%env/templates/metas.template%# @@ -35,10 +34,11 @@ #(/downloadError)#


Downloaded Releases
-

+

#(downloadsAvailable)# No downloaded releases available for deployment. :: +  #(/downloadsAvailable)# #(deployenabled)#::no automated installation on development environments:: +    -    +    #(/deployenabled)#


Automatic Update
diff --git a/htroot/ConfigUpdate_p.java b/htroot/ConfigUpdate_p.java index f11e6ebab..7a49807fe 100644 --- a/htroot/ConfigUpdate_p.java +++ b/htroot/ConfigUpdate_p.java @@ -35,6 +35,7 @@ import java.util.TreeSet; import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.util.ConcurrentLog; +import net.yacy.data.TransactionManager; import net.yacy.kelondro.util.FileUtils; import net.yacy.kelondro.util.OS; import net.yacy.peers.operation.yacyBuildProperties; @@ -46,7 +47,7 @@ import net.yacy.server.serverSwitch; public class ConfigUpdate_p { - public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, final serverObjects post, final serverSwitch env) { + public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) { // return variable that accumulates replacements final serverObjects prop = new serverObjects(); final Switchboard sb = (Switchboard) env; @@ -73,15 +74,6 @@ public class ConfigUpdate_p { prop.put("candeploy_downloadError", "0"); if (post != null) { - // check if update is supposed to be installed and a release is defined - if (post.containsKey("update") && !post.get("releaseinstall", "").isEmpty()) { - prop.put("forwardToSteering", "1"); - prop.putHTML("forwardToSteering_release",post.get("releaseinstall", "")); - prop.put("deploys", "1"); - prop.put("candeploy", "2"); // display nothing else - return prop; - } - if (post.containsKey("downloadRelease")) { // download a release final String release = post.get("releasedownload", ""); @@ -208,6 +200,8 @@ public class ConfigUpdate_p { // check if there are any downloaded releases and if there are enable the update buttons prop.put("candeploy_downloadsAvailable", (downloadedReleases.isEmpty()) ? "0" : "1"); prop.put("candeploy_deployenabled_buttonsActive", (downloadedReleases.isEmpty() || devenvironment) ? "0" : "1"); + /* Acquire a transaction token for the update operation */ + prop.put("candeploy_deployenabled_" + TransactionManager.TRANSACTION_TOKEN_PARAM, TransactionManager.getTransactionToken(header, "/Steering.html")); int relcount = 0; for(final yacyRelease release : downloadedReleases) { diff --git a/htroot/IndexControlRWIs_p.html b/htroot/IndexControlRWIs_p.html index 90f5afaa8..ef9f62738 100644 --- a/htroot/IndexControlRWIs_p.html +++ b/htroot/IndexControlRWIs_p.html @@ -28,6 +28,7 @@ #(limitations)#::
+
Limitations
Index Reference Size
@@ -47,6 +48,7 @@

No entry for word hash #[wordhash]#

::

Search result: + diff --git a/htroot/IndexControlRWIs_p.java b/htroot/IndexControlRWIs_p.java index cf6b023e2..6c80e4346 100644 --- a/htroot/IndexControlRWIs_p.java +++ b/htroot/IndexControlRWIs_p.java @@ -46,6 +46,7 @@ import net.yacy.cora.util.ByteBuffer; import net.yacy.cora.util.ConcurrentLog; import net.yacy.cora.util.SpaceExceededException; import net.yacy.data.ListManager; +import net.yacy.data.TransactionManager; import net.yacy.document.Tokenizer; import net.yacy.kelondro.data.meta.URIMetadataNode; import net.yacy.kelondro.data.word.Word; @@ -80,7 +81,7 @@ public class IndexControlRWIs_p { private final static String errmsg = "not possible to compute word from hash"; - public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, final serverObjects post, final serverSwitch env) { + public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) { // return variable that accumulates replacements final Switchboard sb = (Switchboard) env; final serverObjects prop = new serverObjects(); @@ -89,7 +90,12 @@ public class IndexControlRWIs_p { prop.putHTML("keystring", ""); prop.put("keyhash", ""); prop.put("result", ""); - prop.put("limitations", post == null || post.containsKey("maxReferencesLimit") ? 1 : 0); + final boolean limitationsEnabled = (post == null || post.containsKey("maxReferencesLimit")); + prop.put("limitations", limitationsEnabled ? 1 : 0); + if(limitationsEnabled) { + /* Acquire a transaction token for the next available POST form submission */ + prop.put("limitations_" + TransactionManager.TRANSACTION_TOKEN_PARAM, TransactionManager.getTransactionToken(header)); + } // switch off all optional forms/lists prop.put("searchresult", 0); @@ -152,6 +158,9 @@ public class IndexControlRWIs_p { // set reference limitation if ( post.containsKey("maxReferencesLimit") ) { + /* Check the transaction is valid */ + TransactionManager.checkPostTransaction(header, post); + if ( post.get("maxReferencesRadio", "").equals("on") ) { ReferenceContainer.maxReferences = post.getInt("maxReferences", 0); } else { @@ -162,6 +171,9 @@ public class IndexControlRWIs_p { // delete word if ( post.containsKey("keyhashdeleteall") ) { + /* Check the transaction is valid */ + TransactionManager.checkPostTransaction(header, post); + try { if ( delurl || delurlref ) { // generate urlx: an array of url hashes to be deleted @@ -245,6 +257,9 @@ public class IndexControlRWIs_p { // transfer to other peer if ( post.containsKey("keyhashtransfer") ) { + /* Check the transaction is valid */ + TransactionManager.checkPostTransaction(header, post); + try { if ( keystring.isEmpty() || !ByteBuffer.equals(Word.word2hash(keystring), keyhash) ) { prop.put("keystring", "<" + errmsg + ">"); @@ -454,6 +469,9 @@ public class IndexControlRWIs_p { } if ( prop.getInt("searchresult", 0) == 3 ) { + /* Acquire a transaction token for the next available POST form submissions */ + prop.put("searchresult_" + TransactionManager.TRANSACTION_TOKEN_PARAM, TransactionManager.getTransactionToken(header)); + listHosts(prop, keyhash, sb); } } diff --git a/htroot/IndexControlURLs_p.html b/htroot/IndexControlURLs_p.html index d1d5964ea..e5b0ff38d 100644 --- a/htroot/IndexControlURLs_p.html +++ b/htroot/IndexControlURLs_p.html @@ -94,6 +94,7 @@ function updatepage(str) { + Cleanup
Index Deletion
@@ -115,6 +116,7 @@ function updatepage(str) { #(dumprestore)#:: +
Optimize Solr
 
@@ -125,6 +127,7 @@ function updatepage(str) {
+
Reboot Solr Core
 
@@ -160,6 +163,7 @@ function updatepage(str) {
 
+
@@ -188,6 +192,7 @@ function updatepage(str) {

+ diff --git a/htroot/IndexControlURLs_p.java b/htroot/IndexControlURLs_p.java index 3f993365f..42bbacfe8 100644 --- a/htroot/IndexControlURLs_p.java +++ b/htroot/IndexControlURLs_p.java @@ -39,6 +39,7 @@ import net.yacy.cora.sorting.ReversibleScoreMap; import net.yacy.cora.util.ConcurrentLog; import net.yacy.crawler.data.Cache; import net.yacy.crawler.data.ResultURLs; +import net.yacy.data.TransactionManager; import net.yacy.data.WorkTables; import net.yacy.kelondro.data.meta.URIMetadataNode; import net.yacy.kelondro.data.word.Word; @@ -51,11 +52,15 @@ import net.yacy.server.serverSwitch; public class IndexControlURLs_p { - public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, final serverObjects post, final serverSwitch env) { + public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) { // return variable that accumulates replacements final Switchboard sb = (Switchboard) env; final serverObjects prop = new serverObjects(); + + /* Acquire a transaction token for the next possible POST form submissions */ + final String nextTransactionToken = TransactionManager.getTransactionToken(header); + prop.put(TransactionManager.TRANSACTION_TOKEN_PARAM, nextTransactionToken); Segment segment = sb.index; long ucount = segment.fulltext().collectionSize(); @@ -72,6 +77,7 @@ public class IndexControlURLs_p { prop.put("reload", 0); prop.put("reload", 0); prop.put("dumprestore", 1); + prop.put("dumprestore_" + TransactionManager.TRANSACTION_TOKEN_PARAM, nextTransactionToken); List dumpFiles = segment.fulltext().dumpFiles(); prop.put("dumprestore_dumpfile", dumpFiles.size() == 0 ? "" : dumpFiles.get(dumpFiles.size() - 1).getAbsolutePath()); prop.put("dumprestore_optimizemax", 10); @@ -107,6 +113,9 @@ public class IndexControlURLs_p { // delete everything if ( post.containsKey("deletecomplete") ) { + /* Check the transaction is valid */ + TransactionManager.checkPostTransaction(header, post); + if ( post.get("deleteIndex", "").equals("on") ) { try {segment.fulltext().clearLocalSolr();} catch (final IOException e) {} } @@ -137,12 +146,18 @@ public class IndexControlURLs_p { } if (post.containsKey("urlhashdeleteall")) { + /* Check the transaction is valid */ + TransactionManager.checkPostTransaction(header, post); + ClientIdentification.Agent agent = ClientIdentification.getAgent(post.get("agentName", ClientIdentification.yacyInternetCrawlerAgentName)); int i = segment.removeAllUrlReferences(urlhash.getBytes(), sb.loader, agent, CacheStrategy.IFEXIST); prop.put("result", "Deleted URL and " + i + " references from " + i + " word indexes."); } if (post.containsKey("urlhashdelete")) { + /* Check the transaction is valid */ + TransactionManager.checkPostTransaction(header, post); + DigestURL url; try { url = segment.fulltext().getURL(urlhash); @@ -160,6 +175,9 @@ public class IndexControlURLs_p { } if (post.containsKey("urldelete")) { + /* Check the transaction is valid */ + TransactionManager.checkPostTransaction(header, post); + try { urlhash = ASCII.String((new DigestURL(urlstring)).hash()); } catch (final MalformedURLException e) { @@ -184,7 +202,7 @@ public class IndexControlURLs_p { prop.putHTML("urlstring", urlstring); prop.put("urlhash", ""); } else { - prop.putAll(genUrlProfile(segment, entry, urlhash)); + prop.putAll(genUrlProfile(segment, entry, urlhash, nextTransactionToken)); prop.put("statistics", 0); } } catch (final MalformedURLException e) { @@ -199,23 +217,32 @@ public class IndexControlURLs_p { prop.putHTML("result", "No Entry for URL hash " + urlhash); } else { prop.putHTML("urlstring", entry.url().toNormalform(true)); - prop.putAll(genUrlProfile(segment, entry, urlhash)); + prop.putAll(genUrlProfile(segment, entry, urlhash, nextTransactionToken)); prop.put("statistics", 0); } } if (post.containsKey("optimizesolr")) { + /* Check the transaction is valid */ + TransactionManager.checkPostTransaction(header, post); + final int size = post.getInt("optimizemax", 10); segment.fulltext().optimize(size); sb.tables.recordAPICall(post, "IndexControlURLs_p.html", WorkTables.TABLE_API_TYPE_STEERING, "solr optimize " + size); } if (post.containsKey("rebootsolr")) { + /* Check the transaction is valid */ + TransactionManager.checkPostTransaction(header, post); + segment.fulltext().rebootSolr(); sb.tables.recordAPICall(post, "IndexControlURLs_p.html", WorkTables.TABLE_API_TYPE_STEERING, "solr reboot"); } if (post.containsKey("deletedomain")) { + /* Check the transaction is valid */ + TransactionManager.checkPostTransaction(header, post); + final String domain = post.get("domain"); Set hostnames = new HashSet(); hostnames.add(domain); @@ -238,6 +265,7 @@ public class IndexControlURLs_p { prop.put("statisticslines_domains_" + cnt + "lines", count); while (statsiter.hasNext() && cnt < count) { hostname = statsiter.next(); + prop.put("statisticslines_domains_" + cnt + "_" + TransactionManager.TRANSACTION_TOKEN_PARAM, nextTransactionToken); prop.put("statisticslines_domains_" + cnt + "_dark", (dark) ? "1" : "0"); prop.put("statisticslines_domains_" + cnt + "_domain", hostname); prop.put("statisticslines_domains_" + cnt + "_count", stats.get(hostname)); @@ -257,7 +285,7 @@ public class IndexControlURLs_p { return prop; } - private static serverObjects genUrlProfile(final Segment segment, final URIMetadataNode entry, final String urlhash) { + private static serverObjects genUrlProfile(final Segment segment, final URIMetadataNode entry, final String urlhash, final String nextTransactionToken) { final serverObjects prop = new serverObjects(); if (entry == null) { prop.put("genUrlProfile", "1"); @@ -271,6 +299,7 @@ public class IndexControlURLs_p { return prop; } prop.put("genUrlProfile", "2"); + prop.put("genUrlProfile_" + TransactionManager.TRANSACTION_TOKEN_PARAM, nextTransactionToken); prop.putHTML("genUrlProfile_urlNormalform", entry.url().toNormalform(true)); prop.put("genUrlProfile_urlhash", urlhash); prop.put("genUrlProfile_urlDescr", entry.dc_title()); diff --git a/htroot/IndexDeletion_p.html b/htroot/IndexDeletion_p.html index 0157773ff..0620d72f5 100644 --- a/htroot/IndexDeletion_p.html +++ b/htroot/IndexDeletion_p.html @@ -11,6 +11,7 @@

Index Deletion

The search index contains #[doccount]# documents. You can delete them here. Deletions are made concurrently which can cause that recently deleted documents are not yet reflected in the document count.

+
Delete by URL Matching

Delete all documents within a sub-path of the given urls. That means all documents must start with one of the url stubs as given here.

@@ -31,7 +32,8 @@
-
+ +
Delete by Age

Delete all documents which are older than a given time period.

@@ -78,7 +80,8 @@
-
+ +
Delete Collections

Delete all documents which are inside specific collections.

@@ -104,6 +107,7 @@
+
Delete by Solr Query

This is the most generic option: select a set of documents using a solr query.

diff --git a/htroot/IndexDeletion_p.java b/htroot/IndexDeletion_p.java index 4c690b32a..0d755a824 100644 --- a/htroot/IndexDeletion_p.java +++ b/htroot/IndexDeletion_p.java @@ -37,6 +37,7 @@ import net.yacy.cora.federate.solr.connector.SolrConnector; import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.sorting.ScoreMap; import net.yacy.cora.util.ConcurrentLog; +import net.yacy.data.TransactionManager; import net.yacy.data.WorkTables; import net.yacy.search.Switchboard; import net.yacy.search.query.QueryModifier; @@ -47,10 +48,13 @@ import net.yacy.server.serverSwitch; public class IndexDeletion_p { - public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, final serverObjects post, final serverSwitch env) { + public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) { // return variable that accumulates replacements final Switchboard sb = (Switchboard) env; final serverObjects prop = new serverObjects(); + + /* Acquire a transaction token for the next POST form submission */ + prop.put(TransactionManager.TRANSACTION_TOKEN_PARAM, TransactionManager.getTransactionToken(header)); SolrConnector defaultConnector = sb.index.fulltext().getDefaultConnector(); SolrConnector webgraphConnector = sb.index.fulltext().getWebgraphConnector(); @@ -121,6 +125,9 @@ public class IndexDeletion_p { int count = post == null ? -1 : post.getInt("count", -1); if (post != null && (post.containsKey("simulate-urldelete") || post.containsKey("engage-urldelete"))) { + /* Check the transaction is valid */ + TransactionManager.checkPostTransaction(header, post); + boolean simulate = post.containsKey("simulate-urldelete"); // parse the input urldelete = urldelete.trim(); @@ -180,6 +187,9 @@ public class IndexDeletion_p { } if (post != null && (post.containsKey("simulate-timedelete") || post.containsKey("engage-timedelete"))) { + /* Check the transaction is valid */ + TransactionManager.checkPostTransaction(header, post); + boolean simulate = post.containsKey("simulate-timedelete"); Date deleteageDate = null; long t = timeParser(timedelete_number, timedelete_unit); // year, month, day, hour @@ -206,6 +216,9 @@ public class IndexDeletion_p { } if (post != null && (post.containsKey("simulate-collectiondelete") || post.containsKey("engage-collectiondelete"))) { + /* Check the transaction is valid */ + TransactionManager.checkPostTransaction(header, post); + boolean simulate = post.containsKey("simulate-collectiondelete"); collectiondelete = collectiondelete.replaceAll(" ","").replaceAll(",", "|"); String query = collectiondelete_mode_unassigned_checked ? "-" + CollectionSchema.collection_sxt + AbstractSolrConnector.CATCHALL_DTERM : collectiondelete.length() == 0 ? CollectionSchema.collection_sxt + ":\"\"" : QueryModifier.parseCollectionExpression(collectiondelete); @@ -228,6 +241,9 @@ public class IndexDeletion_p { } if (post != null && (post.containsKey("simulate-querydelete") || post.containsKey("engage-querydelete"))) { + /* Check the transaction is valid */ + TransactionManager.checkPostTransaction(header, post); + boolean simulate = post.containsKey("simulate-querydelete"); SolrConnector connector = schemaName.equals(CollectionSchema.CORE_NAME) ? defaultConnector : sb.index.fulltext().getWebgraphConnector(); diff --git a/htroot/IndexFederated_p.html b/htroot/IndexFederated_p.html index 95dc1e1f7..d5e139de3 100644 --- a/htroot/IndexFederated_p.html +++ b/htroot/IndexFederated_p.html @@ -14,6 +14,7 @@

+
Solr Search Index Solr stores the main search index. It is the home of two cores, the default 'collection1' core for documents and the 'webgraph' core for a web structure graph. Detailed information about the used Solr fields can be edited in the Schema Editor. @@ -81,7 +82,8 @@
-
+ +
Web Structure Index @@ -100,7 +102,8 @@
-
+ +
Peer-to-Peer Operation diff --git a/htroot/IndexFederated_p.java b/htroot/IndexFederated_p.java index 89b122153..c6031dc8b 100644 --- a/htroot/IndexFederated_p.java +++ b/htroot/IndexFederated_p.java @@ -34,6 +34,7 @@ import net.yacy.cora.federate.solr.instance.RemoteInstance; import net.yacy.cora.federate.solr.instance.ShardInstance; import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.util.ConcurrentLog; +import net.yacy.data.TransactionManager; import net.yacy.kelondro.util.OS; import net.yacy.search.Switchboard; import net.yacy.search.SwitchboardConstants; @@ -42,12 +43,15 @@ import net.yacy.server.serverSwitch; public class IndexFederated_p { - public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, final serverObjects post, final serverSwitch env) { + public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) { // return variable that accumulates replacements final serverObjects prop = new serverObjects(); final Switchboard sb = (Switchboard) env; - + if (post != null && post.containsKey("setrwi")) { + /* Check the transaction is valid */ + TransactionManager.checkPostTransaction(header, post); + //yacy boolean post_core_rwi = post.getBoolean("core.service.rwi"); final boolean previous_core_rwi = sb.index.connectedRWI() && env.getConfigBool(SwitchboardConstants.CORE_SERVICE_RWI, false); @@ -61,6 +65,9 @@ public class IndexFederated_p { } if (post != null && post.containsKey("setcitation")) { + /* Check the transaction is valid */ + TransactionManager.checkPostTransaction(header, post); + boolean post_core_citation = post.getBoolean(SwitchboardConstants.CORE_SERVICE_CITATION); final boolean previous_core_citation = sb.index.connectedCitation() && env.getConfigBool(SwitchboardConstants.CORE_SERVICE_CITATION, false); env.setConfig(SwitchboardConstants.CORE_SERVICE_CITATION, post_core_citation); @@ -76,6 +83,9 @@ public class IndexFederated_p { } if (post != null && post.containsKey("setsolr")) { + /* Check the transaction is valid */ + TransactionManager.checkPostTransaction(header, post); + boolean post_core_fulltext = post.getBoolean(SwitchboardConstants.CORE_SERVICE_FULLTEXT); final boolean previous_core_fulltext = sb.index.fulltext().connectedLocalSolr() && env.getConfigBool(SwitchboardConstants.CORE_SERVICE_FULLTEXT, false); env.setConfig(SwitchboardConstants.CORE_SERVICE_FULLTEXT, post_core_fulltext); @@ -155,6 +165,9 @@ public class IndexFederated_p { boolean lazy = post.getBoolean("solr.indexing.lazy"); env.setConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_LAZY, lazy); } + + /* Acquire a transaction token for the next POST form submission */ + prop.put(TransactionManager.TRANSACTION_TOKEN_PARAM, TransactionManager.getTransactionToken(header)); // show solr host table if (!sb.index.fulltext().connectedRemoteSolr()) { diff --git a/htroot/PerformanceQueues_p.html b/htroot/PerformanceQueues_p.html index d438097fe..a5a9c1a52 100644 --- a/htroot/PerformanceQueues_p.html +++ b/htroot/PerformanceQueues_p.html @@ -11,6 +11,7 @@

Performance Settings of Queues and Processes

+
Scheduled tasks overview and waiting time settings: @@ -70,6 +71,7 @@ +
Cache Settings:
@@ -129,6 +131,7 @@ +
Thread Pool Settings:
diff --git a/htroot/PerformanceQueues_p.java b/htroot/PerformanceQueues_p.java index 23dec663a..539eec912 100644 --- a/htroot/PerformanceQueues_p.java +++ b/htroot/PerformanceQueues_p.java @@ -30,6 +30,7 @@ import java.util.Map; import net.yacy.cora.protocol.ConnectionInfo; import net.yacy.cora.protocol.RequestHeader; +import net.yacy.data.TransactionManager; import net.yacy.kelondro.data.word.WordReference; import net.yacy.kelondro.rwi.IndexCell; import net.yacy.kelondro.util.FileUtils; @@ -50,11 +51,17 @@ public class PerformanceQueues_p { final Switchboard sb = (Switchboard) env; final serverObjects prop = new serverObjects(); File defaultSettingsFile = new File(sb.getAppPath(), "defaults/yacy.init"); + + /* Acquire a transaction token for the next POST form submission */ + prop.put(TransactionManager.TRANSACTION_TOKEN_PARAM, TransactionManager.getTransactionToken(header)); // get segment Segment indexSegment = sb.index; if(post != null) { + /* Check the transaction is valid : validation apply then for every uses of this post parameter */ + TransactionManager.checkPostTransaction(header, post); + if(post.containsKey("defaultFile")){ // TODO check file-path! final File value = new File(sb.getAppPath(), post.get("defaultFile", "defaults/yacy.init")); @@ -70,6 +77,9 @@ public class PerformanceQueues_p { sb.setConfig("javastart_Xmx", "Xmx" + xmx + "m"); sb.setConfig("javastart_Xms", "Xms" + xms + "m"); prop.put("setStartupCommit", "1"); + + /* Acquire a transaction token for the restart operation */ + prop.put("setStartupCommit_" + TransactionManager.TRANSACTION_TOKEN_PARAM, TransactionManager.getTransactionToken(header, "/Steering.html")); } if(post.containsKey("diskFree")) { sb.setConfig(SwitchboardConstants.RESOURCE_DISK_FREE_MIN_STEADYSTATE, post.getLong("diskFree", SwitchboardConstants.RESOURCE_DISK_FREE_MIN_STEADYSTATE_DEFAULT)); diff --git a/htroot/Performance_p.html b/htroot/Performance_p.html index fc8bb6671..a0abe5f98 100644 --- a/htroot/Performance_p.html +++ b/htroot/Performance_p.html @@ -23,19 +23,28 @@ +
Memory Settings
MByte    - #(setStartupCommit)#::
Accepted change. This will take effect after restart of YaCy.
restart now
::
#(/setStartupCommit)#
+ #(setStartupCommit)#::
Accepted change. This will take effect after restart of YaCy.
+
+ + + + +
+ ::
#(/setStartupCommit)# +
Resource Observer
@@ -133,6 +142,7 @@
+
Online Caution Settings:

This is the time that the crawler idles when the proxy is accessed, or a local or remote search is done. diff --git a/htroot/Steering.html b/htroot/Steering.html index 7f6003222..0055941d9 100644 --- a/htroot/Steering.html +++ b/htroot/Steering.html @@ -159,6 +159,23 @@ XDtoU7vQ/wIAAP//AwBb7ktEXQ4nqQAAAABJRU5ErkJggg==" width="128" height="64" alt="K #(info)#

No action submitted
Go back to the Settings page.

+ + + + + + +
+ + + + ::

Your system is not protected by a password
Please go to the User Administration page and set an administration password.

diff --git a/htroot/Steering.java b/htroot/Steering.java index fc9dbdfee..e83cb5460 100644 --- a/htroot/Steering.java +++ b/htroot/Steering.java @@ -33,6 +33,7 @@ import net.yacy.cora.protocol.Domains; import net.yacy.cora.protocol.HeaderFramework; import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.util.ConcurrentLog; +import net.yacy.data.TransactionManager; import net.yacy.kelondro.util.FileUtils; import net.yacy.peers.operation.yacyRelease; import net.yacy.search.Switchboard; @@ -42,7 +43,22 @@ import net.yacy.server.serverSwitch; public class Steering { public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch ss) { - if (post == null || ss == null) { return new serverObjects(); } + if (post == null || post.isEmpty() || ss == null) { + final serverObjects prop = new serverObjects(); + + /* For authenticated user only : acquire a transaction token to pass then to the Steering.html post action */ + if(ss != null && ((Switchboard) ss).verifyAuthentication(header)) { + /* YaCyDefaultServlet will detect it and then also fill the custom HTTP response header used by the JavaScript shutdown and restart actions + * or any external API requesting tool */ + prop.put(TransactionManager.TRANSACTION_TOKEN_PARAM, TransactionManager.getTransactionToken(header)); + /* Also add to the Steering.html page info block for eventual display of this page without parameter */ + prop.put("info_" + TransactionManager.TRANSACTION_TOKEN_PARAM, TransactionManager.getTransactionToken(header)); + } else { + prop.authenticationRequired(); + } + + return prop; + } final Switchboard sb = (Switchboard) ss; final serverObjects prop = new serverObjects(); @@ -58,6 +74,7 @@ public class Steering { } if (post.containsKey("shutdown")) { + TransactionManager.checkPostTransaction(header, post); ConcurrentLog.info("STEERING", "shutdown request from " + requestIP); sb.terminate(10, "shutdown request from Steering; ip = " + requestIP); prop.put("info", "3"); @@ -66,6 +83,7 @@ public class Steering { } if (post.containsKey("restart")) { + TransactionManager.checkPostTransaction(header, post); ConcurrentLog.info("STEERING", "restart request from " + requestIP); yacyRelease.restart(); prop.put("info", "4"); @@ -74,6 +92,7 @@ public class Steering { } if (post.containsKey("update")) { + TransactionManager.checkPostTransaction(header, post); ConcurrentLog.info("STEERING", "update request from " + requestIP); final boolean devenvironment = new File(sb.getAppPath(), ".git").exists(); final String releaseFileName = post.get("releaseinstall", ""); @@ -96,6 +115,7 @@ public class Steering { return prop; } + return prop; } diff --git a/htroot/env/templates/header.template b/htroot/env/templates/header.template index 6c19068d8..9b684a293 100644 --- a/htroot/env/templates/header.template +++ b/htroot/env/templates/header.template @@ -22,12 +22,66 @@
+
  • -
    + + - @@ -35,9 +89,12 @@
  •  
  • - + + - diff --git a/htroot/terminal_p.html b/htroot/terminal_p.html index d4ef88134..2042b39b0 100644 --- a/htroot/terminal_p.html +++ b/htroot/terminal_p.html @@ -44,7 +44,7 @@ function init() { <Search Form>     <Crawl Start>     <Status Page>         -<Shutdown> +<Shutdown>
diff --git a/source/net/yacy/cora/protocol/HeaderFramework.java b/source/net/yacy/cora/protocol/HeaderFramework.java index b049def16..f0bf457f0 100644 --- a/source/net/yacy/cora/protocol/HeaderFramework.java +++ b/source/net/yacy/cora/protocol/HeaderFramework.java @@ -103,6 +103,10 @@ public class HeaderFramework extends TreeMap implements Map