Added support for custom DATA path in shell scripts

When the YACY_DATA_PATH environment variable is set, shell scripts will
now use the given path instead of relative ../DATA which remains the
default when the variable is not set.

Necessary in the context of Snap package (see issue #254) as YaCy is
started with startYACY.sh and an absolute DATA parent path in parameter.

bin/apicall.sh

@@ -1,5 +1,8 @@
 #!/usr/bin/env sh
 # Call an HTTP API on the local YaCy peer, authenticated as administrator
+# $1 : API path
+#
+# $YACY_DATA_PATH : path to the YaCy DATA folder to use. When not set, the relative ../DATA path is used as a default.
 #
 # Authentication options :
 # - enable unauthenticated local access as administrator : set adminAccountForLocalhost=true in the DATA/SETTINGS/yacy.conf file
@@ -11,13 +14,15 @@
 # 
 
 cd "`dirname $0`"
-port=$(grep ^port= ../DATA/SETTINGS/yacy.conf |cut -d= -f2)
-admin=$(grep ^adminAccountUserName= ../DATA/SETTINGS/yacy.conf |cut -d= -f2)
-adminAccountForLocalhost=$(grep ^adminAccountForLocalhost= ../DATA/SETTINGS/yacy.conf | cut -d= -f2)
+. ./checkDataFolder.sh
+
+port=$(grep ^port= "$YACY_DATA_PATH/SETTINGS/yacy.conf" |cut -d= -f2)
+admin=$(grep ^adminAccountUserName= "$YACY_DATA_PATH/SETTINGS/yacy.conf" |cut -d= -f2)
+adminAccountForLocalhost=$(grep ^adminAccountForLocalhost= "$YACY_DATA_PATH/SETTINGS/yacy.conf" | cut -d= -f2)
 
-if grep "<auth-method>BASIC</auth-method>" ../defaults/web.xml > /dev/null; then
+if grep "<auth-method>BASIC</auth-method>" "$YACY_APP_PATH/defaults/web.xml" > /dev/null; then
 	# When authentication method is in basic mode, use directly the password hash from the configuration file 
-	YACY_ADMIN_PASSWORD=$(grep ^adminAccountBase64MD5= ../DATA/SETTINGS/yacy.conf |cut -d= -f2)
+	YACY_ADMIN_PASSWORD=$(grep ^adminAccountBase64MD5= "$YACY_DATA_PATH/SETTINGS/yacy.conf" |cut -d= -f2)
 fi
 
 if which curl > /dev/null; then

bin/apicat.sh

@@ -2,6 +2,10 @@
 # Call an HTTP API on the local YaCy peer, authenticated as administrator, then print the result on the standard output
 # Almost the same as apicall.sh, except that wget doesn't print information messages to the standard output, only the result
 #
+# $1 : API path
+#
+# $YACY_DATA_PATH : path to the YaCy DATA folder to use. When not set, the relative ../DATA path is used as a default.
+#
 # Authentication options :
 # - enable unauthenticated local access as administrator : set adminAccountForLocalhost=true in the DATA/SETTINGS/yacy.conf file
 # - OR use the legacy Basic HTTP authentication mode (unsecured for remote access): set the "auth-method" to BASIC in the defaults/web.xml file
@@ -12,13 +16,15 @@
 # 
 
 cd "`dirname $0`"
-port=$(grep ^port= ../DATA/SETTINGS/yacy.conf |cut -d= -f2)
-admin=$(grep ^adminAccountUserName= ../DATA/SETTINGS/yacy.conf |cut -d= -f2)
-adminAccountForLocalhost=$(grep ^adminAccountForLocalhost= ../DATA/SETTINGS/yacy.conf | cut -d= -f2)
+. ./checkDataFolder.sh
+
+port=$(grep ^port= "$YACY_DATA_PATH/SETTINGS/yacy.conf" |cut -d= -f2)
+admin=$(grep ^adminAccountUserName= "$YACY_DATA_PATH/SETTINGS/yacy.conf" |cut -d= -f2)
+adminAccountForLocalhost=$(grep ^adminAccountForLocalhost= "$YACY_DATA_PATH/SETTINGS/yacy.conf" | cut -d= -f2)
 
-if grep "<auth-method>BASIC</auth-method>" ../defaults/web.xml > /dev/null; then
+if grep "<auth-method>BASIC</auth-method>" "$YACY_APP_PATH/defaults/web.xml" > /dev/null; then
 	# When authentication method is in basic mode, use directly the password hash from the configuration file 
-	YACY_ADMIN_PASSWORD=$(grep ^adminAccountBase64MD5= ../DATA/SETTINGS/yacy.conf |cut -d= -f2)
+	YACY_ADMIN_PASSWORD=$(grep ^adminAccountBase64MD5= "$YACY_DATA_PATH/SETTINGS/yacy.conf" |cut -d= -f2)
 fi
 
 if which curl > /dev/null; then

bin/checkDataFolder.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env sh
+# Fill the YACY_BIN_PATH, YACY_APP_PATH and the YACY_DATA_PATH (when empty) relatively to the current working directory
+# Then check that the provided YaCy DATA folder exists
+# Exits with an error message and status 2 when the DATA folder is not found
+
+YACY_BIN_PATH="`pwd`"
+YACY_APP_PATH="`dirname $YACY_BIN_PATH`"
+
+if [ -z "$YACY_DATA_PATH" ]; then
+	YACY_DATA_PATH="$YACY_APP_PATH/DATA"
+fi
+
+if [ ! -d "$YACY_DATA_PATH" ]; then
+	echo "Invalid YaCy DATA folder path : $YACY_DATA_PATH"
+	echo "Please fill the YACY_DATA_PATH environment variable with a valid YaCy DATA folder path."
+	exit 2
+fi

bin/checkalive.sh

@@ -1,11 +1,12 @@
 #!/usr/bin/env sh
-cd "`dirname $0`"
-
 # for a production environment with high-availability requirement,
 # (and if you are using the debian version of yacy)
 # add the following line in /etc/crontab
 # 0 *	* * *	root    cd /usr/share/yacy/bin && ./checkalive.sh
 
+cd "`dirname $0`"
+. ./checkDataFolder.sh
+
 FLAG=0
 if [ `./apicall.sh /Status.html | grep "</html>"` ]; then
   FLAG=1
@@ -15,7 +16,7 @@ if [ $FLAG -eq '0' ]; then
     cd ..
     timeout 30s ./stopYACY.sh
     ./killYACY.sh
-    rm DATA/yacy.running
+    rm "$YACY_DATA_PATH/yacy.running"
     ./startYACY.sh
 fi
 exit

bin/checkindex.sh

@@ -1,8 +1,12 @@
 #!/usr/bin/env sh
-cd "`dirname $0`/.."
-for i in DATA/INDEX/* ; do
+cd "`dirname $0`"
+. ./checkDataFolder.sh
+
+
+for i in "$YACY_DATA_PATH/INDEX"/* ; do
   if [ -d "$i" ]; then
-      java -cp 'lib/*' org.apache.lucene.index.CheckIndex  $i/SEGMENTS/solr_46/collection1/data/index/ -fix
+  	  echo "Checking Solr index at $i..."
+      java -cp "$YACY_APP_PATH/lib/*" org.apache.lucene.index.CheckIndex  "$i/SEGMENTS/solr_6_6/collection1/data/index/" -exorcise
   fi
 done
 cd ..

bin/down.sh

@@ -1,6 +1,8 @@
 #!/usr/bin/env sh
 cd "`dirname $0`"
-port=$(grep ^port= ../DATA/SETTINGS/yacy.conf |cut -d= -f2)
+. ./checkDataFolder.sh
+
+port=$(grep ^port= "$YACY_DATA_PATH/SETTINGS/yacy.conf" |cut -d= -f2)
 
 if which curl > /dev/null; then
   curl -s "http://localhost:$port/Network.xml?page=2&ip=" | awk '/<address>/{ gsub("<address>","" );gsub("<\/address>","" ); print $0 }' | awk '{print $1}'

bin/graphicstest.sh

@@ -1,4 +1,8 @@
-port=$(grep ^port= ../DATA/SETTINGS/yacy.conf |cut -d= -f2)
+#!/usr/bin/env sh
+cd "`dirname $0`"
+. ./checkDataFolder.sh
+
+port=$(grep ^port= "$YACY_DATA_PATH/SETTINGS/yacy.conf" |cut -d= -f2)
 while [ 1 = 1 ]
 do
 curl "http://localhost:$port/NetworkPicture.png?width=768&height=576&bgcolor=FFFFFF" > /dev/null

bin/protectedPostApiCall.sh

@@ -3,6 +3,8 @@
 # $1 : API path
 # $2 : POST parameters (example : "param1=value1&param2=value2")
 #
+# $YACY_DATA_PATH : path to the YaCy DATA folder to use. When not set, the relative ../DATA path is used as a default.
+#
 # Authentication options :
 # - enable unauthenticated local access as administrator : set adminAccountForLocalhost=true in the DATA/SETTINGS/yacy.conf file
 # - OR use the legacy Basic HTTP authentication mode (unsecured for remote access): set the "auth-method" to BASIC in the defaults/web.xml file
@@ -13,13 +15,15 @@
 # 
 
 cd "`dirname $0`"
-port=$(grep ^port= ../DATA/SETTINGS/yacy.conf |cut -d= -f2)
-admin=$(grep ^adminAccountUserName= ../DATA/SETTINGS/yacy.conf |cut -d= -f2)
-adminAccountForLocalhost=$(grep ^adminAccountForLocalhost= ../DATA/SETTINGS/yacy.conf | cut -d= -f2)
+. ./checkDataFolder.sh
+
+port=$(grep ^port= "$YACY_DATA_PATH/SETTINGS/yacy.conf" |cut -d= -f2)
+admin=$(grep ^adminAccountUserName= "$YACY_DATA_PATH/SETTINGS/yacy.conf" |cut -d= -f2)
+adminAccountForLocalhost=$(grep ^adminAccountForLocalhost= "$YACY_DATA_PATH/SETTINGS/yacy.conf" | cut -d= -f2)
 
-if grep "<auth-method>BASIC</auth-method>" ../defaults/web.xml > /dev/null; then
+if grep "<auth-method>BASIC</auth-method>" "$YACY_APP_PATH/defaults/web.xml" > /dev/null; then
 	# When authentication method is in basic mode, use directly the password hash from the configuration file 
-	YACY_ADMIN_PASSWORD=$(grep ^adminAccountBase64MD5= ../DATA/SETTINGS/yacy.conf |cut -d= -f2)
+	YACY_ADMIN_PASSWORD=$(grep ^adminAccountBase64MD5= "$YACY_DATA_PATH/SETTINGS/yacy.conf" |cut -d= -f2)
 fi
 
 if which curl > /dev/null; then

bin/search.sh

@@ -1,4 +1,6 @@
 #!/usr/bin/env sh
 cd "`dirname $0`"
-port=$(grep ^port= ../DATA/SETTINGS/yacy.conf |cut -d= -f2)
+. ./checkDataFolder.sh
+
+port=$(grep ^port= "$YACY_DATA_PATH/SETTINGS/yacy.conf" |cut -d= -f2)
 ./search1.sh -y localhost:$port "$1"

bin/searchall.sh

@@ -1,4 +1,6 @@
 #!/usr/bin/env sh
 cd "`dirname $0`"
-port=$(grep ^port= ../DATA/SETTINGS/yacy.conf |cut -d= -f2)
+. ./checkDataFolder.sh
+
+port=$(grep ^port= "$YACY_DATA_PATH/SETTINGS/yacy.conf" |cut -d= -f2)
 ./searchall1.sh -s localhost:$port $1

bin/up.sh

@@ -2,6 +2,6 @@
 cd "`dirname $0`"
 ./apicall.sh "/Network.xml?page=1&ip=" | awk '/<address>/{ gsub("<address>","" );gsub("<\/address>","" ); print $0 }' | awk '{print $1}';
 
-#port=$(grep ^port= ../DATA/SETTINGS/yacy.conf |cut -d= -f2)
+#port=$(grep ^port= "$YACY_DATA_PATH/SETTINGS/yacy.conf" |cut -d= -f2)
 #./up1.sh localhost:$port