From 1dd9b49e612e2a8e00c3eaaae2528233ce200da1 Mon Sep 17 00:00:00 2001
From: Nikolaus Krismer <nikolaus.krismer@uibk.ac.at>
Date: Thu, 20 Aug 2015 11:42:57 +0200
Subject: [PATCH] heavily increased performance

 - using database exports working with copy (not inserts)
 - using SATA controller I/O cache
 - using virtio network drivers
 - update to centos 7.1
 - update to java jdk8u60
 - activating database logging after data has been imported
 - fixing problems with sqliteadmin (config and db-file copy)
---
 Vagrantfile                          |  5 ++-
 bootstrap/activateDatabaseLogging.sh | 62 ++++++++++++++++++++++++++++
 bootstrap/config.sh                  | 32 ++++++++++++++
 bootstrap/createDatabase.sh          |  2 +-
 bootstrap/importData.sh              |  5 +--
 bootstrap/prepareEnvironment.sh      | 43 +++----------------
 conf/postgresql.conf                 |  4 +-
 7 files changed, 108 insertions(+), 45 deletions(-)
 create mode 100755 bootstrap/activateDatabaseLogging.sh

diff --git a/Vagrantfile b/Vagrantfile
index f052e69..40313a8 100644
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -40,6 +40,8 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
     v.memory = 2048
     v.customize ["modifyvm", :id, "--cpuexecutioncap", "100"]
     v.customize ["modifyvm", :id, "--ostype", "RedHat_64"]
+    v.customize ["modifyvm", :id, "--nictype1", "virtio"]
+    v.customize ["storagectl", :id, "--name", "SATA Controller", "--hostiocache", "on"]
   end
 
   config.trigger.after :destroy do
@@ -51,7 +53,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
   config.cache.enable :yum
 
   # Information about the box itself (name from vagrantcloud)
-  config.vm.box = "jayunit100/centos7"
+  config.vm.box = "box-cutter/centos71"
 
   # Setup of shared folders
   config.vm.synced_folder "conf", "/setup/conf"
@@ -64,6 +66,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
   config.vm.provision "shell", path: "bootstrap/prepareEnvironment.sh", args: ["secretPhdPassword#2014!", (isLocalMode ? "true" : "false")]
   config.vm.provision "shell", path: "bootstrap/createDatabase.sh", args: ["niko", "secretPhdPassword#2014!"]
   config.vm.provision "shell", path: "bootstrap/importData.sh", args: ["niko", "secretPhdPassword#2014!", (isLocalMode ? "true" : "false")]
+  config.vm.provision "shell", path: "bootstrap/activateDatabaseLogging.sh"
 
   # Network configuration
   config.vm.network :forwarded_port, guest: 5432, host: 5432, id: "postgresql", auto_correct: false
diff --git a/bootstrap/activateDatabaseLogging.sh b/bootstrap/activateDatabaseLogging.sh
new file mode 100755
index 0000000..24e5bf4
--- /dev/null
+++ b/bootstrap/activateDatabaseLogging.sh
@@ -0,0 +1,62 @@
+#! /bin/bash
+#############################
+# This script creates postgis (and pgRouting) databases.
+#############################
+
+if [ "$EUID" -ne "0" ]; then
+	echo '- This script must be run as root!'
+	exit 1
+fi
+
+############################
+# Variable definitions     #
+############################
+
+SCRIPT="$(readlink -f ${BASH_SOURCE[0]})"
+SCRIPT_DIR="$(dirname ${SCRIPT})"
+if [ ! -f "${SCRIPT_DIR}/config.sh" ]; then
+	# If config.properties is not besides the bash-script (most likely because vagrant uploaded it into the guest)
+	# we will try to find it in the shared folder
+	SCRIPT_DIR="/vagrant/bootstrap"
+fi
+if [ ! -f "${SCRIPT_DIR}/config.sh" ]; then
+	echo '- No variable declarations found (config.properties file not found)!'
+	exit 1;
+fi
+source "${SCRIPT_DIR}/config.sh"
+
+######################
+# Database creation  #
+######################
+
+echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
+echo " Enabling database logging ($(date +%H:%M:%S)):"
+echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
+
+START=$(date +%s)
+
+POSTGRESQL_GLOB_CONF=$(find /var/lib/pgsql/ | grep postgresql.conf)
+POSTGRESQL_DIR=$(dirname ${POSTGRESQL_GLOB_CONF})
+POSTGRESQL_VERSION=$(cat ${POSTGRESQL_DIR}/PG_VERSION)
+POSTGRESQL_SERVICE_NAME="postgresql-$POSTGRESQL_VERSION"
+
+if [ -f $POSTGRESQL_GLOB_CONF ] && $AVAILABLE_POSTGRES; then
+	echo "Changing settings in postgresql.conf"
+	# Deactivate query logging until data is imported (otherwise the import would be very slow)
+	sed -i 's/log_duration = off/log_duration = on/' $POSTGRESQL_GLOB_CONF
+	sed -i 's/log_min_duration_statement = -1/log_min_duration_statement = 0/' $POSTGRESQL_GLOB_CONF
+
+	echo "Restarting database to activate new settings"
+	fn_service_stop "$POSTGRESQL_SERVICE_NAME"
+	fn_service_start "$POSTGRESQL_SERVICE_NAME"
+else
+	echo "Not changing database logging file. No configuration file found!"
+fi
+
+END=$(date +%s)
+TOTAL=$(( $END - $START ))
+
+echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
+echo " Database logging enabled"
+printf ' Time to enable database logging: %dh:%dm:%ds\n' $(($TOTAL/3600)) $(($TOTAL%3600/60)) $(($TOTAL%60))
+echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
diff --git a/bootstrap/config.sh b/bootstrap/config.sh
index daa77b1..fa7048f 100755
--- a/bootstrap/config.sh
+++ b/bootstrap/config.sh
@@ -131,3 +131,35 @@ fn_download_newer() {
 		fi
 	fi
 }
+
+fn_service_start() {
+	local SERVICE_NAME="$1"
+
+	echo "Starting service \"${SERVICE_NAME}\""
+	touch "service_start_$SERVICE_NAME.log" 2>&1
+	if [ "$DISTRI_TYPE" == "CentOS 6" ]; then
+		service $SERVICE_NAME start >> "service_start_$SERVICE_NAME.log" 2>&1
+		chkconfig $SERVICE_NAME on "service_start_$SERVICE_NAME.log" 2>&1
+	else
+		systemctl start $SERVICE_NAME >> "service_start_$SERVICE_NAME.log" 2>&1
+		systemctl enable $SERVICE_NAME >> "service_start_$SERVICE_NAME.log" 2>&1
+	fi
+
+	sleep 2
+}
+
+fn_service_stop() {
+	local SERVICE_NAME="$1"
+
+	touch "service_stop_$SERVICE_NAME.log" 2>&1
+	if [ "$DISTRI_TYPE" == "CentOS 6" ]; then
+		service $SERVICE_NAME stop >> "service_stop_$SERVICE_NAME.log" 2>&1
+		chkconfig $SERVICE_NAME off "service_stop_$SERVICE_NAME.log" 2>&1
+	else
+		systemctl stop $SERVICE_NAME >> "service_stop_$SERVICE_NAME.log" 2>&1
+		systemctl disable $SERVICE_NAME >> "service_stop_$SERVICE_NAME.log" 2>&1
+	fi
+
+	sleep 2
+}
+
diff --git a/bootstrap/createDatabase.sh b/bootstrap/createDatabase.sh
index 4d87bd3..f53b545 100755
--- a/bootstrap/createDatabase.sh
+++ b/bootstrap/createDatabase.sh
@@ -77,5 +77,5 @@ TOTAL=$(( $END - $START ))
 
 echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
 echo " Database prepared"
-printf 'Time to prepare the database: %dh:%dm:%ds\n' $(($TOTAL/3600)) $(($TOTAL%3600/60)) $(($TOTAL%60))
+printf ' Time to prepare the database: %dh:%dm:%ds\n' $(($TOTAL/3600)) $(($TOTAL%3600/60)) $(($TOTAL%60))
 echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
diff --git a/bootstrap/importData.sh b/bootstrap/importData.sh
index 510fc77..0ff96e7 100755
--- a/bootstrap/importData.sh
+++ b/bootstrap/importData.sh
@@ -229,8 +229,7 @@ fi
 if $AVAILABLE_SPATIALITE_ADMIN; then
 	echo "Importing spatialite database"
 	fn_download_newer $DOWNLOAD_DIR/isochrone.db $CACHE_DIR_REMOTE_DATA/201505_spatialite.db
-
-	cp $DOWNLOAD_DIR/isochrone.db $DATA_DIR/isochrone.db
+	cp -f $DOWNLOAD_DIR/isochrone.db $DATA_DIR/
 
 	echo "  - setting spatialite database permissions"
 	chown -R apache:apache $DATA_DIR/isochrone.db
@@ -242,5 +241,5 @@ TOTAL=$(( $END - $START ))
 
 echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
 echo " Datasets imported"
-printf 'Time to import the datasets: %dh:%dm:%ds\n' $(($TOTAL/3600)) $(($TOTAL%3600/60)) $(($TOTAL%60))
+printf ' Time to import the datasets: %dh:%dm:%ds\n' $(($TOTAL/3600)) $(($TOTAL%3600/60)) $(($TOTAL%60))
 echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
diff --git a/bootstrap/prepareEnvironment.sh b/bootstrap/prepareEnvironment.sh
index 5774013..3e3a378 100755
--- a/bootstrap/prepareEnvironment.sh
+++ b/bootstrap/prepareEnvironment.sh
@@ -59,7 +59,7 @@ GEOSERVER_FILE="geoserver-$GEOSERVER_VERSION-war.zip"
 GEOSERVER_DOWNLOAD_URL="http://sourceforge.net/projects/geoserver/files/GeoServer/$GEOSERVER_VERSION/$GEOSERVER_FILE"
 GEOSERVER_SHELL_VERSION="0.2-SNAPSHOT"
 GEOSERVER_SHELL_FILE="gs-shell-$GEOSERVER_SHELL_VERSION-app.zip"
-JAVA_VERSION="8u51"
+JAVA_VERSION="8u60"
 JAVA_FILE_32="jdk-$JAVA_VERSION-linux-i586.rpm"
 JAVA_FILE_64="jdk-$JAVA_VERSION-linux-x64.rpm"
 JAVA_CRYPTO_FILE="jce_policy-8.zip"
@@ -226,41 +226,6 @@ else
 	JAVA_FILE="$JAVA_FILE_32"
 fi
 
-############################
-# Function definitions     #
-############################
-
-fn_service_start() {
-	local SERVICE_NAME="$1"
-
-	echo "Starting service \"${SERVICE_NAME}\""
-	touch "service_start_$SERVICE_NAME.log" 2>&1
-	if [ "$DISTRI_TYPE" == "CentOS 6" ]; then
-		service $SERVICE_NAME start >> "service_start_$SERVICE_NAME.log" 2>&1
-		chkconfig $SERVICE_NAME on "service_start_$SERVICE_NAME.log" 2>&1
-	else
-		systemctl start $SERVICE_NAME >> "service_start_$SERVICE_NAME.log" 2>&1
-		systemctl enable $SERVICE_NAME >> "service_start_$SERVICE_NAME.log" 2>&1
-	fi
-
-	sleep 2
-}
-
-fn_service_stop() {
-	local SERVICE_NAME="$1"
-
-	touch "service_stop_$SERVICE_NAME.log" 2>&1
-	if [ "$DISTRI_TYPE" == "CentOS 6" ]; then
-		service $SERVICE_NAME stop >> "service_stop_$SERVICE_NAME.log" 2>&1
-		chkconfig $SERVICE_NAME off "service_stop_$SERVICE_NAME.log" 2>&1
-	else
-		systemctl stop $SERVICE_NAME >> "service_stop_$SERVICE_NAME.log" 2>&1
-		systemctl disable $SERVICE_NAME >> "service_stop_$SERVICE_NAME.log" 2>&1
-	fi
-
-	sleep 2
-}
-
 echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
 echo " Performing basic system preparations ($(date +%H:%M:%S)):"
 echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
@@ -572,7 +537,7 @@ if $INSTALL_ISOCHRONE_DATAMODEL; then
 	echo "  - artifact: $NEXUS_SNAPSHOT_ISOCHRONE_DATAMODEL"
 
 	fn_download $DOWNLOAD_DIR/$NEXUS_ARTIFACT_NAME_DATAMODEL.jar $NEXUS_SNAPSHOT_ISOCHRONE_DATAMODEL
-	cp $DOWNLOAD_DIR/$INSTALL_ISOCHRONE_DATAMODEL.jar $DEPLOY_DIR/
+	cp $DOWNLOAD_DIR/$NEXUS_ARTIFACT_NAME_DATAMODEL.jar $DEPLOY_DIR/
 fi
 
 if $INSTALL_ISOCHRONE_TOOLS; then
@@ -794,6 +759,8 @@ if $INSTALL_SPATIALITE_ADMIN; then
 	sed -i ':a;N;$!ba;s/\tarray(/lineWillBeDeleted/2' $DEPLOY_DIR/phpliteadmin/phpliteadmin.config.php
 	sed -i ':a;N;$!ba;s/\t),/lineWillBeDeleted/2' $DEPLOY_DIR/phpliteadmin/phpliteadmin.config.php
 	sed -i "/lineWillBeDeleted/d" $DEPLOY_DIR/phpliteadmin/phpliteadmin.config.php
+
+	ln -sf $DEPLOY_DIR/phpliteadmin/phpliteadmin.php $DEPLOY_DIR/phpliteadmin/index.php
 fi
 
 echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
@@ -823,5 +790,5 @@ touch /var/vagrant_provision
 END=$(date +%s)
 TOTAL=$(( $END - $START ))
 
-printf 'Time to prepare the environment: %dh:%dm:%ds\n' $(($TOTAL/3600)) $(($TOTAL%3600/60)) $(($TOTAL%60))
+printf ' Time to prepare the environment: %dh:%dm:%ds\n' $(($TOTAL/3600)) $(($TOTAL%3600/60)) $(($TOTAL%60))
 echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
diff --git a/conf/postgresql.conf b/conf/postgresql.conf
index 7f09033..69b5e31 100644
--- a/conf/postgresql.conf
+++ b/conf/postgresql.conf
@@ -377,7 +377,7 @@ log_rotation_size = 0			# Automatic rotation of logfiles will
 					#   fatal
 					#   panic (effectively off)
 
-log_min_duration_statement = 0		# -1 is disabled, 0 logs all statements
+log_min_duration_statement = -1		# -1 is disabled, 0 logs all statements
 					# and their durations, > 0 logs only
 					# statements running at least this number
 					# of milliseconds
@@ -392,7 +392,7 @@ log_min_duration_statement = 0		# -1 is disabled, 0 logs all statements
 #log_checkpoints = off
 #log_connections = off
 #log_disconnections = off
-log_duration = on
+log_duration = off
 #log_error_verbosity = default		# terse, default, or verbose messages
 log_hostname = off
 log_line_prefix = '< %m >'		# special values:
-- 
GitLab