Skip to content
Snippets Groups Projects
Commit 1dd9b49e authored by User expired's avatar User expired
Browse files

heavily increased performance

 - using database exports working with copy (not inserts)
 - using SATA controller I/O cache
 - using virtio network drivers
 - update to centos 7.1
 - update to java jdk8u60
 - activating database logging after data has been imported
 - fixing problems with sqliteadmin (config and db-file copy)
parent 7283f7ae
No related branches found
No related tags found
No related merge requests found
...@@ -40,6 +40,8 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| ...@@ -40,6 +40,8 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
v.memory = 2048 v.memory = 2048
v.customize ["modifyvm", :id, "--cpuexecutioncap", "100"] v.customize ["modifyvm", :id, "--cpuexecutioncap", "100"]
v.customize ["modifyvm", :id, "--ostype", "RedHat_64"] v.customize ["modifyvm", :id, "--ostype", "RedHat_64"]
v.customize ["modifyvm", :id, "--nictype1", "virtio"]
v.customize ["storagectl", :id, "--name", "SATA Controller", "--hostiocache", "on"]
end end
config.trigger.after :destroy do config.trigger.after :destroy do
...@@ -51,7 +53,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| ...@@ -51,7 +53,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
config.cache.enable :yum config.cache.enable :yum
# Information about the box itself (name from vagrantcloud) # Information about the box itself (name from vagrantcloud)
config.vm.box = "jayunit100/centos7" config.vm.box = "box-cutter/centos71"
# Setup of shared folders # Setup of shared folders
config.vm.synced_folder "conf", "/setup/conf" config.vm.synced_folder "conf", "/setup/conf"
...@@ -64,6 +66,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| ...@@ -64,6 +66,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
config.vm.provision "shell", path: "bootstrap/prepareEnvironment.sh", args: ["secretPhdPassword#2014!", (isLocalMode ? "true" : "false")] config.vm.provision "shell", path: "bootstrap/prepareEnvironment.sh", args: ["secretPhdPassword#2014!", (isLocalMode ? "true" : "false")]
config.vm.provision "shell", path: "bootstrap/createDatabase.sh", args: ["niko", "secretPhdPassword#2014!"] config.vm.provision "shell", path: "bootstrap/createDatabase.sh", args: ["niko", "secretPhdPassword#2014!"]
config.vm.provision "shell", path: "bootstrap/importData.sh", args: ["niko", "secretPhdPassword#2014!", (isLocalMode ? "true" : "false")] config.vm.provision "shell", path: "bootstrap/importData.sh", args: ["niko", "secretPhdPassword#2014!", (isLocalMode ? "true" : "false")]
config.vm.provision "shell", path: "bootstrap/activateDatabaseLogging.sh"
# Network configuration # Network configuration
config.vm.network :forwarded_port, guest: 5432, host: 5432, id: "postgresql", auto_correct: false config.vm.network :forwarded_port, guest: 5432, host: 5432, id: "postgresql", auto_correct: false
......
#! /bin/bash
#############################
# This script creates postgis (and pgRouting) databases.
#############################
if [ "$EUID" -ne "0" ]; then
echo '- This script must be run as root!'
exit 1
fi
############################
# Variable definitions #
############################
SCRIPT="$(readlink -f ${BASH_SOURCE[0]})"
SCRIPT_DIR="$(dirname ${SCRIPT})"
if [ ! -f "${SCRIPT_DIR}/config.sh" ]; then
# If config.properties is not besides the bash-script (most likely because vagrant uploaded it into the guest)
# we will try to find it in the shared folder
SCRIPT_DIR="/vagrant/bootstrap"
fi
if [ ! -f "${SCRIPT_DIR}/config.sh" ]; then
echo '- No variable declarations found (config.properties file not found)!'
exit 1;
fi
source "${SCRIPT_DIR}/config.sh"
######################
# Database creation #
######################
echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
echo " Enabling database logging ($(date +%H:%M:%S)):"
echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
START=$(date +%s)
POSTGRESQL_GLOB_CONF=$(find /var/lib/pgsql/ | grep postgresql.conf)
POSTGRESQL_DIR=$(dirname ${POSTGRESQL_GLOB_CONF})
POSTGRESQL_VERSION=$(cat ${POSTGRESQL_DIR}/PG_VERSION)
POSTGRESQL_SERVICE_NAME="postgresql-$POSTGRESQL_VERSION"
if [ -f $POSTGRESQL_GLOB_CONF ] && $AVAILABLE_POSTGRES; then
echo "Changing settings in postgresql.conf"
# Deactivate query logging until data is imported (otherwise the import would be very slow)
sed -i 's/log_duration = off/log_duration = on/' $POSTGRESQL_GLOB_CONF
sed -i 's/log_min_duration_statement = -1/log_min_duration_statement = 0/' $POSTGRESQL_GLOB_CONF
echo "Restarting database to activate new settings"
fn_service_stop "$POSTGRESQL_SERVICE_NAME"
fn_service_start "$POSTGRESQL_SERVICE_NAME"
else
echo "Not changing database logging file. No configuration file found!"
fi
END=$(date +%s)
TOTAL=$(( $END - $START ))
echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
echo " Database logging enabled"
printf ' Time to enable database logging: %dh:%dm:%ds\n' $(($TOTAL/3600)) $(($TOTAL%3600/60)) $(($TOTAL%60))
echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
...@@ -131,3 +131,35 @@ fn_download_newer() { ...@@ -131,3 +131,35 @@ fn_download_newer() {
fi fi
fi fi
} }
fn_service_start() {
local SERVICE_NAME="$1"
echo "Starting service \"${SERVICE_NAME}\""
touch "service_start_$SERVICE_NAME.log" 2>&1
if [ "$DISTRI_TYPE" == "CentOS 6" ]; then
service $SERVICE_NAME start >> "service_start_$SERVICE_NAME.log" 2>&1
chkconfig $SERVICE_NAME on "service_start_$SERVICE_NAME.log" 2>&1
else
systemctl start $SERVICE_NAME >> "service_start_$SERVICE_NAME.log" 2>&1
systemctl enable $SERVICE_NAME >> "service_start_$SERVICE_NAME.log" 2>&1
fi
sleep 2
}
fn_service_stop() {
local SERVICE_NAME="$1"
touch "service_stop_$SERVICE_NAME.log" 2>&1
if [ "$DISTRI_TYPE" == "CentOS 6" ]; then
service $SERVICE_NAME stop >> "service_stop_$SERVICE_NAME.log" 2>&1
chkconfig $SERVICE_NAME off "service_stop_$SERVICE_NAME.log" 2>&1
else
systemctl stop $SERVICE_NAME >> "service_stop_$SERVICE_NAME.log" 2>&1
systemctl disable $SERVICE_NAME >> "service_stop_$SERVICE_NAME.log" 2>&1
fi
sleep 2
}
...@@ -77,5 +77,5 @@ TOTAL=$(( $END - $START )) ...@@ -77,5 +77,5 @@ TOTAL=$(( $END - $START ))
echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
echo " Database prepared" echo " Database prepared"
printf 'Time to prepare the database: %dh:%dm:%ds\n' $(($TOTAL/3600)) $(($TOTAL%3600/60)) $(($TOTAL%60)) printf ' Time to prepare the database: %dh:%dm:%ds\n' $(($TOTAL/3600)) $(($TOTAL%3600/60)) $(($TOTAL%60))
echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
...@@ -229,8 +229,7 @@ fi ...@@ -229,8 +229,7 @@ fi
if $AVAILABLE_SPATIALITE_ADMIN; then if $AVAILABLE_SPATIALITE_ADMIN; then
echo "Importing spatialite database" echo "Importing spatialite database"
fn_download_newer $DOWNLOAD_DIR/isochrone.db $CACHE_DIR_REMOTE_DATA/201505_spatialite.db fn_download_newer $DOWNLOAD_DIR/isochrone.db $CACHE_DIR_REMOTE_DATA/201505_spatialite.db
cp -f $DOWNLOAD_DIR/isochrone.db $DATA_DIR/
cp $DOWNLOAD_DIR/isochrone.db $DATA_DIR/isochrone.db
echo " - setting spatialite database permissions" echo " - setting spatialite database permissions"
chown -R apache:apache $DATA_DIR/isochrone.db chown -R apache:apache $DATA_DIR/isochrone.db
...@@ -242,5 +241,5 @@ TOTAL=$(( $END - $START )) ...@@ -242,5 +241,5 @@ TOTAL=$(( $END - $START ))
echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
echo " Datasets imported" echo " Datasets imported"
printf 'Time to import the datasets: %dh:%dm:%ds\n' $(($TOTAL/3600)) $(($TOTAL%3600/60)) $(($TOTAL%60)) printf ' Time to import the datasets: %dh:%dm:%ds\n' $(($TOTAL/3600)) $(($TOTAL%3600/60)) $(($TOTAL%60))
echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
...@@ -59,7 +59,7 @@ GEOSERVER_FILE="geoserver-$GEOSERVER_VERSION-war.zip" ...@@ -59,7 +59,7 @@ GEOSERVER_FILE="geoserver-$GEOSERVER_VERSION-war.zip"
GEOSERVER_DOWNLOAD_URL="http://sourceforge.net/projects/geoserver/files/GeoServer/$GEOSERVER_VERSION/$GEOSERVER_FILE" GEOSERVER_DOWNLOAD_URL="http://sourceforge.net/projects/geoserver/files/GeoServer/$GEOSERVER_VERSION/$GEOSERVER_FILE"
GEOSERVER_SHELL_VERSION="0.2-SNAPSHOT" GEOSERVER_SHELL_VERSION="0.2-SNAPSHOT"
GEOSERVER_SHELL_FILE="gs-shell-$GEOSERVER_SHELL_VERSION-app.zip" GEOSERVER_SHELL_FILE="gs-shell-$GEOSERVER_SHELL_VERSION-app.zip"
JAVA_VERSION="8u51" JAVA_VERSION="8u60"
JAVA_FILE_32="jdk-$JAVA_VERSION-linux-i586.rpm" JAVA_FILE_32="jdk-$JAVA_VERSION-linux-i586.rpm"
JAVA_FILE_64="jdk-$JAVA_VERSION-linux-x64.rpm" JAVA_FILE_64="jdk-$JAVA_VERSION-linux-x64.rpm"
JAVA_CRYPTO_FILE="jce_policy-8.zip" JAVA_CRYPTO_FILE="jce_policy-8.zip"
...@@ -226,41 +226,6 @@ else ...@@ -226,41 +226,6 @@ else
JAVA_FILE="$JAVA_FILE_32" JAVA_FILE="$JAVA_FILE_32"
fi fi
############################
# Function definitions #
############################
fn_service_start() {
local SERVICE_NAME="$1"
echo "Starting service \"${SERVICE_NAME}\""
touch "service_start_$SERVICE_NAME.log" 2>&1
if [ "$DISTRI_TYPE" == "CentOS 6" ]; then
service $SERVICE_NAME start >> "service_start_$SERVICE_NAME.log" 2>&1
chkconfig $SERVICE_NAME on "service_start_$SERVICE_NAME.log" 2>&1
else
systemctl start $SERVICE_NAME >> "service_start_$SERVICE_NAME.log" 2>&1
systemctl enable $SERVICE_NAME >> "service_start_$SERVICE_NAME.log" 2>&1
fi
sleep 2
}
fn_service_stop() {
local SERVICE_NAME="$1"
touch "service_stop_$SERVICE_NAME.log" 2>&1
if [ "$DISTRI_TYPE" == "CentOS 6" ]; then
service $SERVICE_NAME stop >> "service_stop_$SERVICE_NAME.log" 2>&1
chkconfig $SERVICE_NAME off "service_stop_$SERVICE_NAME.log" 2>&1
else
systemctl stop $SERVICE_NAME >> "service_stop_$SERVICE_NAME.log" 2>&1
systemctl disable $SERVICE_NAME >> "service_stop_$SERVICE_NAME.log" 2>&1
fi
sleep 2
}
echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
echo " Performing basic system preparations ($(date +%H:%M:%S)):" echo " Performing basic system preparations ($(date +%H:%M:%S)):"
echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
...@@ -572,7 +537,7 @@ if $INSTALL_ISOCHRONE_DATAMODEL; then ...@@ -572,7 +537,7 @@ if $INSTALL_ISOCHRONE_DATAMODEL; then
echo " - artifact: $NEXUS_SNAPSHOT_ISOCHRONE_DATAMODEL" echo " - artifact: $NEXUS_SNAPSHOT_ISOCHRONE_DATAMODEL"
fn_download $DOWNLOAD_DIR/$NEXUS_ARTIFACT_NAME_DATAMODEL.jar $NEXUS_SNAPSHOT_ISOCHRONE_DATAMODEL fn_download $DOWNLOAD_DIR/$NEXUS_ARTIFACT_NAME_DATAMODEL.jar $NEXUS_SNAPSHOT_ISOCHRONE_DATAMODEL
cp $DOWNLOAD_DIR/$INSTALL_ISOCHRONE_DATAMODEL.jar $DEPLOY_DIR/ cp $DOWNLOAD_DIR/$NEXUS_ARTIFACT_NAME_DATAMODEL.jar $DEPLOY_DIR/
fi fi
if $INSTALL_ISOCHRONE_TOOLS; then if $INSTALL_ISOCHRONE_TOOLS; then
...@@ -794,6 +759,8 @@ if $INSTALL_SPATIALITE_ADMIN; then ...@@ -794,6 +759,8 @@ if $INSTALL_SPATIALITE_ADMIN; then
sed -i ':a;N;$!ba;s/\tarray(/lineWillBeDeleted/2' $DEPLOY_DIR/phpliteadmin/phpliteadmin.config.php sed -i ':a;N;$!ba;s/\tarray(/lineWillBeDeleted/2' $DEPLOY_DIR/phpliteadmin/phpliteadmin.config.php
sed -i ':a;N;$!ba;s/\t),/lineWillBeDeleted/2' $DEPLOY_DIR/phpliteadmin/phpliteadmin.config.php sed -i ':a;N;$!ba;s/\t),/lineWillBeDeleted/2' $DEPLOY_DIR/phpliteadmin/phpliteadmin.config.php
sed -i "/lineWillBeDeleted/d" $DEPLOY_DIR/phpliteadmin/phpliteadmin.config.php sed -i "/lineWillBeDeleted/d" $DEPLOY_DIR/phpliteadmin/phpliteadmin.config.php
ln -sf $DEPLOY_DIR/phpliteadmin/phpliteadmin.php $DEPLOY_DIR/phpliteadmin/index.php
fi fi
echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
...@@ -823,5 +790,5 @@ touch /var/vagrant_provision ...@@ -823,5 +790,5 @@ touch /var/vagrant_provision
END=$(date +%s) END=$(date +%s)
TOTAL=$(( $END - $START )) TOTAL=$(( $END - $START ))
printf 'Time to prepare the environment: %dh:%dm:%ds\n' $(($TOTAL/3600)) $(($TOTAL%3600/60)) $(($TOTAL%60)) printf ' Time to prepare the environment: %dh:%dm:%ds\n' $(($TOTAL/3600)) $(($TOTAL%3600/60)) $(($TOTAL%60))
echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
...@@ -377,7 +377,7 @@ log_rotation_size = 0 # Automatic rotation of logfiles will ...@@ -377,7 +377,7 @@ log_rotation_size = 0 # Automatic rotation of logfiles will
# fatal # fatal
# panic (effectively off) # panic (effectively off)
log_min_duration_statement = 0 # -1 is disabled, 0 logs all statements log_min_duration_statement = -1 # -1 is disabled, 0 logs all statements
# and their durations, > 0 logs only # and their durations, > 0 logs only
# statements running at least this number # statements running at least this number
# of milliseconds # of milliseconds
...@@ -392,7 +392,7 @@ log_min_duration_statement = 0 # -1 is disabled, 0 logs all statements ...@@ -392,7 +392,7 @@ log_min_duration_statement = 0 # -1 is disabled, 0 logs all statements
#log_checkpoints = off #log_checkpoints = off
#log_connections = off #log_connections = off
#log_disconnections = off #log_disconnections = off
log_duration = on log_duration = off
#log_error_verbosity = default # terse, default, or verbose messages #log_error_verbosity = default # terse, default, or verbose messages
log_hostname = off log_hostname = off
log_line_prefix = '< %m >' # special values: log_line_prefix = '< %m >' # special values:
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment