From 0ad16d961d23a2126cfb605e4ce2c3cc52819ecd Mon Sep 17 00:00:00 2001
From: Nikolaus Krismer <nikolaus.krismer@uibk.ac.at>
Date: Thu, 28 Aug 2014 15:17:58 +0200
Subject: [PATCH] deactivating pgrouting in local test deploy mode (by default)
fixing performance problems
---
Vagrantfile | 8 +++--
etc/vagrant/Vagrantfile | 8 +++--
etc/vagrant/bootstrap.sh | 75 +++++++++++++++++++++-------------------
3 files changed, 50 insertions(+), 41 deletions(-)
diff --git a/Vagrantfile b/Vagrantfile
index cfa1f311..4525a4c4 100644
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -18,10 +18,14 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
config.vm.boot_timeout = 600
# Hardware configuration
+ # WARNING: Be VERY careful with this settings!!!
+ # -> giving the client more cores can SLOW DOWN the VM (if there is not enough workload on the guest, two less cores on the host...)
config.vm.provider "virtualbox" do |v|
v.name = (isLocalMode ? "isochrone-local" : "isochrone")
- v.cpus = 2
- v.memory = 3072
+ v.cpus = 1
+ v.memory = 2048
+ v.customize ["modifyvm", :id, "--cpuexecutioncap", "100"]
+ v.customize ["modifyvm", :id, "--ostype", "RedHat_64"]
end
# Enable vagrant-cachier plugin
diff --git a/etc/vagrant/Vagrantfile b/etc/vagrant/Vagrantfile
index f39fb806..6123b1bb 100644
--- a/etc/vagrant/Vagrantfile
+++ b/etc/vagrant/Vagrantfile
@@ -6,12 +6,14 @@ VAGRANTFILE_API_VERSION = "2"
Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
# Hardware configuration
+ # WARNING: Be VERY careful with this settings!!!
+ # -> giving the client more cores can SLOW DOWN the VM (if there is not enough workload on the guest, two less cores on the host...)
config.vm.provider "virtualbox" do |v|
v.name = "isochrone"
- v.cpus = 2
- v.memory = 3072
+ v.cpus = 1
+ v.memory = 2048
v.customize ["modifyvm", :id, "--cpuexecutioncap", "100"]
- v.customize ["modifyvm", :id, " –ostype" "RedHat_64"]
+ v.customize ["modifyvm", :id, "--ostype", "RedHat_64"]
end
# Enable vagrant-cachier plugin
diff --git a/etc/vagrant/bootstrap.sh b/etc/vagrant/bootstrap.sh
index f4d5c3f1..8e668e8e 100644
--- a/etc/vagrant/bootstrap.sh
+++ b/etc/vagrant/bootstrap.sh
@@ -34,8 +34,9 @@ fi
# Variable definitions #
############################
-IMPORT_OSMDATA=false
-IMPORT_OSMDATA_CACHED=true
+IMPORT_DATA_OSM=false
+IMPORT_DATA_OSM_CACHED=false
+IMPORT_DATA_SQL=true
IMPORT_SCHEDULES=true
INSTALL_GEOSERVER=false
INSTALL_HTTPD=true
@@ -44,7 +45,7 @@ INSTALL_ISOCHRONE_RELEASE=true # installs latest isochrone release from nexus re
INSTALL_ISOCHRONE_SNAPSHOT=true # installs latest isochrone snapshot from nexus repository
INSTALL_NEO4J=false
INSTALL_NEO4J_SPATIAL=false
-INSTALL_PGROUTING=true
+INSTALL_PGROUTING=false
INSTALL_POSTGIS=true
INSTALL_POSTGRES=true
INSTALL_POSTGRES_ADMIN=true
@@ -238,32 +239,29 @@ fi
# Calculated configuration #
############################
+IMPORT_DATA=false
INSTALL_JAVA=false
INSTALL_MAVEN=false
-INSTALL_OSMOSIS=false # should be set to true to automatically resolve issues with cached CACHED_OSM_DATA
INSTALL_TOMCAT=false
if $IS_LOCAL_TEST_DEPLOY; then
# Some projects will never be installed when in local deploy mode (like latest release/snapshot from nexus)
INSTALL_PROJECT_CANIGET2=false
-# INSTALL_PROJECT_PSIPROBE=false # for some reason uncommenting this line causes performance issues (is this due to VM dynamic hdd size)
+ INSTALL_PROJECT_PSIPROBE=false
INSTALL_ISOCHRONE_LOCAL=false
INSTALL_ISOCHRONE_RELEASE=false
INSTALL_ISOCHRONE_SNAPSHOT=false
-else
- # Some projects will only be installed when in local deploy mode (like pgRouting that can be used for performance comparisons)
- INSTALL_PGROUTING=false
fi
-if $IMPORT_OSMDATA; then
- INSTALL_OSMOSIS=true
+if $IMPORT_DATA_OSM || $IMPORT_DATA_OSM_CACHED || $IMPORT_DATA_SQL; then
+ IMPORT_DATA=true
fi
if $INSTALL_GEOSERVER || $INSTALL_PROJECT_CANIGET2 || $INSTALL_ISOCHRONE_LOCAL || $INSTALL_ISOCHRONE_RELEASE || $INSTALL_ISOCHRONE_SNAPSHOT || $INSTALL_PROJECT_PSIPROBE; then
INSTALL_TOMCAT=true
fi
if $INSTALL_NEO4J_SPATIAL; then
- INSTALL_MAVEN=$IMPORT_OSMDATA || $IMPORT_OSMDATA_CACHED
+ INSTALL_MAVEN=$IMPORT_DATA_OSM || $IMPORT_DATA_OSM_CACHED
fi
-if $INSTALL_TOMCAT || $INSTALL_GEOSERVER || $INSTALL_NEO4J || $INSTALL_NEO4J_SPATIAL || $IMPORT_OSMDATA; then
+if $INSTALL_TOMCAT || $INSTALL_GEOSERVER || $INSTALL_NEO4J || $INSTALL_NEO4J_SPATIAL || $IMPORT_DATA_OSM; then
INSTALL_JAVA=true
fi
if $INSTALL_PGROUTING; then
@@ -328,7 +326,7 @@ fn_import_data()
local TABLE_PREFIX="$6"
local VALID_SQL_EXPORT=false
- if $INSTALL_POSTGIS && $IMPORT_SCHEDULES; then
+ if $IMPORT_DATA_SQL && $INSTALL_POSTGIS && $IMPORT_SCHEDULES; then
if [ -n "$SQL_EXPORT_FILE" ]; then
VALID_SQL_EXPORT=true
fi
@@ -340,14 +338,34 @@ fn_import_data()
return 1
fi
- if $IMPORT_OSMDATA_CACHED; then
+ if $VALID_SQL_EXPORT; then
+ echo " - downloading sql export"
+ fn_download_newer $DOWNLOAD_DIR/$SQL_EXPORT_FILE $CACHE_DIR_REMOTE/$SQL_EXPORT_FILE
+ cp $DOWNLOAD_DIR/$SQL_EXPORT_FILE $DEPLOY_DIR/$SQL_EXPORT_FILE >> /dev/null 2>&1
+
+ echo " - importing sql export into postgis database"
+ gunzip -c $DEPLOY_DIR/$SQL_EXPORT_FILE | PGPASSWORD="$PG_DB_PASSWORD" psql -U "$PG_DB_USER" -h localhost "$PG_DB_NAME" >> "$WORKING_DIR/import_schedule_$NAME.log" 2>&1
+
+ echo " - fixing table permissions"
+ tables=`PGPASSWORD="$PG_DB_PASSWORD" psql -qAt -U "$PG_DB_USER" -h localhost -c "SELECT tablename FROM pg_tables WHERE schemaname = 'public' AND tableowner = 'postgres';" "$PG_DB_NAME"`
+ for tbl in $tables ; do
+ PGPASSWORD="$PG_DB_PASSWORD" psql -qAt -U "$PG_DB_USER" -h localhost -c "ALTER TABLE $tbl OWNER TO $PG_DB_USER" "$PG_DB_NAME";
+ done
+ fi
+
+ if ! ( $INSTALL_PGROUTING || $INSTALL_NEO4J_SPATIAL ) || ! ( $IMPORT_DATA_OSM_CACHED || $IMPORT_DATA_OSM ); then
+ # osm data does not need to be downloaded (since it would not be imported afterwards)
+ return 0
+ fi
+
+ if $IMPORT_DATA_OSM_CACHED; then
echo " - downloading cached OpenStreetMap file"
fn_download_newer $DOWNLOAD_DIR/$OSM_FILE_FILTERED $CACHE_DIR_REMOTE/$OSM_FILE_FILTERED
cp $DOWNLOAD_DIR/$OSM_FILE_FILTERED $DEPLOY_DIR/$OSM_FILE_FILTERED >> /dev/null 2>&1
fi
if [ ! -f $DEPLOY_DIR/$OSM_FILE_FILTERED ]; then
- if $INSTALL_OSMOSIS; then
+ if $IMPORT_DATA_OSM; then
echo " - downloading OpenStreetMap file (no cached file found)"
OSM_FILENAME=$DOWNLOAD_DIR/${OSM_FILE##*/}
fn_download_newer $OSM_FILENAME $OSM_DOWNLOAD_MIRROR/$OSM_FILE
@@ -363,21 +381,6 @@ fn_import_data()
fi
fi
- if $VALID_SQL_EXPORT; then
- echo " - downloading sql export"
- fn_download_newer $DOWNLOAD_DIR/$SQL_EXPORT_FILE $CACHE_DIR_REMOTE/$SQL_EXPORT_FILE
- cp $DOWNLOAD_DIR/$SQL_EXPORT_FILE $DEPLOY_DIR/$SQL_EXPORT_FILE >> /dev/null 2>&1
-
- echo " - importing sql export into postgis database"
- gunzip -c $DEPLOY_DIR/$SQL_EXPORT_FILE | PGPASSWORD="$PG_DB_PASSWORD" psql -U "$PG_DB_USER" -h localhost "$PG_DB_NAME" >> "$WORKING_DIR/import_schedule_$NAME.log" 2>&1
-
- echo " - fixing table permissions"
- tables=`PGPASSWORD="$PG_DB_PASSWORD" psql -qAt -U "$PG_DB_USER" -h localhost -c "SELECT tablename FROM pg_tables WHERE schemaname = 'public' AND tableowner = 'postgres';" "$PG_DB_NAME"`
- for tbl in $tables ; do
- PGPASSWORD="$PG_DB_PASSWORD" psql -qAt -U "$PG_DB_USER" -h localhost -c "ALTER TABLE $tbl OWNER TO $PG_DB_USER" "$PG_DB_NAME";
- done
- fi
-
if $INSTALL_PGROUTING; then
echo " - importing filtered OpenStreetMap data into pgRouting"
$DEPLOY_DIR/osm2pgrouting/build/osm2pgrouting -file "$DEPLOY_DIR/$OSM_FILE_FILTERED" -conf "$OSM2PGROUTING_MAPPING_CONF" -dbname $PG_DB_NAME -user $PG_DB_USER -passwd $PG_DB_PASSWORD -clean -prefixtables $TABLE_PREFIX >> "import_osm2pgrouting_$NAME.log" 2>&1
@@ -544,8 +547,8 @@ fi
if $INSTALL_TOMCAT; then echo " - tomcat $TOMCAT_VERSION"; fi
if $INSTALL_GEOSERVER; then echo " - geoserver $GEOSERVER_VERSION"; fi
if $INSTALL_POSTGRES_ADMIN; then echo " - phpPgAdmin"; fi
-if $IMPORT_OSMDATA; then echo " - osmosis $OSMOSIS_VERSION"; fi
-if $IMPORT_OSMDATA_CACHED; then echo "Importing cached data from OSM (if possible)"; elif $IMPORT_OSMDATA; then echo "Importing data from OSM"; fi
+if $IMPORT_DATA_OSM; then echo " - osmosis $OSMOSIS_VERSION"; fi
+if $IMPORT_DATA_OSM_CACHED; then echo "Importing cached data from OSM (if possible)"; elif $IMPORT_DATA_OSM; then echo "Importing data from OSM"; fi
if $IMPORT_SCHEDULES; then echo "Importing schedules"; fi
if $INSTALL_PROJECT_CANIGET2; then echo "Importing project \"CanIGet2\""; fi
if $INSTALL_ISOCHRONE_LOCAL; then echo "Importing project \"isochrone (local testing)\""; fi
@@ -789,7 +792,7 @@ if $INSTALL_NEO4J_SPATIAL; then
git checkout -q $NEO4J_SPATIAL_VERSION >> /dev/null 2>&1
if $INSTALL_GEOSERVER; then
- if $IMPORT_OSMDATA || $IMPORT_OSMDATA_CACHED; then
+ if $IMPORT_DATA_OSM || $IMPORT_DATA_OSM_CACHED; then
# We need to compile the source when importing data, so that the main class gets compiled
echo " - copying neo4j-spatial into geoserver (from source)"
mvn clean install -DskipTests >> $WORKING_DIR/install_neo4j_spatial.log 2>&1
@@ -803,7 +806,7 @@ if $INSTALL_NEO4J_SPATIAL; then
cd $WORKING_DIR
fi
-if $INSTALL_OSMOSIS; then
+if $IMPORT_DATA_OSM; then
echo "Installing osmosis $OSMOSIS_VERSION"
fn_download_newer $DOWNLOAD_DIR/$OSMOSIS_FILE "$OSMOSIS_DOWNLOAD_URL"
mkdir -p $DEPLOY_DIR/osmosis-$OSMOSIS_VERSION
@@ -812,7 +815,7 @@ if $INSTALL_OSMOSIS; then
chmod a+x $DEPLOY_DIR/osmosis/bin/osmosis
fi
-if ( $IMPORT_OSMDATA || $IMPORT_OSMDATA_CACHED ) && $INSTALL_PGROUTING; then
+if ( $IMPORT_DATA_OSM || $IMPORT_DATA_OSM_CACHED ) && $INSTALL_PGROUTING; then
echo "Installing osm2pgrouting dependencies"
yum install -y expat-devel boost-devel >> install_osm2pgrouting_deps.log 2>&1
@@ -944,7 +947,7 @@ fi
echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
echo " Services started!"
-if $IMPORT_OSMDATA || $IMPORT_OSMDATA_CACHED; then
+if $IMPORT_DATA; then
echo " Importing data:"
echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
--
GitLab