From fa2c3f286b491fa58565c5935d590a7e3f832339 Mon Sep 17 00:00:00 2001
From: Nikolaus Krismer <nikolaus.krismer@uibk.ac.at>
Date: Sun, 6 Sep 2015 14:15:55 +0200
Subject: [PATCH] added possibility to export spatialite database from postgis

---
 CHANGELOG.md                    |  7 ++---
 bootstrap/config.sh             |  9 +++++++
 bootstrap/importData.sh         | 18 ++++++++++---
 bootstrap/prepareEnvironment.sh | 47 ++++++++++++++++++++++++++-------
 4 files changed, 65 insertions(+), 16 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 6637b24..a1e7f7e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,12 +1,10 @@
 Upcoming version:
 -----------------
+  - added possibility to export spatialite database from postgis (Nikolaus Krismer)
   - activating database logging in testcase now (not globally anymore) (Nikolaus Krismer)
   - updating gradle scmversion plugin (Nikolaus Krismer)
   - added contribution guidelines (Nikolaus Krismer)
   - moving gradle changelog tasks to separate buildscript (Nikolaus Krismer)
-
-Version v0.5.1:
----------------
   - fixing problems in changelog generation (again) (Nikolaus Krismer)
   - Merge branch 'master' of git@dbis-git.uibk.ac.at:krismer/isochrone-vm.git (Nikolaus Krismer)
   - calling grgit add before grgit commit (by now push is empty) (Nikolaus Krismer)
@@ -19,6 +17,9 @@ Version v0.5.1:
   - fixing problems with gradle's clean task (Nikolaus Krismer)
   - fixing invalid clean task definition (Nikolaus Krismer)
   - adding removal of CHANGELOG to clean task (Nikolaus Krismer)
+
+Version v0.5.1:
+---------------
   - removed hard-coded version from gradle (is set using scmversion plugin) (Nikolaus Krismer)
   - added density generation for synthethic networks (Nikolaus Krismer)
   - removed pgRouting / osm2pgrouting from files (not needed and never used) (Nikolaus Krismer)
diff --git a/bootstrap/config.sh b/bootstrap/config.sh
index 263ef89..99e1f3f 100755
--- a/bootstrap/config.sh
+++ b/bootstrap/config.sh
@@ -8,6 +8,7 @@ IMPORT_DATA_OSM_CACHED=false
 IMPORT_DATA_SQL=true
 IMPORT_SCHEDULES=true
 
+INSTALL_GDAL=false
 INSTALL_GEOSERVER=false
 INSTALL_HTTPD=true
 # install isochrone from vagrant start directory (eclipse project)
@@ -29,6 +30,13 @@ INSTALL_PROJECT_PSIPROBE=true
 INSTALL_SPATIALITE=false
 INSTALL_SPATIALITE_ADMIN=true
 
+UPDATE_EXPORTS=false
+if $UPDATE_EXPORTS; then
+	INSTALL_GDAL=true
+	INSTALL_SPATIALITE=true
+	INSTALL_ISOCHRONE_TOOLS=true
+fi
+
 CACHE_DIR_LOCAL="/tmp/vagrant-cache"
 CACHE_DIR_REMOTE="http://dbis-informatik.uibk.ac.at/static/ma/niko"
 CACHE_DIR_REMOTE_ISO="${CACHE_DIR_REMOTE}/isochrone"
@@ -57,6 +65,7 @@ if [ ! -d ${POSTGRES_SHARE} ]; then
 	POSTGRES_SHARE="$(dirname /usr/pgsql*/.)""/share"
 fi
 
+AVAILABLE_GDAL=$([ -d "${DEPLOY_DIR}/gdal" ] && echo true || echo false)
 AVAILABLE_GEOSERVER=$([ -d "${DEPLOY_DIR}/geoserver" ] && echo true || echo false)
 AVAILABLE_ISOCHRONE_DATAMODEL=$([ -f "${DEPLOY_DIR}/isochrone-datamodel.jar" ] && echo true || echo false)
 AVAILABLE_ISOCHRONE_TOOLS=$([ -f "${DEPLOY_DIR}/isochrone-tools.jar" ] && echo true || echo false)
diff --git a/bootstrap/importData.sh b/bootstrap/importData.sh
index 9197781..be45193 100755
--- a/bootstrap/importData.sh
+++ b/bootstrap/importData.sh
@@ -177,8 +177,8 @@ fn_import_data "San Francisco" "$OSM_FILE_CALIFORNIA" "$OSM_FILE_SF_FILTERED" "$
 if $AVAILABLE_POSTGIS && $AVAILABLE_ISOCHRONE_TOOLS; then
 	java -cp $DOWNLOAD_DIR/isochrone-tools.jar at.uibk.dbis.isochrone.generator.GridNetworkGenerator -d 100 -l 60 > "$WORKING_DIR/generate_gridNetwork.log" 2>&1
 	java -cp $DOWNLOAD_DIR/isochrone-tools.jar at.uibk.dbis.isochrone.generator.SpiderNetworkGenerator -d 6 -lvl 1000 -l 60 > "$WORKING_DIR/generate_spiderNetwork.log" 2>&1
-	java -cp $DOWNLOAD_DIR/isochrone-tools.jar at.uibk.dbis.isochrone.generator.density.DensityGenerator -t grid_s100 -d 60,120,180,240,300
-	java -cp $DOWNLOAD_DIR/isochrone-tools.jar at.uibk.dbis.isochrone.generator.density.DensityGenerator -t spider_l1000 -d 60,120,180,240,300
+	java -cp $DOWNLOAD_DIR/isochrone-tools.jar at.uibk.dbis.isochrone.generator.density.DensityGenerator -t grid_s100 -d 60,120,180,240,300 >> "$WORKING_DIR/generate_gridNetwork.log" 2>&1
+	java -cp $DOWNLOAD_DIR/isochrone-tools.jar at.uibk.dbis.isochrone.generator.density.DensityGenerator -t spider_l1000 -d 60,120,180,240,300 >> "$WORKING_DIR/generate_spiderNetwork.log" 2>&1
 fi
 
 if $AVAILABLE_GEOSERVER; then
@@ -221,7 +221,19 @@ if $AVAILABLE_GEOSERVER; then
 	fi
 fi
 
-if $AVAILABLE_SPATIALITE_ADMIN; then
+if $AVAILABLE_GDAL; then
+	if [ -f $DATA_DIR/$OSM_FILE_FILTERED ]; then
+		echo "Deleting outdated spatialite database"
+		rm -rf $DATA_DIR/isochrone.db
+	fi
+
+	echo "Exporting PostGIS data into spatialite database" 
+	$DEPLOY_DIR/gdal/apps/ogr2ogr --config PG_LIST_ALL_TABLES YES --config PG_SKIP_VIEWS YES -progress -f "SQLite" "$DATA_DIR/isochrone.db" PG:"host=localhost dbname=isochrone user=$PG_DB_USER password=$PG_DB_PASSWORD" -lco LAUNDER=yes -dsco SPATIALITE=yes -lco SPATIAL_INDEX=yes -gt 65536  >> setup_spatialite_neo4j.log 2>&1
+
+	echo "  - setting spatialite database permissions"
+	chown -R apache:apache "$DATA_DIR/isochrone.db"
+	chmod -R -g+rwX "$DATA_DIR/isochrone.db"
+elif $AVAILABLE_SPATIALITE_ADMIN; then
 	echo "Importing spatialite database"
 	fn_download_newer $DOWNLOAD_DIR/isochrone.db $CACHE_DIR_REMOTE_DATA/201505_spatialite.db
 	cp -f $DOWNLOAD_DIR/isochrone.db $DATA_DIR/
diff --git a/bootstrap/prepareEnvironment.sh b/bootstrap/prepareEnvironment.sh
index 0ca4a71..f20a851 100755
--- a/bootstrap/prepareEnvironment.sh
+++ b/bootstrap/prepareEnvironment.sh
@@ -52,6 +52,9 @@ fn_arg2boolean IS_LOCAL_TEST_DEPLOY "${2}" "${IS_LOCAL_TEST_DEPLOY}"
 #Armadillo from epel is too new for postgis (version 4 instead of 3), so use hard-coded file here
 ARMADILLO_FILE="armadillo-3.800.2-1.el6.x86_64.rpm"
 ARMADILLO_URL="$CACHE_DIR_REMOTE_ISO/$ARMADILLO_FILE"
+GDAL_VERSION="2.0.0"
+GDAL_FILE="gdal-$GDAL_VERSION.tar.gz"
+GDAL_DOWNLOAD_URL="http://download.osgeo.org/gdal/$GDAL_VERSION/$GDAL_FILE"
 GEOSERVER_VERSION="2.7.2"
 GEOSERVER_FILE="geoserver-$GEOSERVER_VERSION-war.zip"
 GEOSERVER_DOWNLOAD_URL="http://sourceforge.net/projects/geoserver/files/GeoServer/$GEOSERVER_VERSION/$GEOSERVER_FILE"
@@ -65,7 +68,7 @@ MAVEN_VERSION="3.2.5"
 MAVEN_FILE="apache-maven-$MAVEN_VERSION-bin.zip"
 MAVEN_DOWNLOAD_URL="http://tweedo.com/mirror/apache/maven/maven-3/$MAVEN_VERSION/binaries/$MAVEN_FILE"
 NEO4J_DB_USER="neo4j"
-NEO4J_VERSION="2.1.7"
+NEO4J_VERSION="2.2.5"
 NEO4J_FILE="neo4j-community-$NEO4J_VERSION-unix.tar.gz"
 NEO4J_DOWNLOAD_URL="http://dist.neo4j.org/$NEO4J_FILE"
 NEO4J_SERVICE_NAME="neo4j-service"
@@ -83,11 +86,11 @@ NEXUS_RELEASE_ISOCHRONE_WEB="$NEXUS_BASE_URL/service/local/artifact/maven/conten
 NEXUS_SNAPSHOT_ISOCHRONE_WEB="$NEXUS_BASE_URL/service/local/artifact/maven/content?r=$NEXUS_PUBLIC_REPO&g=at.uibk.dbis&a=$NEXUS_ARTIFACT_NAME_WEB&v=LATEST&p=war"
 NEXUS_SNAPSHOT_ISOCHRONE_TOOLS="$NEXUS_BASE_URL/service/local/artifact/maven/content?r=$NEXUS_PUBLIC_REPO&g=at.uibk.dbis&a=$NEXUS_ARTIFACT_NAME_TOOLS&v=LATEST&c=all&p=jar"
 NEXUS_SNAPSHOT_ISOCHRONE_DATAMODEL="$NEXUS_BASE_URL/service/local/artifact/maven/content?r=$NEXUS_PUBLIC_REPO&g=at.uibk.dbis&a=$NEXUS_ARTIFACT_NAME_DATAMODEL&v=LATEST&c=all&p=jar"
-OSMOSIS_VERSION="0.43.1"
+OSMOSIS_VERSION="0.44.1"
 OSMOSIS_FILE="osmosis-$OSMOSIS_VERSION.zip"
 OSMOSIS_DOWNLOAD_URL="http://bretth.dev.openstreetmap.org/osmosis-build/$OSMOSIS_FILE"
-PHPLITEADMIN_VERSION="v1-9-5"
-PHPLITEADMIN_URL="https://phpliteadmin.googlecode.com/files/phpliteAdmin_$PHPLITEADMIN_VERSION.zip"
+PHPLITEADMIN_VERSION="v1-9-6"
+PHPLITEADMIN_URL="https://bitbucket.org/phpliteadmin/public/downloads/phpLiteAdmin_$PHPLITEADMIN_VERSION.zip"
 POSTGIS_GIT_URL="https://github.com/postgis/postgis.git"
 POSTGIS_VERSION="auto"      # activate to install from repository
 #POSTGIS_VERSION="2.1.5"    # activate to install from source
@@ -105,14 +108,14 @@ REPO_RPMFORGE_32="auto"
 REPO_RPMFORGE_64="auto"
 REPO_RPMFORGE_CONFIG="/etc/yum.repos.d/rpmforge.repo"
 SELINUX_CONFIG="/etc/selinux/config"
-SPATIALITE_VERSION="4.2.0"
+SPATIALITE_VERSION="4.3.0"
 SPATIALITE_FILE="libspatialite-$SPATIALITE_VERSION.tar.gz"
-SPATIALITE_URL="http://www.gaia-gis.it/gaia-sins/$SPATIALITE_FILE"
+SPATIALITE_URL="http://www.gaia-gis.it/gaia-sins/libspatialite-sources/$SPATIALITE_FILE"
 TEMPLATE_PSIPROBE_VERSION="2.3.3"
 TEMPLATE_PSIPROBE_FILE="probe-$TEMPLATE_PSIPROBE_VERSION.zip"
 TEMPLATE_PSIPROBE_DOWNLOAD_URL="https://psi-probe.googlecode.com/files/$TEMPLATE_PSIPROBE_FILE"
 TEMPLATE_DIR_ISOCHRONE="/vagrant"
-TOMCAT_VERSION="8.0.23"
+TOMCAT_VERSION="8.0.26"
 TOMCAT_FILE="apache-tomcat-$TOMCAT_VERSION.tar.gz"
 TOMCAT_DOWNLOAD_URL="http://tweedo.com/mirror/apache/tomcat/tomcat-8/v$TOMCAT_VERSION/bin/$TOMCAT_FILE"
 TOMCAT_MEMORY_OPTS="-Xmx2048m -Xms512m"
@@ -181,6 +184,10 @@ if $IS_LOCAL_TEST_DEPLOY; then
 	INSTALL_ISOCHRONE_RELEASE=false
 	INSTALL_ISOCHRONE_SNAPSHOT=false
 fi
+if $INSTALL_GDAL; then
+	INSTALL_SPATIALITE=true
+	INSTALL_SPATIALITE_ADMIN=true
+fi
 if $INSTALL_SPATIALITE_ADMIN; then
 	INSTALL_HTTPD=true
 fi
@@ -321,6 +328,7 @@ if $INSTALL_POSTGIS; then
 fi
 if $INSTALL_SPATIALITE; then echo "  - spatialite $SPATIALITE_VERSION"; fi
 if $INSTALL_SPATIALITE_ADMIN; then echo "  - phpLiteAdmin $PHPLITEADMIN_VERSION"; fi
+if $INSTALL_GDAL; then echo "  - gdal $GDAL_VERSION"; fi
 if $INSTALL_TOMCAT; then echo "  - tomcat $TOMCAT_VERSION"; fi
 if $INSTALL_GEOSERVER; then echo "  - geoserver $GEOSERVER_VERSION"; fi
 if $INSTALL_POSTGRES_ADMIN; then echo "  - phpPgAdmin"; fi
@@ -350,6 +358,9 @@ if $INSTALL_POSTGRES; then
 	groupadd -g 26 postgres
 	useradd -u 26 -g 26 -c "PostgreSQL Server" postgres
 	yum -y install $POSTGRESQL_PKG_NAME $POSTGRESQL_PKG_NAME-contrib $POSTGRESQL_PKG_NAME-server >> $WORKING_DIR/install_postgresql.log 2>&1
+	if $INSTALL_GDAL; then
+		yum -y install $POSTGRESQL_PKG_NAME-devel
+	fi	
 
 	echo "  - exporting postgresql options"
 	echo "export PostgreSQL_ROOT=/usr/pgsql-$POSTGRESQL_VERSION" >> /etc/profile.d/postgresql.sh
@@ -572,7 +583,7 @@ fi
 
 if $INSTALL_SPATIALITE; then
 	echo "Installing spatialite dependencies"
-	yum -y install sqlite-devel zlib-devel proj-devel freexl-devel geos-devel libxml2-devel >> $WORKING_DIR/install_spatialite_deps.log 2>&1
+	yum -y install sqlite-devel zlib-devel proj-devel geos-devel libxml2-devel >> $WORKING_DIR/install_spatialite_deps.log 2>&1
 
 	echo "Installing spatialite $SPATIALITE_VERSION"
 	echo "  - downloading & unpacking source"
@@ -580,9 +591,9 @@ if $INSTALL_SPATIALITE; then
 	tar -xvf $DOWNLOAD_DIR/$SPATIALITE_FILE -C $DEPLOY_DIR >> $WORKING_DIR/install_spatialite.log 2>&1 
 	ln -sf $DEPLOY_DIR/libspatialite* $DEPLOY_DIR/libspatialite
 
-	echo "  - compiling from source"
+	echo "  - compiling libspatialite from source"
 	cd $DEPLOY_DIR/libspatialite
-	./configure >> $WORKING_DIR/install_spatialite.log 2>&1
+	./configure --disable-freexl >> $WORKING_DIR/install_spatialite.log 2>&1
 	make >> $WORKING_DIR/install_spatialite.log 2>&1
 	make install-strip >> $WORKING_DIR/install_spatialite.log 2>&1
 
@@ -602,6 +613,22 @@ if $INSTALL_SPATIALITE_ADMIN; then
 	chown -R apache:apache /var/www/html/phpliteadmin
 fi
 
+if $INSTALL_GDAL; then
+	echo "  - installing GDAL (which contains the ogr2ogr tool)"
+	cd $DEPLOY_DIR
+	fn_download_newer $DOWNLOAD_DIR/$GDAL_FILE $GDAL_DOWNLOAD_URL
+	tar -xvf $DOWNLOAD_DIR/$GDAL_FILE -C $DEPLOY_DIR >> $WORKING_DIR/install_gdal.log 2>&1
+	ln -sf $DEPLOY_DIR/gdal* $DEPLOY_DIR/gdal
+
+	echo "  - compiling GDAL $GDAL_VERSION from source"
+	cd $DEPLOY_DIR/gdal
+	./configure --with-spatialite >> $WORKING_DIR/install_gdal.log 2>&1
+	make >> $WORKING_DIR/install_gdal.log 2>&1
+	make install >> $WORKING_DIR/install_gdal.log 2>&1
+
+	cd $WORKING_DIR
+fi
+
 echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
 echo " Installations performed"
 echo " Preparing services ($(date +%H:%M:%S)):"
-- 
GitLab