diff --git a/CHANGELOG.md b/CHANGELOG.md index 6637b24a23a615e498236d96adeec39ec7457eaa..a1e7f7eda8e2a8499ce2087fb57654e94337fa2f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,12 +1,10 @@ Upcoming version: ----------------- + - added possibility to export spatialite database from postgis (Nikolaus Krismer) - activating database logging in testcase now (not globally anymore) (Nikolaus Krismer) - updating gradle scmversion plugin (Nikolaus Krismer) - added contribution guidelines (Nikolaus Krismer) - moving gradle changelog tasks to separate buildscript (Nikolaus Krismer) - -Version v0.5.1: ---------------- - fixing problems in changelog generation (again) (Nikolaus Krismer) - Merge branch 'master' of git@dbis-git.uibk.ac.at:krismer/isochrone-vm.git (Nikolaus Krismer) - calling grgit add before grgit commit (by now push is empty) (Nikolaus Krismer) @@ -19,6 +17,9 @@ Version v0.5.1: - fixing problems with gradle's clean task (Nikolaus Krismer) - fixing invalid clean task definition (Nikolaus Krismer) - adding removal of CHANGELOG to clean task (Nikolaus Krismer) + +Version v0.5.1: +--------------- - removed hard-coded version from gradle (is set using scmversion plugin) (Nikolaus Krismer) - added density generation for synthethic networks (Nikolaus Krismer) - removed pgRouting / osm2pgrouting from files (not needed and never used) (Nikolaus Krismer) diff --git a/bootstrap/config.sh b/bootstrap/config.sh index 263ef8928f595309260b8daaea0ed0a798010571..99e1f3fa48f4a7bbb6116d17f46cd3674246274f 100755 --- a/bootstrap/config.sh +++ b/bootstrap/config.sh @@ -8,6 +8,7 @@ IMPORT_DATA_OSM_CACHED=false IMPORT_DATA_SQL=true IMPORT_SCHEDULES=true +INSTALL_GDAL=false INSTALL_GEOSERVER=false INSTALL_HTTPD=true # install isochrone from vagrant start directory (eclipse project) @@ -29,6 +30,13 @@ INSTALL_PROJECT_PSIPROBE=true INSTALL_SPATIALITE=false INSTALL_SPATIALITE_ADMIN=true +UPDATE_EXPORTS=false +if $UPDATE_EXPORTS; then + INSTALL_GDAL=true + INSTALL_SPATIALITE=true + INSTALL_ISOCHRONE_TOOLS=true +fi + CACHE_DIR_LOCAL="/tmp/vagrant-cache" CACHE_DIR_REMOTE="http://dbis-informatik.uibk.ac.at/static/ma/niko" CACHE_DIR_REMOTE_ISO="${CACHE_DIR_REMOTE}/isochrone" @@ -57,6 +65,7 @@ if [ ! -d ${POSTGRES_SHARE} ]; then POSTGRES_SHARE="$(dirname /usr/pgsql*/.)""/share" fi +AVAILABLE_GDAL=$([ -d "${DEPLOY_DIR}/gdal" ] && echo true || echo false) AVAILABLE_GEOSERVER=$([ -d "${DEPLOY_DIR}/geoserver" ] && echo true || echo false) AVAILABLE_ISOCHRONE_DATAMODEL=$([ -f "${DEPLOY_DIR}/isochrone-datamodel.jar" ] && echo true || echo false) AVAILABLE_ISOCHRONE_TOOLS=$([ -f "${DEPLOY_DIR}/isochrone-tools.jar" ] && echo true || echo false) diff --git a/bootstrap/importData.sh b/bootstrap/importData.sh index 91977810ba8baa99b87a837e42afabfc52b29b91..be45193002512b8e44a4614246e839a38adcddc0 100755 --- a/bootstrap/importData.sh +++ b/bootstrap/importData.sh @@ -177,8 +177,8 @@ fn_import_data "San Francisco" "$OSM_FILE_CALIFORNIA" "$OSM_FILE_SF_FILTERED" "$ if $AVAILABLE_POSTGIS && $AVAILABLE_ISOCHRONE_TOOLS; then java -cp $DOWNLOAD_DIR/isochrone-tools.jar at.uibk.dbis.isochrone.generator.GridNetworkGenerator -d 100 -l 60 > "$WORKING_DIR/generate_gridNetwork.log" 2>&1 java -cp $DOWNLOAD_DIR/isochrone-tools.jar at.uibk.dbis.isochrone.generator.SpiderNetworkGenerator -d 6 -lvl 1000 -l 60 > "$WORKING_DIR/generate_spiderNetwork.log" 2>&1 - java -cp $DOWNLOAD_DIR/isochrone-tools.jar at.uibk.dbis.isochrone.generator.density.DensityGenerator -t grid_s100 -d 60,120,180,240,300 - java -cp $DOWNLOAD_DIR/isochrone-tools.jar at.uibk.dbis.isochrone.generator.density.DensityGenerator -t spider_l1000 -d 60,120,180,240,300 + java -cp $DOWNLOAD_DIR/isochrone-tools.jar at.uibk.dbis.isochrone.generator.density.DensityGenerator -t grid_s100 -d 60,120,180,240,300 >> "$WORKING_DIR/generate_gridNetwork.log" 2>&1 + java -cp $DOWNLOAD_DIR/isochrone-tools.jar at.uibk.dbis.isochrone.generator.density.DensityGenerator -t spider_l1000 -d 60,120,180,240,300 >> "$WORKING_DIR/generate_spiderNetwork.log" 2>&1 fi if $AVAILABLE_GEOSERVER; then @@ -221,7 +221,19 @@ if $AVAILABLE_GEOSERVER; then fi fi -if $AVAILABLE_SPATIALITE_ADMIN; then +if $AVAILABLE_GDAL; then + if [ -f $DATA_DIR/$OSM_FILE_FILTERED ]; then + echo "Deleting outdated spatialite database" + rm -rf $DATA_DIR/isochrone.db + fi + + echo "Exporting PostGIS data into spatialite database" + $DEPLOY_DIR/gdal/apps/ogr2ogr --config PG_LIST_ALL_TABLES YES --config PG_SKIP_VIEWS YES -progress -f "SQLite" "$DATA_DIR/isochrone.db" PG:"host=localhost dbname=isochrone user=$PG_DB_USER password=$PG_DB_PASSWORD" -lco LAUNDER=yes -dsco SPATIALITE=yes -lco SPATIAL_INDEX=yes -gt 65536 >> setup_spatialite_neo4j.log 2>&1 + + echo " - setting spatialite database permissions" + chown -R apache:apache "$DATA_DIR/isochrone.db" + chmod -R -g+rwX "$DATA_DIR/isochrone.db" +elif $AVAILABLE_SPATIALITE_ADMIN; then echo "Importing spatialite database" fn_download_newer $DOWNLOAD_DIR/isochrone.db $CACHE_DIR_REMOTE_DATA/201505_spatialite.db cp -f $DOWNLOAD_DIR/isochrone.db $DATA_DIR/ diff --git a/bootstrap/prepareEnvironment.sh b/bootstrap/prepareEnvironment.sh index 0ca4a71381ec082b8180d4646893459c257583d0..f20a85114692cc6ce587263fd260dee6ca0e81e7 100755 --- a/bootstrap/prepareEnvironment.sh +++ b/bootstrap/prepareEnvironment.sh @@ -52,6 +52,9 @@ fn_arg2boolean IS_LOCAL_TEST_DEPLOY "${2}" "${IS_LOCAL_TEST_DEPLOY}" #Armadillo from epel is too new for postgis (version 4 instead of 3), so use hard-coded file here ARMADILLO_FILE="armadillo-3.800.2-1.el6.x86_64.rpm" ARMADILLO_URL="$CACHE_DIR_REMOTE_ISO/$ARMADILLO_FILE" +GDAL_VERSION="2.0.0" +GDAL_FILE="gdal-$GDAL_VERSION.tar.gz" +GDAL_DOWNLOAD_URL="http://download.osgeo.org/gdal/$GDAL_VERSION/$GDAL_FILE" GEOSERVER_VERSION="2.7.2" GEOSERVER_FILE="geoserver-$GEOSERVER_VERSION-war.zip" GEOSERVER_DOWNLOAD_URL="http://sourceforge.net/projects/geoserver/files/GeoServer/$GEOSERVER_VERSION/$GEOSERVER_FILE" @@ -65,7 +68,7 @@ MAVEN_VERSION="3.2.5" MAVEN_FILE="apache-maven-$MAVEN_VERSION-bin.zip" MAVEN_DOWNLOAD_URL="http://tweedo.com/mirror/apache/maven/maven-3/$MAVEN_VERSION/binaries/$MAVEN_FILE" NEO4J_DB_USER="neo4j" -NEO4J_VERSION="2.1.7" +NEO4J_VERSION="2.2.5" NEO4J_FILE="neo4j-community-$NEO4J_VERSION-unix.tar.gz" NEO4J_DOWNLOAD_URL="http://dist.neo4j.org/$NEO4J_FILE" NEO4J_SERVICE_NAME="neo4j-service" @@ -83,11 +86,11 @@ NEXUS_RELEASE_ISOCHRONE_WEB="$NEXUS_BASE_URL/service/local/artifact/maven/conten NEXUS_SNAPSHOT_ISOCHRONE_WEB="$NEXUS_BASE_URL/service/local/artifact/maven/content?r=$NEXUS_PUBLIC_REPO&g=at.uibk.dbis&a=$NEXUS_ARTIFACT_NAME_WEB&v=LATEST&p=war" NEXUS_SNAPSHOT_ISOCHRONE_TOOLS="$NEXUS_BASE_URL/service/local/artifact/maven/content?r=$NEXUS_PUBLIC_REPO&g=at.uibk.dbis&a=$NEXUS_ARTIFACT_NAME_TOOLS&v=LATEST&c=all&p=jar" NEXUS_SNAPSHOT_ISOCHRONE_DATAMODEL="$NEXUS_BASE_URL/service/local/artifact/maven/content?r=$NEXUS_PUBLIC_REPO&g=at.uibk.dbis&a=$NEXUS_ARTIFACT_NAME_DATAMODEL&v=LATEST&c=all&p=jar" -OSMOSIS_VERSION="0.43.1" +OSMOSIS_VERSION="0.44.1" OSMOSIS_FILE="osmosis-$OSMOSIS_VERSION.zip" OSMOSIS_DOWNLOAD_URL="http://bretth.dev.openstreetmap.org/osmosis-build/$OSMOSIS_FILE" -PHPLITEADMIN_VERSION="v1-9-5" -PHPLITEADMIN_URL="https://phpliteadmin.googlecode.com/files/phpliteAdmin_$PHPLITEADMIN_VERSION.zip" +PHPLITEADMIN_VERSION="v1-9-6" +PHPLITEADMIN_URL="https://bitbucket.org/phpliteadmin/public/downloads/phpLiteAdmin_$PHPLITEADMIN_VERSION.zip" POSTGIS_GIT_URL="https://github.com/postgis/postgis.git" POSTGIS_VERSION="auto" # activate to install from repository #POSTGIS_VERSION="2.1.5" # activate to install from source @@ -105,14 +108,14 @@ REPO_RPMFORGE_32="auto" REPO_RPMFORGE_64="auto" REPO_RPMFORGE_CONFIG="/etc/yum.repos.d/rpmforge.repo" SELINUX_CONFIG="/etc/selinux/config" -SPATIALITE_VERSION="4.2.0" +SPATIALITE_VERSION="4.3.0" SPATIALITE_FILE="libspatialite-$SPATIALITE_VERSION.tar.gz" -SPATIALITE_URL="http://www.gaia-gis.it/gaia-sins/$SPATIALITE_FILE" +SPATIALITE_URL="http://www.gaia-gis.it/gaia-sins/libspatialite-sources/$SPATIALITE_FILE" TEMPLATE_PSIPROBE_VERSION="2.3.3" TEMPLATE_PSIPROBE_FILE="probe-$TEMPLATE_PSIPROBE_VERSION.zip" TEMPLATE_PSIPROBE_DOWNLOAD_URL="https://psi-probe.googlecode.com/files/$TEMPLATE_PSIPROBE_FILE" TEMPLATE_DIR_ISOCHRONE="/vagrant" -TOMCAT_VERSION="8.0.23" +TOMCAT_VERSION="8.0.26" TOMCAT_FILE="apache-tomcat-$TOMCAT_VERSION.tar.gz" TOMCAT_DOWNLOAD_URL="http://tweedo.com/mirror/apache/tomcat/tomcat-8/v$TOMCAT_VERSION/bin/$TOMCAT_FILE" TOMCAT_MEMORY_OPTS="-Xmx2048m -Xms512m" @@ -181,6 +184,10 @@ if $IS_LOCAL_TEST_DEPLOY; then INSTALL_ISOCHRONE_RELEASE=false INSTALL_ISOCHRONE_SNAPSHOT=false fi +if $INSTALL_GDAL; then + INSTALL_SPATIALITE=true + INSTALL_SPATIALITE_ADMIN=true +fi if $INSTALL_SPATIALITE_ADMIN; then INSTALL_HTTPD=true fi @@ -321,6 +328,7 @@ if $INSTALL_POSTGIS; then fi if $INSTALL_SPATIALITE; then echo " - spatialite $SPATIALITE_VERSION"; fi if $INSTALL_SPATIALITE_ADMIN; then echo " - phpLiteAdmin $PHPLITEADMIN_VERSION"; fi +if $INSTALL_GDAL; then echo " - gdal $GDAL_VERSION"; fi if $INSTALL_TOMCAT; then echo " - tomcat $TOMCAT_VERSION"; fi if $INSTALL_GEOSERVER; then echo " - geoserver $GEOSERVER_VERSION"; fi if $INSTALL_POSTGRES_ADMIN; then echo " - phpPgAdmin"; fi @@ -350,6 +358,9 @@ if $INSTALL_POSTGRES; then groupadd -g 26 postgres useradd -u 26 -g 26 -c "PostgreSQL Server" postgres yum -y install $POSTGRESQL_PKG_NAME $POSTGRESQL_PKG_NAME-contrib $POSTGRESQL_PKG_NAME-server >> $WORKING_DIR/install_postgresql.log 2>&1 + if $INSTALL_GDAL; then + yum -y install $POSTGRESQL_PKG_NAME-devel + fi echo " - exporting postgresql options" echo "export PostgreSQL_ROOT=/usr/pgsql-$POSTGRESQL_VERSION" >> /etc/profile.d/postgresql.sh @@ -572,7 +583,7 @@ fi if $INSTALL_SPATIALITE; then echo "Installing spatialite dependencies" - yum -y install sqlite-devel zlib-devel proj-devel freexl-devel geos-devel libxml2-devel >> $WORKING_DIR/install_spatialite_deps.log 2>&1 + yum -y install sqlite-devel zlib-devel proj-devel geos-devel libxml2-devel >> $WORKING_DIR/install_spatialite_deps.log 2>&1 echo "Installing spatialite $SPATIALITE_VERSION" echo " - downloading & unpacking source" @@ -580,9 +591,9 @@ if $INSTALL_SPATIALITE; then tar -xvf $DOWNLOAD_DIR/$SPATIALITE_FILE -C $DEPLOY_DIR >> $WORKING_DIR/install_spatialite.log 2>&1 ln -sf $DEPLOY_DIR/libspatialite* $DEPLOY_DIR/libspatialite - echo " - compiling from source" + echo " - compiling libspatialite from source" cd $DEPLOY_DIR/libspatialite - ./configure >> $WORKING_DIR/install_spatialite.log 2>&1 + ./configure --disable-freexl >> $WORKING_DIR/install_spatialite.log 2>&1 make >> $WORKING_DIR/install_spatialite.log 2>&1 make install-strip >> $WORKING_DIR/install_spatialite.log 2>&1 @@ -602,6 +613,22 @@ if $INSTALL_SPATIALITE_ADMIN; then chown -R apache:apache /var/www/html/phpliteadmin fi +if $INSTALL_GDAL; then + echo " - installing GDAL (which contains the ogr2ogr tool)" + cd $DEPLOY_DIR + fn_download_newer $DOWNLOAD_DIR/$GDAL_FILE $GDAL_DOWNLOAD_URL + tar -xvf $DOWNLOAD_DIR/$GDAL_FILE -C $DEPLOY_DIR >> $WORKING_DIR/install_gdal.log 2>&1 + ln -sf $DEPLOY_DIR/gdal* $DEPLOY_DIR/gdal + + echo " - compiling GDAL $GDAL_VERSION from source" + cd $DEPLOY_DIR/gdal + ./configure --with-spatialite >> $WORKING_DIR/install_gdal.log 2>&1 + make >> $WORKING_DIR/install_gdal.log 2>&1 + make install >> $WORKING_DIR/install_gdal.log 2>&1 + + cd $WORKING_DIR +fi + echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" echo " Installations performed" echo " Preparing services ($(date +%H:%M:%S)):"