re-activated Neo4J-spatial (with help from Robert Bierbauer's and his master thesis)

CHANGELOG.md

 Upcoming version:
 -----------------
+  - re-activated Neo4J-spatial (with help from Robert Bierbauer's and his master thesis) (Nikolaus Krismer)
+  - now creating node_densities for updated datasets (Nikolaus Krismer)
   - updateing gradle to version 2.7 (Nikolaus Krismer)
   - updateing readme (tag location) (Nikolaus Krismer)
   - #2 fixed (Nikolaus Krismer)

Vagrantfile

@@ -71,6 +71,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
 
   # Network configuration
   config.vm.network :forwarded_port, guest: 5432, host: 5432, id: "postgresql", auto_correct: false
+  config.vm.network :forwarded_port, guest: 7473, host: 7473, id: "neo4j-ssl", auto_correct: false
   config.vm.network :forwarded_port, guest: 7474, host: 7474, id: "neo4j", auto_correct: false
   config.vm.network :forwarded_port, guest: 80,   host: 8000, id: "httpd", auto_correct: false
   unless isLocalMode

bootstrap/config.sh

@@ -3,7 +3,7 @@
 # This is used to configure the behavior of the various bootstrap scripts.
 #############################
 
-IMPORT_DATA_NEO4J_SPATIAL=false
+IMPORT_DATA_NEO4J_SPATIAL=true
 IMPORT_DATA_POSTGIS=true
 IMPORT_DATA_SPATIALITE=true
 

bootstrap/importData.sh

@@ -69,61 +69,25 @@ function fn_import_dataset() {
 	local BOUNDING="$4"
 	local SQL_EXPORT_FILE="${NAME,,}_export.sql.gz"
 
-	local VALID_SQL=false
-	if $IMPORT_DATA_POSTGIS && $AVAILABLE_POSTGIS; then
-		if [ -n "$SQL_EXPORT_FILE" ]; then
-			VALID_SQL=true
-		fi
-	fi
-
 	echo "Importing data for region of $NAME"
-	if ! ( $VALID_SQL || $IMPORT_DATA_NEO4J_SPATIAL ); then
-		echo "  - skipping import"
-		return 1
-	fi
+	echo "  - importing into postgis"
+	fn_import_dataset_postgis "$NAME" "$SQL_EXPORT_FILE"
 
-	if $VALID_SQL; then
-		fn_import_dataset_postgis "$NAME" "$SQL_EXPORT_FILE"
-	fi
-
-	if $IMPORT_DATA_NEO4J_SPATIAL; then
-		fn_import_dataset_neo4j_spatial "$NAME" "$OSM_FILE" "$OSM_FILE_FILTERED" "$BOUNDING"
+	if $IMPORT_DATA_SPATIALITE; then
+		echo "  - importing into neo4j"
+		fn_import_dataset_neo4j_spatial "$NAME"
 	fi
 }
 
 function fn_import_dataset_neo4j_spatial() {
 	local NAME="$1"
-	local OSM_FILE="$2"
-	local OSM_FILE_FILTERED="$3"
-	local BOUNDING="$4"
+	local SQL_EXPORT_FILE="$2"
 
 	local CITY=${NAME// /}
 	CITY=${CITY,,}
 
-	if !$UPDATE_DATA; then
-		echo "  - downloading cached OpenStreetMap file"
-		fn_download_newer $DOWNLOAD_DIR/$OSM_FILE_FILTERED $CACHE_DIR_REMOTE_DATA/$OSM_FILE_FILTERED
-		cp $DOWNLOAD_DIR/$OSM_FILE_FILTERED $DATA_DIR/$OSM_FILE_FILTERED >> /dev/null 2>&1
-	fi
-
-	if $UPDATE_DATA && [ -f "$DATA_DIR/$OSM_FILE_FILTERED" ]; then
-		echo "  - deleting outdated OpenStreetMap file"
-		rm -rf "$DATA_DIR/$OSM_FILE_FILTERED"
-	fi
-
-	if $AVAILABLE_NEO4J_SPATIAL && [ ! -f "$DATA_DIR/$OSM_FILE_FILTERED" ]; then
-		echo "  - downloading OpenStreetMap file (no cached file found)"
-		OSM_FILENAME=$DOWNLOAD_DIR/${OSM_FILE##*/}
-		fn_download_newer $OSM_FILENAME $OSM_DOWNLOAD_MIRROR/$OSM_FILE
-
-		echo "  - filtering OpenStreetMap data"
-		osmosis --read-pbf file="$OSM_FILENAME" --bounding-polygon file="$BOUNDING" --way-key keyList="$OSM_KEYS" --used-node --write-xml file="$DATA_DIR/$OSM_FILE_FILTERED" >> "$WORKING_DIR/create_neo4jData_$CITY.log" 2>&1
-
-		echo "  - importing filtered OpenStreetMap data into neo4j-spatial"
-		cd $DEPLOY_DIR/neo4j-spatial
-		mvn exec:java -Dexec.mainClass=org.neo4j.gis.spatial.osm.OSMImporter -Dexec.args="$DEPLOY_DIR/neo4j/data/graph.db $DATA_DIR/$OSM_FILE_FILTERED" >> "$WORKING_DIR/create_neo4jData_$CITY.log" 2>&1
-		cd $WORKING_DIR
-	fi
+	# We copy data from postgis for specified dataset (after postgis data import)
+	java -cp $DOWNLOAD_DIR/isochrone-tools.jar at.uibk.dbis.isochrone.exporter.ImportDataNeo4j -t "${CITY}" > "$WORKING_DIR/import_neo4j_$CITY.log" 2>&1
 }
 
 function fn_import_dataset_postgis() {
@@ -272,7 +236,7 @@ if $AVAILABLE_POSTGIS && $AVAILABLE_ISOCHRONE_TOOLS; then
 	java -cp $DOWNLOAD_DIR/isochrone-tools.jar at.uibk.dbis.isochrone.generator.density.DensityGenerator -t spider_l1000 -d 60,120,180,240,300 >> "$WORKING_DIR/generate_spiderNetwork.log" 2>&1
 fi
 
-# Not importing datasets directly into spatialite -> we copy data from postgis to spatialite (after postgis data import)
+# Not importing datasets one-by-one into spatialite -> we copy data from postgis (after postgis data import)
 if $IMPORT_DATA_SPATIALITE; then
 	fn_import_spatialite
 fi

bootstrap/prepareEnvironment.sh

@@ -48,7 +48,7 @@ INSTALL_POSTGRES=false
 INSTALL_POSTGRES_ADMIN=true
 INSTALL_PROJECT_PSIPROBE=true
 INSTALL_SPATIALITE=false
-INSTALL_SPATIALITE_ADMIN=true
+INSTALL_SPATIALITE_ADMIN=false
 
 SCRIPT="$(readlink -f ${BASH_SOURCE[0]})"
 SCRIPT_DIR="$(dirname ${SCRIPT})"
@@ -85,17 +85,18 @@ JAVA_VERSION="8u60"
 JAVA_FILE_32="jdk-$JAVA_VERSION-linux-i586.rpm"
 JAVA_FILE_64="jdk-$JAVA_VERSION-linux-x64.rpm"
 JAVA_CRYPTO_FILE="jce_policy-8.zip"
-MAVEN_VERSION="3.2.5"
+MAVEN_VERSION="3.3.3"
 MAVEN_FILE="apache-maven-$MAVEN_VERSION-bin.zip"
-MAVEN_DOWNLOAD_URL="http://tweedo.com/mirror/apache/maven/maven-3/$MAVEN_VERSION/binaries/$MAVEN_FILE"
+MAVEN_DOWNLOAD_URL="http://www.eu.apache.org/dist/maven/maven-3/$MAVEN_VERSION/binaries/$MAVEN_FILE"
 NEO4J_DB_USER="neo4j"
 NEO4J_VERSION="2.2.5"
 NEO4J_FILE="neo4j-community-$NEO4J_VERSION-unix.tar.gz"
 NEO4J_DOWNLOAD_URL="http://dist.neo4j.org/$NEO4J_FILE"
-NEO4J_SERVICE_NAME="neo4j-service"
-NEO4J_SPATIAL_GIT_URL="https://github.com/neo4j/spatial.git"
-NEO4J_SPATIAL_VERSION="0.13-neo4j-2.1"
-NEO4J_SPATIAL_PLUGIN_DOWNLOAD_URL="http://dist.neo4j.org.s3.amazonaws.com/spatial/neo4j-spatial-$NEO4J_SPATIAL_VERSION-server-plugin.zip"
+NEO4J_SERVICE_NAME="neo4j"
+#NEO4J_SPATIAL_GIT_URL="https://github.com/neo4j/spatial.git"
+#NEO4J_SPATIAL_VERSION="0.13-neo4j-2.1"
+NEO4J_SPATIAL_GIT_URL="https://github.com/RobertBierbauer/spatial.git"
+NEO4J_SPATIAL_VERSION="Enhanced-Functionality"
 NEXUS_ARTIFACT_NAME_WEB="isochrone-web"
 NEXUS_ARTIFACT_NAME_TOOLS="isochrone-tools"
 NEXUS_ARTIFACT_NAME_DATAMODEL="isochrone-datamodel"
@@ -219,6 +220,7 @@ if $IS_LOCAL_TEST_DEPLOY; then
 	INSTALL_ISOCHRONE_SNAPSHOT=false
 fi
 if $IMPORT_DATA_NEO4J_SPATIAL; then
+	INSTALL_ISOCHRONE_TOOLS=true
 	INSTALL_NEO4J_SPATIAL=true
 	INSTALL_OSMOSIS=true
 fi
@@ -236,6 +238,7 @@ if $INSTALL_GEOSERVER || $INSTALL_ISOCHRONE_LOCAL || $INSTALL_ISOCHRONE_RELEASE
 	INSTALL_TOMCAT=true
 fi
 if $INSTALL_NEO4J_SPATIAL; then
+	INSTALL_NEO4J=true
 	INSTALL_MAVEN=$IMPORT_DATA_NEO4J_SPATIAL
 fi
 if $INSTALL_ISOCHRONE_TOOLS || $INSTALL_ISOCHRONE_DATAMODEL || $INSTALL_TOMCAT || $INSTALL_GEOSERVER || $INSTALL_NEO4J || $INSTALL_NEO4J_SPATIAL || $IMPORT_DATA_NEO4J_SPATIAL; then
@@ -598,8 +601,13 @@ if $INSTALL_NEO4J; then
 	tar -xvf $DOWNLOAD_DIR/$NEO4J_FILE -C $DEPLOY_DIR >> $WORKING_DIR/install_neo4j.log 2>&1
 	ln -sf $DEPLOY_DIR/neo4j-community-$NEO4J_VERSION $DEPLOY_DIR/neo4j
 
-	echo "  - enabling neo4j network access"
+	echo "  - enabling neo4j network access (without user)"
 	echo "org.neo4j.server.webserver.address=0.0.0.0" >> $DEPLOY_DIR/neo4j/conf/neo4j-server.properties
+	sed -i "s/dbms\.security\.auth\_enabled=true/dbms\.security\.auth\_enabled=false/" $DEPLOY_DIR/neo4j/conf/neo4-server.properties 
+
+	echo "  - creating systemd service"
+	cp "$SHARED_CONF_DIR/neo4j.service" "/etc/systemd/system/neo4j.service" >> $DEPLOY_DIR/neo4j/conf/neo4j-server.properties
+	systemctl daemon-reload >> $DEPLOY_DIR/neo4j/conf/neo4j-server.properties
 fi
 
 if $INSTALL_NEO4J_SPATIAL; then
@@ -609,18 +617,16 @@ if $INSTALL_NEO4J_SPATIAL; then
 	cd $DEPLOY_DIR/neo4j-spatial
 	git checkout -q $NEO4J_SPATIAL_VERSION >> /dev/null 2>&1
 
+	echo "  - copying neo4j-spatial into geoserver (from source)"
+	mvn clean install -DskipTests >> $WORKING_DIR/install_neo4j_spatial.log 2>&1
+
+	if $INSTALL_NEO4J; then
+		unzip -o -d $DEPLOY_DIR/neo4j/plugins $DEPLOY_DIR/neo4j-spatial/target/*server-plugin.zip >> $WORKING_DIR/install_neo4j_spatial.log 2>&1
+	fi
 	if $INSTALL_GEOSERVER; then
-		if $IMPORT_DATA_NEO4J_SPATIAL; then
-			# We need to compile the source when importing data, so that the main class gets compiled
-			echo "  - copying neo4j-spatial into geoserver (from source)"
-			mvn clean install -DskipTests >> $WORKING_DIR/install_neo4j_spatial.log 2>&1
-			unzip -o -d $TOMCAT_WEBAPP_DIR/geoserver/WEB-INF/lib $DEPLOY_DIR/neo4j-spatial/target/*server-plugin.zip >> $WORKING_DIR/install_neo4j_spatial_plugin.log 2>&1
-		else
-			echo "  - copying neo4j-spatial into geoserver (from binary)"
-			fn_download_newer $DOWNLOAD_DIR/neo4j-spatial-$NEO4J_SPATIAL_VERSION-server-plugin.zip "$NEO4J_SPATIAL_PLUGIN_DOWNLOAD_URL"
-			unzip -o -d $TOMCAT_WEBAPP_DIR/geoserver/WEB-INF/lib $DOWNLOAD_DIR/neo4j-spatial-$NEO4J_SPATIAL_VERSION-server-plugin.zip >> $WORKING_DIR/install_neo4j_spatial_plugin.log 2>&1
-		fi
+		unzip -o -d $TOMCAT_WEBAPP_DIR/geoserver/WEB-INF/lib $DEPLOY_DIR/neo4j-spatial/target/*server-plugin.zip >> $WORKING_DIR/install_neo4j_spatial.log 2>&1
 	fi
+
 	cd "$WORKING_DIR"
 fi
 

conf/neo4j.service

+[Unit]
+Description=Neo4j
+
+[Service]
+Type=forking
+ExecStart=/opt/neo4j/bin/neo4j start
+ExecStop=/opt/neo4j/bin/neo4j stop
+PIDFile=/opt/neo4j/data/neo4j-service.pid