diff --git a/CHANGELOG.md b/CHANGELOG.md index 0684e137992e9c557159c5d5cd831404aecac49e..0da4c482de0b5713784515a8863d857480e2c6a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,7 @@ Upcoming version: ----------------- + - re-activated Neo4J-spatial (with help from Robert Bierbauer's and his master thesis) (Nikolaus Krismer) + - now creating node_densities for updated datasets (Nikolaus Krismer) - updateing gradle to version 2.7 (Nikolaus Krismer) - updateing readme (tag location) (Nikolaus Krismer) - #2 fixed (Nikolaus Krismer) diff --git a/Vagrantfile b/Vagrantfile index 7b3cf6bf01fdb94dbc2ed27d430d66a2ed5f9b84..84486fce62a548b796e37aaf06de6a83d8c41f0e 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -71,6 +71,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| # Network configuration config.vm.network :forwarded_port, guest: 5432, host: 5432, id: "postgresql", auto_correct: false + config.vm.network :forwarded_port, guest: 7473, host: 7473, id: "neo4j-ssl", auto_correct: false config.vm.network :forwarded_port, guest: 7474, host: 7474, id: "neo4j", auto_correct: false config.vm.network :forwarded_port, guest: 80, host: 8000, id: "httpd", auto_correct: false unless isLocalMode diff --git a/bootstrap/config.sh b/bootstrap/config.sh index 885124869162c3ffc002fd7812623b8712f8f221..5651a17fb7a16e066e0fb732c7dd670a7c9a133d 100755 --- a/bootstrap/config.sh +++ b/bootstrap/config.sh @@ -3,7 +3,7 @@ # This is used to configure the behavior of the various bootstrap scripts. ############################# -IMPORT_DATA_NEO4J_SPATIAL=false +IMPORT_DATA_NEO4J_SPATIAL=true IMPORT_DATA_POSTGIS=true IMPORT_DATA_SPATIALITE=true diff --git a/bootstrap/importData.sh b/bootstrap/importData.sh index 13e0be9e5df76ae40dbcd7c25fd55f7c0db460f8..09b9c2d9f9b8cf823a3e1064b5df7ebd136fe6bc 100755 --- a/bootstrap/importData.sh +++ b/bootstrap/importData.sh @@ -69,61 +69,25 @@ function fn_import_dataset() { local BOUNDING="$4" local SQL_EXPORT_FILE="${NAME,,}_export.sql.gz" - local VALID_SQL=false - if $IMPORT_DATA_POSTGIS && $AVAILABLE_POSTGIS; then - if [ -n "$SQL_EXPORT_FILE" ]; then - VALID_SQL=true - fi - fi - echo "Importing data for region of $NAME" - if ! ( $VALID_SQL || $IMPORT_DATA_NEO4J_SPATIAL ); then - echo " - skipping import" - return 1 - fi + echo " - importing into postgis" + fn_import_dataset_postgis "$NAME" "$SQL_EXPORT_FILE" - if $VALID_SQL; then - fn_import_dataset_postgis "$NAME" "$SQL_EXPORT_FILE" - fi - - if $IMPORT_DATA_NEO4J_SPATIAL; then - fn_import_dataset_neo4j_spatial "$NAME" "$OSM_FILE" "$OSM_FILE_FILTERED" "$BOUNDING" + if $IMPORT_DATA_SPATIALITE; then + echo " - importing into neo4j" + fn_import_dataset_neo4j_spatial "$NAME" fi } function fn_import_dataset_neo4j_spatial() { local NAME="$1" - local OSM_FILE="$2" - local OSM_FILE_FILTERED="$3" - local BOUNDING="$4" + local SQL_EXPORT_FILE="$2" local CITY=${NAME// /} CITY=${CITY,,} - if !$UPDATE_DATA; then - echo " - downloading cached OpenStreetMap file" - fn_download_newer $DOWNLOAD_DIR/$OSM_FILE_FILTERED $CACHE_DIR_REMOTE_DATA/$OSM_FILE_FILTERED - cp $DOWNLOAD_DIR/$OSM_FILE_FILTERED $DATA_DIR/$OSM_FILE_FILTERED >> /dev/null 2>&1 - fi - - if $UPDATE_DATA && [ -f "$DATA_DIR/$OSM_FILE_FILTERED" ]; then - echo " - deleting outdated OpenStreetMap file" - rm -rf "$DATA_DIR/$OSM_FILE_FILTERED" - fi - - if $AVAILABLE_NEO4J_SPATIAL && [ ! -f "$DATA_DIR/$OSM_FILE_FILTERED" ]; then - echo " - downloading OpenStreetMap file (no cached file found)" - OSM_FILENAME=$DOWNLOAD_DIR/${OSM_FILE##*/} - fn_download_newer $OSM_FILENAME $OSM_DOWNLOAD_MIRROR/$OSM_FILE - - echo " - filtering OpenStreetMap data" - osmosis --read-pbf file="$OSM_FILENAME" --bounding-polygon file="$BOUNDING" --way-key keyList="$OSM_KEYS" --used-node --write-xml file="$DATA_DIR/$OSM_FILE_FILTERED" >> "$WORKING_DIR/create_neo4jData_$CITY.log" 2>&1 - - echo " - importing filtered OpenStreetMap data into neo4j-spatial" - cd $DEPLOY_DIR/neo4j-spatial - mvn exec:java -Dexec.mainClass=org.neo4j.gis.spatial.osm.OSMImporter -Dexec.args="$DEPLOY_DIR/neo4j/data/graph.db $DATA_DIR/$OSM_FILE_FILTERED" >> "$WORKING_DIR/create_neo4jData_$CITY.log" 2>&1 - cd $WORKING_DIR - fi + # We copy data from postgis for specified dataset (after postgis data import) + java -cp $DOWNLOAD_DIR/isochrone-tools.jar at.uibk.dbis.isochrone.exporter.ImportDataNeo4j -t "${CITY}" > "$WORKING_DIR/import_neo4j_$CITY.log" 2>&1 } function fn_import_dataset_postgis() { @@ -272,7 +236,7 @@ if $AVAILABLE_POSTGIS && $AVAILABLE_ISOCHRONE_TOOLS; then java -cp $DOWNLOAD_DIR/isochrone-tools.jar at.uibk.dbis.isochrone.generator.density.DensityGenerator -t spider_l1000 -d 60,120,180,240,300 >> "$WORKING_DIR/generate_spiderNetwork.log" 2>&1 fi -# Not importing datasets directly into spatialite -> we copy data from postgis to spatialite (after postgis data import) +# Not importing datasets one-by-one into spatialite -> we copy data from postgis (after postgis data import) if $IMPORT_DATA_SPATIALITE; then fn_import_spatialite fi diff --git a/bootstrap/prepareEnvironment.sh b/bootstrap/prepareEnvironment.sh index 682d9c4cf5c4289a92912c035d460139bb2fdf23..a55c3e6c061344371a241b72ee6bcd56b7c3da3f 100755 --- a/bootstrap/prepareEnvironment.sh +++ b/bootstrap/prepareEnvironment.sh @@ -48,7 +48,7 @@ INSTALL_POSTGRES=false INSTALL_POSTGRES_ADMIN=true INSTALL_PROJECT_PSIPROBE=true INSTALL_SPATIALITE=false -INSTALL_SPATIALITE_ADMIN=true +INSTALL_SPATIALITE_ADMIN=false SCRIPT="$(readlink -f ${BASH_SOURCE[0]})" SCRIPT_DIR="$(dirname ${SCRIPT})" @@ -85,17 +85,18 @@ JAVA_VERSION="8u60" JAVA_FILE_32="jdk-$JAVA_VERSION-linux-i586.rpm" JAVA_FILE_64="jdk-$JAVA_VERSION-linux-x64.rpm" JAVA_CRYPTO_FILE="jce_policy-8.zip" -MAVEN_VERSION="3.2.5" +MAVEN_VERSION="3.3.3" MAVEN_FILE="apache-maven-$MAVEN_VERSION-bin.zip" -MAVEN_DOWNLOAD_URL="http://tweedo.com/mirror/apache/maven/maven-3/$MAVEN_VERSION/binaries/$MAVEN_FILE" +MAVEN_DOWNLOAD_URL="http://www.eu.apache.org/dist/maven/maven-3/$MAVEN_VERSION/binaries/$MAVEN_FILE" NEO4J_DB_USER="neo4j" NEO4J_VERSION="2.2.5" NEO4J_FILE="neo4j-community-$NEO4J_VERSION-unix.tar.gz" NEO4J_DOWNLOAD_URL="http://dist.neo4j.org/$NEO4J_FILE" -NEO4J_SERVICE_NAME="neo4j-service" -NEO4J_SPATIAL_GIT_URL="https://github.com/neo4j/spatial.git" -NEO4J_SPATIAL_VERSION="0.13-neo4j-2.1" -NEO4J_SPATIAL_PLUGIN_DOWNLOAD_URL="http://dist.neo4j.org.s3.amazonaws.com/spatial/neo4j-spatial-$NEO4J_SPATIAL_VERSION-server-plugin.zip" +NEO4J_SERVICE_NAME="neo4j" +#NEO4J_SPATIAL_GIT_URL="https://github.com/neo4j/spatial.git" +#NEO4J_SPATIAL_VERSION="0.13-neo4j-2.1" +NEO4J_SPATIAL_GIT_URL="https://github.com/RobertBierbauer/spatial.git" +NEO4J_SPATIAL_VERSION="Enhanced-Functionality" NEXUS_ARTIFACT_NAME_WEB="isochrone-web" NEXUS_ARTIFACT_NAME_TOOLS="isochrone-tools" NEXUS_ARTIFACT_NAME_DATAMODEL="isochrone-datamodel" @@ -219,6 +220,7 @@ if $IS_LOCAL_TEST_DEPLOY; then INSTALL_ISOCHRONE_SNAPSHOT=false fi if $IMPORT_DATA_NEO4J_SPATIAL; then + INSTALL_ISOCHRONE_TOOLS=true INSTALL_NEO4J_SPATIAL=true INSTALL_OSMOSIS=true fi @@ -236,6 +238,7 @@ if $INSTALL_GEOSERVER || $INSTALL_ISOCHRONE_LOCAL || $INSTALL_ISOCHRONE_RELEASE INSTALL_TOMCAT=true fi if $INSTALL_NEO4J_SPATIAL; then + INSTALL_NEO4J=true INSTALL_MAVEN=$IMPORT_DATA_NEO4J_SPATIAL fi if $INSTALL_ISOCHRONE_TOOLS || $INSTALL_ISOCHRONE_DATAMODEL || $INSTALL_TOMCAT || $INSTALL_GEOSERVER || $INSTALL_NEO4J || $INSTALL_NEO4J_SPATIAL || $IMPORT_DATA_NEO4J_SPATIAL; then @@ -598,8 +601,13 @@ if $INSTALL_NEO4J; then tar -xvf $DOWNLOAD_DIR/$NEO4J_FILE -C $DEPLOY_DIR >> $WORKING_DIR/install_neo4j.log 2>&1 ln -sf $DEPLOY_DIR/neo4j-community-$NEO4J_VERSION $DEPLOY_DIR/neo4j - echo " - enabling neo4j network access" + echo " - enabling neo4j network access (without user)" echo "org.neo4j.server.webserver.address=0.0.0.0" >> $DEPLOY_DIR/neo4j/conf/neo4j-server.properties + sed -i "s/dbms\.security\.auth\_enabled=true/dbms\.security\.auth\_enabled=false/" $DEPLOY_DIR/neo4j/conf/neo4-server.properties + + echo " - creating systemd service" + cp "$SHARED_CONF_DIR/neo4j.service" "/etc/systemd/system/neo4j.service" >> $DEPLOY_DIR/neo4j/conf/neo4j-server.properties + systemctl daemon-reload >> $DEPLOY_DIR/neo4j/conf/neo4j-server.properties fi if $INSTALL_NEO4J_SPATIAL; then @@ -609,18 +617,16 @@ if $INSTALL_NEO4J_SPATIAL; then cd $DEPLOY_DIR/neo4j-spatial git checkout -q $NEO4J_SPATIAL_VERSION >> /dev/null 2>&1 + echo " - copying neo4j-spatial into geoserver (from source)" + mvn clean install -DskipTests >> $WORKING_DIR/install_neo4j_spatial.log 2>&1 + + if $INSTALL_NEO4J; then + unzip -o -d $DEPLOY_DIR/neo4j/plugins $DEPLOY_DIR/neo4j-spatial/target/*server-plugin.zip >> $WORKING_DIR/install_neo4j_spatial.log 2>&1 + fi if $INSTALL_GEOSERVER; then - if $IMPORT_DATA_NEO4J_SPATIAL; then - # We need to compile the source when importing data, so that the main class gets compiled - echo " - copying neo4j-spatial into geoserver (from source)" - mvn clean install -DskipTests >> $WORKING_DIR/install_neo4j_spatial.log 2>&1 - unzip -o -d $TOMCAT_WEBAPP_DIR/geoserver/WEB-INF/lib $DEPLOY_DIR/neo4j-spatial/target/*server-plugin.zip >> $WORKING_DIR/install_neo4j_spatial_plugin.log 2>&1 - else - echo " - copying neo4j-spatial into geoserver (from binary)" - fn_download_newer $DOWNLOAD_DIR/neo4j-spatial-$NEO4J_SPATIAL_VERSION-server-plugin.zip "$NEO4J_SPATIAL_PLUGIN_DOWNLOAD_URL" - unzip -o -d $TOMCAT_WEBAPP_DIR/geoserver/WEB-INF/lib $DOWNLOAD_DIR/neo4j-spatial-$NEO4J_SPATIAL_VERSION-server-plugin.zip >> $WORKING_DIR/install_neo4j_spatial_plugin.log 2>&1 - fi + unzip -o -d $TOMCAT_WEBAPP_DIR/geoserver/WEB-INF/lib $DEPLOY_DIR/neo4j-spatial/target/*server-plugin.zip >> $WORKING_DIR/install_neo4j_spatial.log 2>&1 fi + cd "$WORKING_DIR" fi diff --git a/conf/neo4j.service b/conf/neo4j.service new file mode 100644 index 0000000000000000000000000000000000000000..acdf1c6565868b45c267d0c63137cfc1d0f54e84 --- /dev/null +++ b/conf/neo4j.service @@ -0,0 +1,8 @@ +[Unit] +Description=Neo4j + +[Service] +Type=forking +ExecStart=/opt/neo4j/bin/neo4j start +ExecStop=/opt/neo4j/bin/neo4j stop +PIDFile=/opt/neo4j/data/neo4j-service.pid