OO-2723 Upgrader to migrate all blogs and podcasts to the database

src/main/java/org/olat/modules/webFeed/manager/FeedManager.java

@@ -224,14 +224,20 @@ public abstract class FeedManager {
 	public abstract Feed loadFeedFromXML(Path feedDir);
 	
 	/**
-	 * Import the feed and all items from an feed XML file.<br>
-	 * The XML File is read and the feed is stored in the database. All XML
-	 * files of the items are read and the items stored in the database.<br>
-	 * At the end the XML Files are deleted.
+	 * In the early days all information about a feed where stored in XML files.
+	 * This method migrates that old feeds from the XML files to the database.
+	 * It first checks it the feed has to be migrated. If it has to, the XML
+	 * files are read, the feed and the items are saved in the database and at
+	 * the end the XML files are deleted. If the Feed is imported from an other
+	 * system the identity keys should be deleted because they do not
+	 * correspondent with the keys in the actual system.
 	 * 
 	 * @param ores
+	 * @param removeIdentityKeys
+	 *            If true, the identity keys of the author and the modifier are
+	 *            set to null.
 	 */
-	public abstract void importFeedFromXML(OLATResource ores);
+	public abstract void importFeedFromXML(OLATResource ores, boolean removeIdentityKeys);
 
 	/**
 	 * Returns the media file of the item

src/main/java/org/olat/modules/webFeed/manager/FeedManagerImpl.java

@@ -32,6 +32,7 @@ import java.util.Date;
 import java.util.List;
 
 import org.olat.admin.quota.QuotaConstants;
+import org.olat.basesecurity.BaseSecurity;
 import org.olat.basesecurity.BaseSecurityManager;
 import org.olat.core.CoreSpringFactory;
 import org.olat.core.commons.modules.bc.meta.tagged.MetaTagged;
@@ -81,6 +82,7 @@ import org.olat.repository.RepositoryEntry;
 import org.olat.repository.RepositoryManager;
 import org.olat.resource.OLATResource;
 import org.olat.resource.OLATResourceManager;
+import org.olat.user.UserManager;
 import org.springframework.beans.factory.annotation.Autowired;
 
 import com.rometools.rome.feed.synd.SyndEntry;
@@ -124,6 +126,8 @@ public class FeedManagerImpl extends FeedManager {
 	private ExternalFeedFetcher externalFeedFetcher;
 	@Autowired
 	private NotificationsManager notificationsManager;
+	@Autowired
+	private BaseSecurity securityManager;
 
 	/**
 	 * spring only
@@ -141,28 +145,16 @@ public class FeedManagerImpl extends FeedManager {
 		
 	}
 
-	/**
-	 * 
-	 * @param repositoryManager
-	 */
 	public void setRepositoryManager(RepositoryManager repositoryManager) {
 		this.repositoryManager = repositoryManager;
 	}
 
-	/**
-	 * 
-	 * @see org.olat.modules.webFeed.manager.FeedManager#createPodcastResource()
-	 */
 	@Override
 	public OLATResourceable createPodcastResource() {
 		FeedFileResource podcastResource = new PodcastFileResource();
 		return createFeedResource(podcastResource);
 	}
 
-	/**
-	 * 
-	 * @see org.olat.modules.webFeed.manager.FeedManager#createPodcastResource()
-	 */
 	@Override
 	public OLATResourceable createBlogResource() {
 		FeedFileResource blogResource = new BlogFileResource();
@@ -198,8 +190,6 @@ public class FeedManagerImpl extends FeedManager {
 	 */
 	@Override
 	public Feed loadFeed(OLATResourceable ores) {
-		migrateFeedFromXmlToDb(ores);
-
 		Feed feed = feedDAO.loadFeed(ores);
 		
 		feed = enrichFeedByRepositoryEntryInfromation(feed);
@@ -226,22 +216,6 @@ public class FeedManagerImpl extends FeedManager {
 		return feed;
 	}
 
-	/**
-	 * In the early days all information about a feed where stored in XML files.
-	 * This method migrates that old feeds from the XML files to the database.
-	 * It first checks it the feed has to be migrated. If it has to the XML
-	 * files are read, the feed and the items are saved to the database and at
-	 * the end the XML files are deleted. 
-	 * 
-	 * @param ores
-	 */
-	private void migrateFeedFromXmlToDb(OLATResourceable ores) {
-		OLATResource resource = resourceManager.findResourceable(ores);
-		if (resource != null) {
-			importFeedFromXML(resource);
-		}
-	}
-
 	@Override
 	public boolean hasItems(Feed feed) {
 		return itemDAO.hasItems(feed);
@@ -867,7 +841,7 @@ public class FeedManagerImpl extends FeedManager {
 	}
 
 	@Override
-	public void importFeedFromXML(OLATResource ores) {
+	public void importFeedFromXML(OLATResource ores, boolean removeIdentityKeys) {
 		Feed feedFromXml = feedFileStorage.loadFeedFromXML(ores);
 		if (feedFromXml == null) return;
 		
@@ -877,22 +851,48 @@ public class FeedManagerImpl extends FeedManager {
 		Feed feed = feedDAO.loadFeed(ores);
 		if (feed == null) {
 			feedFromXml.setResourceableId(ores.getResourceableId());
+			// Use the display name instead of the username
+			if (!removeIdentityKeys && feedFromXml.getAuthor() != null) {
+				String authorName = UserManager.getInstance().getUserDisplayName(feedFromXml.getAuthor());
+				if (authorName != null) {
+					feedFromXml.setAuthor(authorName);
+				}
+			}
 			feed = feedDAO.createFeed(feedFromXml);
+			log.info("Feed imported " + "(" + ores.getResourceableTypeName() + "): " + ores.getResourceableId());
 		}
 		
 		List<Item> itemsFromXml = feedFileStorage.loadItemsFromXML(ores);
 		itemsFromXml = fixFeedVersionIssues(feedFromXml, itemsFromXml);
 		for (Item itemFromXml : itemsFromXml) {
-			// Check if the feed already exits or create it. 
+			// Check if the item already exits or create it. 
 			Item item = itemDAO.loadItemByGuid(feed.getKey(), itemFromXml.getGuid());
 			if (item == null) {
-				itemFromXml.setAuthorKey(null);
-				itemFromXml.setModifierKey(null);
+				if (removeIdentityKeys) {
+					itemFromXml.setAuthorKey(null);
+					itemFromXml.setModifierKey(null);
+				} else {
+					// Check if the identity exists
+					if (itemFromXml.getAuthorKey() != null
+							&& securityManager.loadIdentityShortByKey(itemFromXml.getAuthorKey()) == null) {
+						itemFromXml.setAuthorKey(null);
+					}
+					if (itemFromXml.getModifierKey() != null
+							&& securityManager.loadIdentityShortByKey(itemFromXml.getModifierKey()) == null) {
+						itemFromXml.setModifierKey(null);
+					}
+				}
 				itemDAO.createItem(feed, itemFromXml);
+				log.info("Item imported: " + itemFromXml.getGuid());
 			}
 			feedFileStorage.deleteItemXML(itemFromXml);
 		}
 		
+		if (feed.isExternal()) {
+			saveExternalItems(feed);
+			saveExternalFeed(feed);
+		}
+		
 		feedFileStorage.deleteFeedXML(feed);
 	}
 
@@ -931,9 +931,6 @@ public class FeedManagerImpl extends FeedManager {
 
 	@Override
 	public LockResult acquireLock(OLATResourceable feed, Identity identity) {
-		// OLATResourceable itemLock =
-		// OresHelper.createOLATResourceableInstance("podcastlock_" +
-		// feed.getResourceableId() + "_meta", item.getId())
 		LockResult lockResult = coordinator.getLocker().acquireLock(feed, identity, null);
 		return lockResult;
 	}

src/main/java/org/olat/modules/webFeed/manager/RomeFeedFetcher.java

@@ -119,7 +119,7 @@ public class RomeFeedFetcher implements ExternalFeedFetcher {
 			syndFeed = syndFeedInput.build(xmlReader);
 			log.info("Read external feed: " + feedURL);
 		} catch (Exception e) {
-			log.warn("Cannot read external feed: : " + feedURL, e);
+			log.warn("Cannot read external feed: : " + feedURL);
 		}
 
 		return syndFeed;

src/main/java/org/olat/repository/handlers/BlogHandler.java

@@ -113,7 +113,7 @@ public class BlogHandler implements RepositoryHandler {
 		File fResourceFileroot = FileResourceManager.getInstance().getFileResourceRootImpl(resource).getBasefile();
 		File blogRoot = new File(fResourceFileroot, FeedManager.getInstance().getFeedKind(resource));
 		FileResource.copyResource(file, filename, blogRoot);
-		FeedManager.getInstance().importFeedFromXML(resource);
+		FeedManager.getInstance().importFeedFromXML(resource, true);
 		RepositoryEntry re = CoreSpringFactory.getImpl(RepositoryService.class)
 				.create(initialAuthor, null, "", displayname, description, resource, RepositoryEntry.ACC_OWNERS);
 		DBFactory.getInstance().commit();

src/main/java/org/olat/repository/handlers/PodcastHandler.java

@@ -113,7 +113,7 @@ public class PodcastHandler implements RepositoryHandler {
 		File fResourceFileroot = FileResourceManager.getInstance().getFileResourceRootImpl(resource).getBasefile();
 		File blogRoot = new File(fResourceFileroot, FeedManager.getInstance().getFeedKind(resource));
 		FileResource.copyResource(file, filename, blogRoot);
-		FeedManager.getInstance().importFeedFromXML(resource);
+		FeedManager.getInstance().importFeedFromXML(resource, true);
 		RepositoryEntry re = CoreSpringFactory.getImpl(RepositoryService.class)
 				.create(initialAuthor, null, "", displayname, description, resource, RepositoryEntry.ACC_OWNERS);
 		DBFactory.getInstance().commit();

src/main/java/org/olat/upgrade/OLATUpgrade.java

@@ -68,7 +68,7 @@ public abstract class OLATUpgrade {
 	 * still in the init method of the servlet. Here you have full access to the database
 	 * layer and you can savely access every module.
 	 * @param upgradeManager
-	 * @returntrue if anything has been upgraded, false if nothing has been 
+	 * @return true if anything has been upgraded, false if nothing has been 
 	 * upgraded (e.g. since the upgrade is already installed). false does not indicate
 	 * a failure! In case of failure, throw an exception!
 	 */
@@ -97,8 +97,8 @@ public abstract class OLATUpgrade {
 	 * @param dataSource
 	 */
 	public void executePlainSQLDBStatement(String query, DataSource dataSource) {
-    JdbcTemplate template = new JdbcTemplate(dataSource);
-    template.update(query);
+		JdbcTemplate template = new JdbcTemplate(dataSource);
+		template.update(query);
 	}
 
 	

src/main/java/org/olat/upgrade/OLATUpgrade_12_0_0.java

+/**
+ * <a href="http://www.openolat.org">
+ * OpenOLAT - Online Learning and Training</a><br>
+ * <p>
+ * Licensed under the Apache License, Version 2.0 (the "License"); <br>
+ * you may not use this file except in compliance with the License.<br>
+ * You may obtain a copy of the License at the
+ * <a href="http://www.apache.org/licenses/LICENSE-2.0">Apache homepage</a>
+ * <p>
+ * Unless required by applicable law or agreed to in writing,<br>
+ * software distributed under the License is distributed on an "AS IS" BASIS, <br>
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. <br>
+ * See the License for the specific language governing permissions and <br>
+ * limitations under the License.
+ * <p>
+ * Initial code contributed and copyrighted by<br>
+ * frentix GmbH, http://www.frentix.com
+ * <p>
+ */
+package org.olat.upgrade;
+
+import java.util.Arrays;
+import java.util.List;
+
+import org.olat.core.commons.persistence.DB;
+import org.olat.fileresource.types.BlogFileResource;
+import org.olat.fileresource.types.PodcastFileResource;
+import org.olat.modules.webFeed.manager.FeedManager;
+import org.olat.resource.OLATResource;
+import org.olat.resource.OLATResourceManager;
+import org.springframework.beans.factory.annotation.Autowired;
+
+/**
+ * 
+ * Initial date: 03.07.2017<br>
+ * @author uhensler, urs.hensler@frentix.com, http://www.frentix.com
+ *
+ */
+public class OLATUpgrade_12_0_0 extends OLATUpgrade {
+
+	private static final String VERSION = "OLAT_12.0.0";
+	private static final String FEED_XML_TO_DB = "FEED XML TO DB";
+	
+	@Autowired
+	private DB dbInstance;
+	@Autowired
+	FeedManager feedManager;
+	
+	public OLATUpgrade_12_0_0() {
+		super();
+	}
+	
+	@Override
+	public String getVersion() {
+		return VERSION;
+	}
+
+	@Override
+	public boolean doPreSystemInitUpgrade(UpgradeManager upgradeManager) {
+		return false;
+	}
+
+	@Override
+	public boolean doPostSystemInitUpgrade(UpgradeManager upgradeManager) {
+		UpgradeHistoryData uhd = upgradeManager.getUpgradesHistory(VERSION);
+		if (uhd == null) {
+			// has never been called, initialize
+			uhd = new UpgradeHistoryData();
+		} else if (uhd.isInstallationComplete()) {
+			return false;
+		}
+		
+		boolean allOk = true;
+		allOk &= upgradeBlogXmlToDb(upgradeManager, uhd);
+
+		uhd.setInstallationComplete(allOk);
+		upgradeManager.setUpgradesHistory(uhd, VERSION);
+		if(allOk) {
+			log.audit("Finished OLATUpgrade_12_0_0 successfully!");
+		} else {
+			log.audit("OLATUpgrade_12_0_0 not finished, try to restart OpenOLAT!");
+		}
+		return allOk;
+	}
+	
+	private boolean upgradeBlogXmlToDb(UpgradeManager upgradeManager, UpgradeHistoryData uhd) {
+		boolean allOk = true;
+		if (!uhd.getBooleanDataValue(FEED_XML_TO_DB)) {
+			
+			List<String> feedTypes = Arrays.asList(BlogFileResource.TYPE_NAME, PodcastFileResource.TYPE_NAME);
+			List<OLATResource> feeds = OLATResourceManager.getInstance().findResourceByTypes(feedTypes);
+			log.info("Number of feeds to upgrade: " + feeds.size());
+			for (OLATResource ores : feeds) {
+				log.info("Upgrade feed " + "(" + ores.getResourceableTypeName() + "): " + ores.getResourceableId());
+				try {
+					feedManager.importFeedFromXML(ores, false);
+				} catch (Exception e) {
+					allOk &= false;
+					log.error("", e);
+				}
+				dbInstance.commitAndCloseSession();
+			}
+			
+			uhd.setBooleanDataValue(FEED_XML_TO_DB, allOk);
+			upgradeManager.setUpgradesHistory(uhd, VERSION);
+		}
+		return allOk;
+	}
+
+}

src/main/java/org/olat/upgrade/_spring/upgradeContext.xml

@@ -51,6 +51,7 @@
 				<bean id="upgrade_11_2_1" class="org.olat.upgrade.OLATUpgrade_11_2_1"/>
 				<bean id="upgrade_11_3_0" class="org.olat.upgrade.OLATUpgrade_11_3_0"/>
 				<bean id="upgrade_11_4_0" class="org.olat.upgrade.OLATUpgrade_11_4_0"/>				
+				<bean id="upgrade_12_0_0" class="org.olat.upgrade.OLATUpgrade_12_0_0"/>
 			</list>
 		</property>
 	</bean>

src/test/java/org/olat/modules/webFeed/manager/FeedManagerImplTest.java

@@ -43,6 +43,7 @@ import org.olat.core.util.coordinate.Coordinator;
 import org.olat.core.util.coordinate.CoordinatorManager;
 import org.olat.core.util.coordinate.Syncer;
 import org.olat.fileresource.FileResourceManager;
+import org.olat.modules.webFeed.ExternalFeedFetcher;
 import org.olat.modules.webFeed.Feed;
 import org.olat.modules.webFeed.Item;
 import org.olat.resource.OLATResource;
@@ -64,6 +65,8 @@ public class FeedManagerImplTest {
 	@Mock
 	private FeedFileStorge feedFileStorageMock;
 	@Mock
+	private ExternalFeedFetcher  feedFetcherMock;
+	@Mock
 	private OLATResourceManager resourceManagerMock;
 	@Mock
 	private FileResourceManager fileResourceManagerMock;
@@ -92,6 +95,7 @@ public class FeedManagerImplTest {
 		ReflectionTestUtils.setField(sut, "feedDAO", feedDAOMock);
 		ReflectionTestUtils.setField(sut, "itemDAO", itemDAOMock);
 		ReflectionTestUtils.setField(sut, "feedFileStorage", feedFileStorageMock);
+		ReflectionTestUtils.setField(sut, "externalFeedFetcher", feedFetcherMock);
 		
 	}
 	
@@ -99,7 +103,7 @@ public class FeedManagerImplTest {
 	public void importShouldNothingDoIfNoXmlFileIsPresent() {
 		when(feedFileStorageMock.loadFeedFromXML(any(OLATResource.class))).thenReturn(null);
 		
-		sut.importFeedFromXML(any(OLATResource.class));
+		sut.importFeedFromXML(any(OLATResource.class), true);
 		
 		verifyZeroInteractions(feedDAOMock);
 		verifyZeroInteractions(itemDAOMock);
@@ -108,8 +112,10 @@ public class FeedManagerImplTest {
 	@Test
 	public void importShouldSaveFeedToDatabase() {
 		when(feedFileStorageMock.loadFeedFromXML(any(OLATResource.class))).thenReturn(feedDummy);
+		when(feedDAOMock.createFeed(any(Feed.class))).thenReturn(feedDummy);	
+		when(feedDummy.isExternal()).thenReturn(false);
 		
-		sut.importFeedFromXML(resourceDummy);
+		sut.importFeedFromXML(resourceDummy, true);
 		
 		verify(feedDAOMock).createFeed(any(Feed.class));
 	}
@@ -122,7 +128,7 @@ public class FeedManagerImplTest {
 		when(feedFileStorageMock.loadFeedFromXML(any(OLATResource.class))).thenReturn(feedDummy);
 		when(feedFileStorageMock.loadItemsFromXML(resourceDummy)).thenReturn(items);
 		
-		sut.importFeedFromXML(resourceDummy);
+		sut.importFeedFromXML(resourceDummy, true);
 		
 		verify(itemDAOMock, times(3)).createItem(any(Feed.class), any(Item.class));
 	}
@@ -135,7 +141,7 @@ public class FeedManagerImplTest {
 		when(feedFileStorageMock.loadFeedFromXML(any(OLATResource.class))).thenReturn(feedDummy);
 		when(feedFileStorageMock.loadItemsFromXML(resourceDummy)).thenReturn(emtpyList);
 		
-		sut.importFeedFromXML(resourceDummy);
+		sut.importFeedFromXML(resourceDummy, true);
 
 		verifyZeroInteractions(itemDAOMock);
 	}
@@ -149,7 +155,7 @@ public class FeedManagerImplTest {
 		when(feedFileStorageMock.loadItemsFromXML(resourceDummy)).thenReturn(items);
 		when(itemDAOMock.loadItemByGuid(any(Long.class), any(String.class))).thenReturn(itemDummy);
 		
-		sut.importFeedFromXML(resourceDummy);
+		sut.importFeedFromXML(resourceDummy, true);
 		
 		verify(itemDummy).setAuthorKey(null);
 	}
@@ -163,7 +169,7 @@ public class FeedManagerImplTest {
 		when(feedFileStorageMock.loadItemsFromXML(resourceDummy)).thenReturn(items);
 		when(itemDAOMock.loadItemByGuid(any(Long.class), any(String.class))).thenReturn(itemDummy);
 		
-		sut.importFeedFromXML(resourceDummy);
+		sut.importFeedFromXML(resourceDummy, true);
 		
 		verify(itemDummy).setModifierKey(null);
 	}
@@ -173,7 +179,7 @@ public class FeedManagerImplTest {
 		when(feedDAOMock.loadFeed(any(OLATResource.class))).thenReturn(feedDummy);		
 		when(feedFileStorageMock.loadFeedFromXML(any(OLATResource.class))).thenReturn(feedDummy);
 
-		sut.importFeedFromXML(resourceDummy);
+		sut.importFeedFromXML(resourceDummy, true);
 		
 		verify(feedFileStorageMock).deleteFeedXML(feedDummy);
 	}
@@ -186,11 +192,26 @@ public class FeedManagerImplTest {
 		when(feedFileStorageMock.loadFeedFromXML(any(OLATResource.class))).thenReturn(feedDummy);
 		when(feedFileStorageMock.loadItemsFromXML(resourceDummy)).thenReturn(items);
 		
-		sut.importFeedFromXML(resourceDummy);
+		sut.importFeedFromXML(resourceDummy, true);
 
 		verify(feedFileStorageMock, times(3)).deleteItemXML(itemDummy);
 	}
 	
+	@Test
+	public void importShoulLoadItemsOfExternalFeeds() {
+		List<Item> items = java.util.Arrays.asList(itemDummy);
+
+		when(feedDAOMock.loadFeed(any(OLATResource.class))).thenReturn(feedDummy);		
+		when(feedFileStorageMock.loadFeedFromXML(any(OLATResource.class))).thenReturn(feedDummy);
+		when(feedFileStorageMock.loadItemsFromXML(resourceDummy)).thenReturn(items);
+		when(feedDummy.isExternal()).thenReturn(true);
+		
+		sut.importFeedFromXML(resourceDummy, true);
+
+		verify(feedFetcherMock).fetchFeed(feedDummy);
+		verify(feedFetcherMock).fetchItems(feedDummy);
+	}
+	
 	@Test
 	public void exportShouldWriteFeedToXmlFile() {
 		when(feedDAOMock.loadFeed(any(OLATResource.class))).thenReturn(feedDummy);