OO-4468: take deleted rows in account, move code in service

src/main/java/org/olat/admin/sysinfo/LargeFilesController.java

@@ -19,7 +19,6 @@
  */
 package org.olat.admin.sysinfo;
 
-import java.io.File;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
@@ -36,7 +35,6 @@ import org.olat.admin.sysinfo.gui.LargeFilesTrashedCellRenderer;
 import org.olat.admin.sysinfo.model.LargeFilesTableContentRow;
 import org.olat.admin.sysinfo.model.LargeFilesTableModel;
 import org.olat.admin.sysinfo.model.LargeFilesTableModel.LargeFilesTableColumns;
-import org.olat.core.commons.persistence.DB;
 import org.olat.core.commons.persistence.SortKey;
 import org.olat.core.commons.services.taskexecutor.TaskExecutorManager;
 import org.olat.core.commons.services.vfs.VFSFilterKeys;
@@ -78,7 +76,6 @@ import org.olat.core.util.Formatter;
 import org.olat.core.util.StringHelper;
 import org.olat.core.util.mail.ContactList;
 import org.olat.core.util.mail.ContactMessage;
-import org.olat.core.util.vfs.VFSItem;
 import org.olat.modules.co.ContactFormController;
 import org.springframework.beans.factory.annotation.Autowired;
 
@@ -119,8 +116,6 @@ public class LargeFilesController extends FormBasicController implements Extende
 	private DialogBoxController confirmMetadataCleanupBox;
 	private CloseableCalloutWindowController pathInfoCalloutCtrl;
 
-	@Autowired
-	private DB dbInstance;
 	@Autowired
 	private VFSRepositoryService vfsRepositoryService;
 	@Autowired
@@ -245,6 +240,7 @@ public class LargeFilesController extends FormBasicController implements Extende
 	@Override
 	protected void initForm(FormItemContainer formLayout, Controller listener, UserRequest ureq) {
 		cleanupMetadataButton = uifactory.addFormLink("metadata.cleanup", formLayout, Link.BUTTON);
+		cleanupMetadataButton.getComponent().isSuppressDirtyFormWarning();
 
 		FormLayoutContainer leftContainer = FormLayoutContainer.createDefaultFormLayout_6_6("filter_left", getTranslator());
 		leftContainer.setRootForm(mainForm);
@@ -534,54 +530,13 @@ public class LargeFilesController extends FormBasicController implements Extende
 		cleanupMetadataButton.setEnabled(false);
 		
 		taskExecutorManager.execute(() -> {
-			cleanupMetadata();
+			vfsRepositoryService.cleanMetadatas();
 			if(cleanupMetadataButton != null) {
 				cleanupMetadataButton.setIconLeftCSS("");
 				cleanupMetadataButton.setEnabled(true);
 			}
 		});
 	}
-	
-	private void cleanupMetadata() {
-		try {
-			int counter = 0;
-			int batchSize = 10000;
-			List<VFSMetadata> metadata;
-			do {
-				metadata = vfsRepositoryService.getMetadatas(counter, batchSize);
-				for(VFSMetadata data:metadata) {
-					checkMetadata(data);
-				}
-				counter += metadata.size();
-				getLogger().info("Metadata processed: {}, total metadata processed ({})", metadata.size(), counter);
-				dbInstance.commitAndCloseSession();
-			} while(metadata.size() == batchSize);
-		} catch (Exception e) {
-			dbInstance.closeSession();
-			logError("", e);
-		}
-	}
-	
-	private void checkMetadata(VFSMetadata data) {
-		VFSItem item = vfsRepositoryService.getItemFor(data);
-		if(item == null || !item.exists()) {
-			boolean exists = false;
-			List<VFSRevision> revisions = vfsRepositoryService.getRevisions(data);
-			for(VFSRevision revision:revisions) {
-				File revFile = vfsRepositoryService.getRevisionFile(revision);
-				exists = revFile != null && revFile.exists();
-			}
-			
-			if(!exists) {
-				data = vfsRepositoryService.getMetadata(data);
-				if(data != null) {
-					getLogger().info("Delete metadata and associated: {}/{}", data.getRelativePath(), data.getFilename());
-					vfsRepositoryService.deleteMetadata(data);
-					dbInstance.commit();
-				}
-			}
-		}
-	}
 
 	@Override
 	public void setEnabled(boolean enable) {

src/main/java/org/olat/core/commons/services/vfs/VFSRepositoryService.java

@@ -71,15 +71,8 @@ public interface VFSRepositoryService {
 	 */
 	public List<VFSMetadata> getChildren(VFSMetadataRef parentMetadata);
 	
-	/**
-	 * The method is sorted by key, and doesn't fetch any
-	 * associated objects.
-	 * 
-	 * @param startPosition Start position (mandatory)
-	 * @param maxResults Max. number of rows to return
-	 * @return A list of metadata without any fetched objects
-	 */
-	public List<VFSMetadata> getMetadatas(int startPosition, int maxResults);
+	
+	public void cleanMetadatas();
 	
 	public List<VFSMetadata> getMostDownloaded(VFSMetadata ancestorMetadata, int maxResults);
 	
@@ -109,7 +102,7 @@ public interface VFSRepositoryService {
 	 * 
 	 * @param data The metadata to remove
 	 */
-	public void deleteMetadata(VFSMetadata data);
+	public int deleteMetadata(VFSMetadata data);
 	
 	/**
 	 * Delete the metadata (but not the file), the thumbnails and versions

src/main/java/org/olat/core/commons/services/vfs/manager/VFSMetadataDAO.java

@@ -160,6 +160,14 @@ public class VFSMetadataDAO {
 		return metadata == null || metadata.isEmpty() ? null : metadata.get(0);
 	}
 	
+	/**
+	 * The method is sorted by key, and doesn't fetch any
+	 * associated objects.
+	 * 
+	 * @param startPosition Start position (mandatory)
+	 * @param maxResults Max. number of rows to return
+	 * @return A list of metadata without any fetched objects
+	 */
 	public List<VFSMetadata> getMetadatas(int startPosition, int batchSize) {
 		String query = "select metadata from filemetadata metadata order by key";
 		return dbInstance.getCurrentEntityManager()

src/main/java/org/olat/core/commons/services/vfs/manager/VFSRepositoryServiceImpl.java

@@ -193,8 +193,62 @@ public class VFSRepositoryServiceImpl implements VFSRepositoryService, GenericEv
 	}
 
 	@Override
-	public List<VFSMetadata> getMetadatas(int startPosition, int maxResults) {
-		return metadataDao.getMetadatas(startPosition, maxResults);
+	public void cleanMetadatas() {
+		try {
+			int loop = 0;
+			int counter = 0;
+			int processed = 0;
+			int batchSize = 10000;
+			int maxLoops = 10000;// allow 100'000'000 rows, but prevent an infinite loop in case
+			int totalDeleted = 0;
+			List<VFSMetadata> metadata;
+			do {
+				metadata = metadataDao.getMetadatas(counter, batchSize);
+				int deleted = 0;
+				for(VFSMetadata data:metadata) {
+					deleted += checkMetadata(data);
+				}
+				counter += metadata.size() - deleted;
+				totalDeleted += deleted;
+				if(counter < 0) {
+					counter = 0;
+				}
+				loop++;
+				processed += metadata.size();
+				log.info("Metadata processed: {}, deleted {}, total metadata processed ({})", metadata.size(), deleted, processed);
+				dbInstance.commitAndCloseSession();
+			} while(metadata.size() == batchSize && loop < maxLoops);
+			
+			log.info("Cleanup metadata ended: deleted {}, total metadata processed ({})", totalDeleted, processed);
+		} catch (Exception e) {
+			dbInstance.closeSession();
+			log.error("", e);
+		}
+	}
+	
+	private int checkMetadata(VFSMetadata data) {
+		int deleted = 0;
+		
+		VFSItem item = getItemFor(data);
+		if(item == null || !item.exists() || item.getName().startsWith("._oo_")) {
+			boolean exists = false;
+			List<VFSRevision> revisions = getRevisions(data);
+			for(VFSRevision revision:revisions) {
+				File revFile = getRevisionFile(revision);
+				exists = revFile != null && revFile.exists();
+			}
+			
+			if(!exists) {
+				data = getMetadata(data);
+				if(data != null) {
+					log.info("Delete metadata and associated: {}/{}", data.getRelativePath(), data.getFilename());
+					deleted = deleteMetadata(data);
+					dbInstance.commit();
+				}
+			}
+		}
+		
+		return deleted;
 	}
 
 	@Override
@@ -376,8 +430,8 @@ public class VFSRepositoryServiceImpl implements VFSRepositoryService, GenericEv
 	}
 
 	@Override
-	public void deleteMetadata(VFSMetadata data) {
-		if(data == null) return; // nothing to do
+	public int deleteMetadata(VFSMetadata data) {
+		if(data == null) return 0; // nothing to do
 		
 		List<VFSThumbnailMetadata> thumbnails = thumbnailDao.loadByMetadata(data);
 		for(VFSThumbnailMetadata thumbnail:thumbnails) {
@@ -401,11 +455,15 @@ public class VFSRepositoryServiceImpl implements VFSRepositoryService, GenericEv
 			revisionDao.deleteRevision(revision);
 		}
 		
+		int deleted = 0;
+		
 		List<VFSMetadata> children = getChildren(data);
 		for(VFSMetadata child:children) {
-			deleteMetadata(child);
+			deleted += deleteMetadata(child);
 		}
 		metadataDao.removeMetadata(data);
+		deleted++;
+		return deleted;
 	}
 
 	@Override