OO-4468: clean up method for metadata

src/main/java/org/olat/admin/sysinfo/LargeFilesController.java

@@ -19,6 +19,7 @@
  */
 package org.olat.admin.sysinfo;
 
+import java.io.File;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
@@ -35,7 +36,9 @@ import org.olat.admin.sysinfo.gui.LargeFilesTrashedCellRenderer;
 import org.olat.admin.sysinfo.model.LargeFilesTableContentRow;
 import org.olat.admin.sysinfo.model.LargeFilesTableModel;
 import org.olat.admin.sysinfo.model.LargeFilesTableModel.LargeFilesTableColumns;
+import org.olat.core.commons.persistence.DB;
 import org.olat.core.commons.persistence.SortKey;
+import org.olat.core.commons.services.taskexecutor.TaskExecutorManager;
 import org.olat.core.commons.services.vfs.VFSFilterKeys;
 import org.olat.core.commons.services.vfs.VFSMetadata;
 import org.olat.core.commons.services.vfs.VFSRepositoryModule;
@@ -67,12 +70,15 @@ import org.olat.core.gui.control.WindowControl;
 import org.olat.core.gui.control.generic.closablewrapper.CalloutSettings;
 import org.olat.core.gui.control.generic.closablewrapper.CloseableCalloutWindowController;
 import org.olat.core.gui.control.generic.closablewrapper.CloseableModalController;
+import org.olat.core.gui.control.generic.modal.DialogBoxController;
+import org.olat.core.gui.control.generic.modal.DialogBoxUIFactory;
 import org.olat.core.gui.util.CSSHelper;
 import org.olat.core.id.Identity;
 import org.olat.core.util.Formatter;
 import org.olat.core.util.StringHelper;
 import org.olat.core.util.mail.ContactList;
 import org.olat.core.util.mail.ContactMessage;
+import org.olat.core.util.vfs.VFSItem;
 import org.olat.modules.co.ContactFormController;
 import org.springframework.beans.factory.annotation.Autowired;
 
@@ -104,18 +110,23 @@ public class LargeFilesController extends FormBasicController implements Extende
 	private TextElement maxResultEl;
 	private TextElement minSizeEl;
 	private FormLink resetButton;
+	private FormLink cleanupMetadataButton;
 
 	private List<LargeFilesTableContentRow> rows;
 
 	private CloseableModalController cmc;
 	private ContactFormController contactCtrl;
+	private DialogBoxController confirmMetadataCleanupBox;
 	private CloseableCalloutWindowController pathInfoCalloutCtrl;
 
+	@Autowired
+	private DB dbInstance;
 	@Autowired
 	private VFSRepositoryService vfsRepositoryService;
 	@Autowired
 	private VFSRepositoryModule vfsRepositoryModule;
-
+	@Autowired
+	private TaskExecutorManager taskExecutorManager;
 
 	public LargeFilesController(UserRequest ureq, WindowControl wControl) {
 		super(ureq, wControl, "large_files");
@@ -137,7 +148,6 @@ public class LargeFilesController extends FormBasicController implements Extende
 		}
 		if(StringHelper.containsNonWhitespace(minSizeEl.getValue())) {
 			minSize = (int) (Double.parseDouble(minSizeEl.getValue()) * Formatter.BYTE_UNIT * Formatter.BYTE_UNIT);
-			System.out.println(minSize);
 		}
 		if(StringHelper.containsNonWhitespace(downloadCountMinEl.getValue())) {
 			downloadCountMin = Integer.parseInt(downloadCountMinEl.getValue());
@@ -234,9 +244,7 @@ public class LargeFilesController extends FormBasicController implements Extende
 
 	@Override
 	protected void initForm(FormItemContainer formLayout, Controller listener, UserRequest ureq) {
-		FormLayoutContainer largefFilesTitle = FormLayoutContainer.createVerticalFormLayout("largeFilesTitle", getTranslator());
-		formLayout.add(largefFilesTitle);
-		largefFilesTitle.setFormTitle(translate("largefiles.title"));
+		cleanupMetadataButton = uifactory.addFormLink("metadata.cleanup", formLayout, Link.BUTTON);
 
 		FormLayoutContainer leftContainer = FormLayoutContainer.createDefaultFormLayout_6_6("filter_left", getTranslator());
 		leftContainer.setRootForm(mainForm);
@@ -355,7 +363,7 @@ public class LargeFilesController extends FormBasicController implements Extende
 
 	@Override
 	protected void doDispose() {
-
+		//
 	}
 
 	@Override
@@ -386,6 +394,8 @@ public class LargeFilesController extends FormBasicController implements Extende
 			}
 		} else if(source == resetButton) {
 			resetForm();
+		} else if(cleanupMetadataButton == source) {
+			doConfirmMetadataCleanup(ureq);
 		} else if(source instanceof FormLink) {
 			FormLink link = (FormLink) source;
 
@@ -413,7 +423,11 @@ public class LargeFilesController extends FormBasicController implements Extende
 		} else if (source == contactCtrl) {
 			cmc.deactivate();
 			cleanUp();
-		} 
+		} else if(source == confirmMetadataCleanupBox) {
+			if (DialogBoxUIFactory.isYesEvent(event)) {
+				doCleanupMetadata();
+			}
+		}
 		super.event(ureq, source, event);
 	}
 
@@ -509,6 +523,65 @@ public class LargeFilesController extends FormBasicController implements Extende
 		cmc.activate();
 		listenTo(cmc);
 	}
+	
+	private void doConfirmMetadataCleanup(UserRequest ureq) {
+		String msg = translate("confirm.cleanup.metadata");
+		confirmMetadataCleanupBox = activateYesNoDialog(ureq, translate("confirm.cleanup.metadata.title"), msg, confirmMetadataCleanupBox);
+	}
+	
+	private void doCleanupMetadata() {
+		cleanupMetadataButton.setIconLeftCSS("o_icon o_icon_busy o_icon-spin");
+		cleanupMetadataButton.setEnabled(false);
+		
+		taskExecutorManager.execute(() -> {
+			cleanupMetadata();
+			if(cleanupMetadataButton != null) {
+				cleanupMetadataButton.setIconLeftCSS("");
+				cleanupMetadataButton.setEnabled(true);
+			}
+		});
+	}
+	
+	private void cleanupMetadata() {
+		try {
+			int counter = 0;
+			int batchSize = 10000;
+			List<VFSMetadata> metadata;
+			do {
+				metadata = vfsRepositoryService.getMetadatas(counter, batchSize);
+				for(VFSMetadata data:metadata) {
+					checkMetadata(data);
+				}
+				counter += metadata.size();
+				getLogger().info("Metadata processed: {}, total metadata processed ({})", metadata.size(), counter);
+				dbInstance.commitAndCloseSession();
+			} while(metadata.size() == batchSize);
+		} catch (Exception e) {
+			dbInstance.closeSession();
+			logError("", e);
+		}
+	}
+	
+	private void checkMetadata(VFSMetadata data) {
+		VFSItem item = vfsRepositoryService.getItemFor(data);
+		if(item == null || !item.exists()) {
+			boolean exists = false;
+			List<VFSRevision> revisions = vfsRepositoryService.getRevisions(data);
+			for(VFSRevision revision:revisions) {
+				File revFile = vfsRepositoryService.getRevisionFile(revision);
+				exists = revFile != null && revFile.exists();
+			}
+			
+			if(!exists) {
+				data = vfsRepositoryService.getMetadata(data);
+				if(data != null) {
+					getLogger().info("Delete metadata and associated: {}/{}", data.getRelativePath(), data.getFilename());
+					vfsRepositoryService.deleteMetadata(data);
+					dbInstance.commit();
+				}
+			}
+		}
+	}
 
 	@Override
 	public void setEnabled(boolean enable) {

src/main/java/org/olat/admin/sysinfo/_content/large_files.html

-$r.render("largeFilesTitle")
-<div class="row">
-	<div class="col-md-6">
-		$r.render("filter_left")
+<fieldset>
+	<legend>$r.translate("largefiles.title")</legend>
+	<div class="o_button_group o_button_group_right">
+	$r.render("metadata.cleanup")
 	</div>
-	<div class="col-md-6">
-		$r.render("filter_right")
+	<div class="row">
+		<div class="col-md-6">
+			$r.render("filter_left")
+		</div>
+		<div class="col-md-6">
+			$r.render("filter_right")
+		</div>
 	</div>
-</div>
-<div class="row">
-	<div class="col-md-offset-3 col-md-9">
-		$r.render("filter_buttons")
+	<div class="row">
+		<div class="col-md-offset-3 col-md-9">
+			$r.render("filter_buttons")
+		</div>
 	</div>
-</div>
-$r.render("large_files")
+	$r.render("large_files")
+</fieldset>
\ No newline at end of file

src/main/java/org/olat/admin/sysinfo/_i18n/LocalStrings_de.properties

@@ -8,6 +8,8 @@ block.login.sure=Sind Sie sicher, dass Sie f\u00FCr alle neuen Benutzer das Logi
 block.login.title=Login sperren
 buildinfo=Build Information
 clear.hibernate.statistics=Hibernate Statistik l\u00F6schen
+confirm.cleanup.metadata=Wollen Sie den Metadata aufra\u00FCmen? Der Prozess kann mehrere Minuten dauern.
+confirm.cleanup.metadata.title=Metadata aufra\u00FCmen
 core.functions=Funktions\u00FCbersicht
 core.jsMath=JS Math
 core.restapi=REST API
@@ -157,6 +159,7 @@ maintenancemsgClearCluster=Wartungsmeldung l\u00F6schen
 maintenancemsgEdit=Wartungsmeldung editieren
 maintenancemsgEditCluster=Wartungsmeldung editieren
 max.sessions.label=Maximale Anzahl Sessions
+metadata.cleanup=Metadata aufra\u00FCmen
 multiuser.title=Information \u00FCber Multiuser-Ereignisse
 nbr.session.label=Anzahl Sessions die invalidiert werden sollen (Sortiert nach letztem Zugriff)
 oldest.session.button=Sessions invalidieren

src/main/java/org/olat/admin/sysinfo/_i18n/LocalStrings_en.properties

@@ -8,6 +8,8 @@ block.login.sure=Do you really want to block the login for all new users?
 block.login.title=Block logins
 buildinfo=Build information
 clear.hibernate.statistics=Delete hibernate statistics
+confirm.cleanup.metadata=Do you want to clean up the metadata? This process can take several minutes.
+confirm.cleanup.metadata.title=Clean up metadata
 core.functions=Function overview
 core.jsMath=JS Math
 core.restapi=REST API
@@ -157,6 +159,7 @@ maintenancemsgClearCluster=Delete maintenance message
 maintenancemsgEdit=Edit maintenance message
 maintenancemsgEditCluster=Edit maintenance information
 max.sessions.label=Maximum number of sessions
+metadata.cleanup=Clean up metadata
 multiuser.title=Information on multi-user events
 nbr.session.label=Number of sessions to be invalidated (ordered by last access)
 oldest.session.button=Invalidate sessions 

src/main/java/org/olat/core/commons/services/vfs/VFSRepositoryService.java

@@ -71,6 +71,16 @@ public interface VFSRepositoryService {
 	 */
 	public List<VFSMetadata> getChildren(VFSMetadataRef parentMetadata);
 	
+	/**
+	 * The method is sorted by key, and doesn't fetch any
+	 * associated objects.
+	 * 
+	 * @param startPosition Start position (mandatory)
+	 * @param maxResults Max. number of rows to return
+	 * @return A list of metadata without any fetched objects
+	 */
+	public List<VFSMetadata> getMetadatas(int startPosition, int maxResults);
+	
 	public List<VFSMetadata> getMostDownloaded(VFSMetadata ancestorMetadata, int maxResults);
 	
 	/**

src/main/java/org/olat/core/commons/services/vfs/manager/VFSMetadataDAO.java

@@ -160,6 +160,15 @@ public class VFSMetadataDAO {
 		return metadata == null || metadata.isEmpty() ? null : metadata.get(0);
 	}
 	
+	public List<VFSMetadata> getMetadatas(int startPosition, int batchSize) {
+		String query = "select metadata from filemetadata metadata order by key";
+		return dbInstance.getCurrentEntityManager()
+				.createQuery(query, VFSMetadata.class)
+				.setFirstResult(startPosition)
+				.setMaxResults(batchSize)
+				.getResultList();
+	}
+	
 	/**
 	 * This is an exact match to find the direct children of a specific
 	 * directory.

src/main/java/org/olat/core/commons/services/vfs/manager/VFSRepositoryServiceImpl.java

@@ -192,6 +192,11 @@ public class VFSRepositoryServiceImpl implements VFSRepositoryService, GenericEv
 		return metadataDao.loadMetadata(ref.getKey());
 	}
 
+	@Override
+	public List<VFSMetadata> getMetadatas(int startPosition, int maxResults) {
+		return metadataDao.getMetadatas(startPosition, maxResults);
+	}
+
 	@Override
 	public VFSMetadata getMetadataFor(VFSItem path) {
 		File file = toFile(path);
@@ -390,7 +395,7 @@ public class VFSRepositoryServiceImpl implements VFSRepositoryService, GenericEv
 				try {
 					Files.delete(revFile.toPath());
 				} catch (IOException e) {
-					log.error("Cannot delete thumbnail: " + revFile, e);
+					log.error("Cannot delete thumbnail: {}", revFile, e);
 				}
 			}
 			revisionDao.deleteRevision(revision);

src/main/java/org/olat/core/util/FileUtils.java

@@ -52,9 +52,13 @@ import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 import org.apache.logging.log4j.Logger;
+import org.olat.core.CoreSpringFactory;
+import org.olat.core.commons.services.vfs.VFSRepositoryModule;
+import org.olat.core.commons.services.vfs.VFSRepositoryService;
 import org.olat.core.logging.AssertException;
 import org.olat.core.logging.OLATRuntimeException;
 import org.olat.core.logging.Tracing;
+import org.olat.core.util.vfs.VFSConstants;
 import org.olat.core.util.vfs.VFSContainer;
 import org.olat.core.util.vfs.VFSItem;
 import org.olat.core.util.vfs.VFSLeaf;
@@ -548,6 +552,9 @@ public class FileUtils {
 	public static boolean deleteFile(File file) {
 		boolean deleted = false;
 		try {
+			if(VFSRepositoryModule.canMeta(file) == VFSConstants.YES) {
+				CoreSpringFactory.getImpl(VFSRepositoryService.class).deleteMetadata(file);
+			}
 			deleted = Files.deleteIfExists(file.toPath());
 		} catch (IOException e) {
 			log.error("", e);