From d5e0275657d92829d72ba573d44c0267487792d9 Mon Sep 17 00:00:00 2001
From: srosse <none@none>
Date: Wed, 23 Jul 2014 17:08:14 +0200
Subject: [PATCH] OO-990: refresh the index searcher if needed
---
.../java/org/olat/core/util/CodeHelper.java | 6 +
.../org/olat/search/_spring/searchContext.xml | 4 +-
.../olat/search/service/SearchCallable.java | 4 +-
.../search/service/SearchOrderByCallable.java | 6 +-
.../search/service/SearchServiceImpl.java | 249 ++++++++++++------
.../olat/search/service/indexer/Index.java | 19 +-
.../service/indexer/IndexWriterHolder.java | 17 +-
.../search/service/indexer/IndexerEvent.java | 43 +++
.../search/service/indexer/JmsIndexer.java | 29 +-
.../service/indexer/OlatFullIndexer.java | 39 +--
10 files changed, 289 insertions(+), 127 deletions(-)
create mode 100644 src/main/java/org/olat/search/service/indexer/IndexerEvent.java
diff --git a/src/main/java/org/olat/core/util/CodeHelper.java b/src/main/java/org/olat/core/util/CodeHelper.java
index d31ffa27559..e9f03803cdd 100644
--- a/src/main/java/org/olat/core/util/CodeHelper.java
+++ b/src/main/java/org/olat/core/util/CodeHelper.java
@@ -104,6 +104,12 @@ public class CodeHelper {
return UUID.randomUUID().toString();
}
+ public static long nanoToMilliTime(long start) {
+ long end = System.nanoTime();
+ long takes = (end - start) / 1000000;
+ return takes;
+ }
+
public static void printNanoTime(long start, String action) {
long end = System.nanoTime();
long takes = (end - start) / 1000000;
diff --git a/src/main/java/org/olat/search/_spring/searchContext.xml b/src/main/java/org/olat/search/_spring/searchContext.xml
index 22c7c3a1daa..d43f697c454 100644
--- a/src/main/java/org/olat/search/_spring/searchContext.xml
+++ b/src/main/java/org/olat/search/_spring/searchContext.xml
@@ -20,7 +20,8 @@
<constructor-arg index="0" ref="searchModule" />
<constructor-arg index="1" ref="mainIndexer" />
<constructor-arg index="2" ref="searchProvider" />
- <constructor-arg index="3" ref="schedulerFactoryBean"/>
+ <constructor-arg index="3" ref="coordinatorManager"/>
+ <constructor-arg index="4" ref="schedulerFactoryBean"/>
<property name="lifeIndexer" ref="jmsIndexer"/>
<property name="metadataFields" ref="SearchMetadataFieldsProvider" />
<property name="searchExecutor" ref="searchExecutor" />
@@ -42,6 +43,7 @@
<bean id="jmsIndexer" class="org.olat.search.service.indexer.JmsIndexer" init-method="springInit" destroy-method="stop">
<constructor-arg index="0" ref="searchModule" />
+ <constructor-arg index="1" ref="coordinatorManager"/>
<property name="connectionFactory" ref="indexConnectionFactory"/>
<property name="jmsQueue" ref="indexQueue"/>
<property name="searchService" ref="org.olat.search.service.${search.service}" />
diff --git a/src/main/java/org/olat/search/service/SearchCallable.java b/src/main/java/org/olat/search/service/SearchCallable.java
index 9071799bd25..a3bab180741 100644
--- a/src/main/java/org/olat/search/service/SearchCallable.java
+++ b/src/main/java/org/olat/search/service/SearchCallable.java
@@ -68,6 +68,7 @@ class SearchCallable implements Callable<SearchResults> {
@Override
public SearchResults call() throws ParseException {
+ IndexSearcher searcher = null;
try {
boolean debug = log.isDebug();
@@ -77,7 +78,7 @@ class SearchCallable implements Callable<SearchResults> {
}
if(debug) log.debug("queryString=" + queryString);
- IndexSearcher searcher = searchService.getIndexSearcher();
+ searcher = searchService.getIndexSearcher();
BooleanQuery query = searchService.createQuery(queryString, condQueries);
if(debug) log.debug("query=" + query);
@@ -99,6 +100,7 @@ class SearchCallable implements Callable<SearchResults> {
log.error("", naex);
return null;
} finally {
+ searchService.releaseIndexSearcher(searcher);
DBFactory.getInstance().commitAndCloseSession();
}
}
diff --git a/src/main/java/org/olat/search/service/SearchOrderByCallable.java b/src/main/java/org/olat/search/service/SearchOrderByCallable.java
index 25657c30181..47b50713e5c 100644
--- a/src/main/java/org/olat/search/service/SearchOrderByCallable.java
+++ b/src/main/java/org/olat/search/service/SearchOrderByCallable.java
@@ -68,6 +68,7 @@ class SearchOrderByCallable implements Callable<List<Long>> {
@Override
public List<Long> call() {
+ IndexSearcher searcher = null;
try {
if (!searchService.existIndex()) {
log.warn("Index does not exist, can't search for queryString: "+queryString);
@@ -75,11 +76,11 @@ class SearchOrderByCallable implements Callable<List<Long>> {
}
log.info("queryString=" + queryString);
- IndexSearcher searcher = searchService.getIndexSearcher();
+ searcher = searchService.getIndexSearcher();
BooleanQuery query = searchService.createQuery(queryString, condQueries);
//log.info("query=" + query);
- int n = SearchServiceFactory.getService().getSearchModuleConfig().getMaxHits();
+ int n = searchService.getSearchModuleConfig().getMaxHits();
TopDocs docs;
if(orderBy != null && orderBy.length > 0 && orderBy[0] != null) {
SortField[] sortFields = new SortField[orderBy.length];
@@ -112,6 +113,7 @@ class SearchOrderByCallable implements Callable<List<Long>> {
log.error("", naex);
return null;
} finally {
+ searchService.releaseIndexSearcher(searcher);
DBFactory.getInstance().commitAndCloseSession();
}
}
diff --git a/src/main/java/org/olat/search/service/SearchServiceImpl.java b/src/main/java/org/olat/search/service/SearchServiceImpl.java
index 4058b85725b..c16e970ea07 100644
--- a/src/main/java/org/olat/search/service/SearchServiceImpl.java
+++ b/src/main/java/org/olat/search/service/SearchServiceImpl.java
@@ -35,10 +35,12 @@ import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
+import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.MultiReader;
import org.apache.lucene.queryparser.classic.MultiFieldQueryParser;
import org.apache.lucene.queryparser.classic.ParseException;
@@ -47,10 +49,11 @@ import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
+import org.apache.lucene.search.ReferenceManager;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
-import org.olat.core.CoreSpringFactory;
import org.olat.core.commons.persistence.SortKey;
+import org.olat.core.gui.control.Event;
import org.olat.core.id.Identity;
import org.olat.core.id.Roles;
import org.olat.core.logging.AssertException;
@@ -58,6 +61,8 @@ import org.olat.core.logging.OLog;
import org.olat.core.logging.Tracing;
import org.olat.core.util.ArrayHelper;
import org.olat.core.util.StringHelper;
+import org.olat.core.util.coordinate.CoordinatorManager;
+import org.olat.core.util.event.GenericEventListener;
import org.olat.modules.qpool.model.QItemDocument;
import org.olat.search.QueryException;
import org.olat.search.SearchModule;
@@ -68,6 +73,7 @@ import org.olat.search.ServiceNotAvailableException;
import org.olat.search.model.AbstractOlatDocument;
import org.olat.search.service.indexer.FullIndexerStatus;
import org.olat.search.service.indexer.Index;
+import org.olat.search.service.indexer.IndexerEvent;
import org.olat.search.service.indexer.LifeFullIndexer;
import org.olat.search.service.indexer.MainIndexer;
import org.olat.search.service.searcher.JmsSearchProvider;
@@ -80,18 +86,16 @@ import org.quartz.SchedulerException;
*
* @author Christian Guretzki
*/
-public class SearchServiceImpl implements SearchService {
+public class SearchServiceImpl implements SearchService, GenericEventListener {
private static final OLog log = Tracing.createLoggerFor(SearchServiceImpl.class);
private Index indexer;
private SearchModule searchModuleConfig;
private MainIndexer mainIndexer;
private Scheduler scheduler;
+ private CoordinatorManager coordinatorManager;
private Analyzer analyzer;
- private IndexSearcher searcher;
- private DirectoryReader reader;
- private DirectoryReader permReader;
private LifeFullIndexer lifeIndexer;
private SearchSpellChecker searchSpellChecker;
@@ -100,9 +104,9 @@ public class SearchServiceImpl implements SearchService {
/** Counts number of search queries since last restart. */
private long queryCount = 0;
- private Object createIndexSearcherLock = new Object();
private ExecutorService searchExecutor;
+ private OOSearcherManager indexSearcherRefMgr;
private String fields[] = {
AbstractOlatDocument.TITLE_FIELD_NAME, AbstractOlatDocument.DESCRIPTION_FIELD_NAME,
@@ -122,14 +126,17 @@ public class SearchServiceImpl implements SearchService {
/**
* [used by spring]
*/
- private SearchServiceImpl(SearchModule searchModule, MainIndexer mainIndexer, JmsSearchProvider searchProvider,
+ private SearchServiceImpl(SearchModule searchModule, MainIndexer mainIndexer,
+ JmsSearchProvider searchProvider, CoordinatorManager coordinatorManager,
Scheduler scheduler) {
log.info("Start SearchServiceImpl constructor...");
this.scheduler = scheduler;
this.searchModuleConfig = searchModule;
this.mainIndexer = mainIndexer;
+ this.coordinatorManager = coordinatorManager;
analyzer = new StandardAnalyzer(SearchService.OO_LUCENE_VERSION);
searchProvider.setSearchService(this);
+ coordinatorManager.getCoordinator().getEventBus().registerFor(this, null, IndexerEvent.INDEX_ORES);
}
public void setSearchExecutor(ExecutorService searchExecutor) {
@@ -155,11 +162,11 @@ public class SearchServiceImpl implements SearchService {
/**
* Start the job indexer
*/
+ @Override
public void startIndexing() {
if (indexer==null) throw new AssertException ("Try to call startIndexing() but indexer is null");
try {
- Scheduler scheduler = CoreSpringFactory.getImpl(Scheduler.class);
JobDetail detail = scheduler.getJobDetail("org.olat.search.job.enabled", Scheduler.DEFAULT_GROUP);
scheduler.triggerJob(detail.getName(), detail.getGroup());
log.info("startIndexing...");
@@ -171,11 +178,11 @@ public class SearchServiceImpl implements SearchService {
/**
* Interrupt the job indexer
*/
+ @Override
public void stopIndexing() {
if (indexer==null) throw new AssertException ("Try to call stopIndexing() but indexer is null");
try {
- Scheduler scheduler = CoreSpringFactory.getImpl(Scheduler.class);
JobDetail detail = scheduler.getJobDetail("org.olat.search.job.enabled", Scheduler.DEFAULT_GROUP);
scheduler.interrupt(detail.getName(), detail.getGroup());
log.info("stopIndexing.");
@@ -188,9 +195,9 @@ public class SearchServiceImpl implements SearchService {
return indexer;
}
+ @Override
public void init() {
log.info("init searchModuleConfig=" + searchModuleConfig);
-
log.info("Running with indexPath=" + searchModuleConfig.getFullIndexPath());
log.info(" tempIndexPath=" + searchModuleConfig.getFullTempIndexPath());
log.info(" generateAtStartup=" + searchModuleConfig.getGenerateAtStartup());
@@ -202,20 +209,45 @@ public class SearchServiceImpl implements SearchService {
searchSpellChecker.setSpellCheckEnabled(searchModuleConfig.getSpellCheckEnabled());
searchSpellChecker.setSearchExecutor(searchExecutor);
- indexer = new Index(searchModuleConfig, searchSpellChecker, mainIndexer, lifeIndexer);
+ indexer = new Index(searchModuleConfig, searchSpellChecker, mainIndexer, lifeIndexer, coordinatorManager);
- indexPath = searchModuleConfig.getFullIndexPath();
- permanentIndexPath = searchModuleConfig.getFullPermanentIndexPath();
+ indexPath = searchModuleConfig.getFullIndexPath();
+ permanentIndexPath = searchModuleConfig.getFullPermanentIndexPath();
+
+ createIndexSearcherManager();
- if (startingFullIndexingAllowed()) {
- try {
+ if (startingFullIndexingAllowed()) {
+ try {
JobDetail detail = scheduler.getJobDetail("org.olat.search.job.enabled", Scheduler.DEFAULT_GROUP);
scheduler.triggerJob(detail.getName(), detail.getGroup());
} catch (SchedulerException e) {
log.error("", e);
}
- }
- log.info("init DONE");
+ }
+ log.info("init DONE");
+ }
+
+ private void createIndexSearcherManager() {
+ try {
+ if(indexSearcherRefMgr == null) {
+ if(existIndex()) {
+ indexSearcherRefMgr = new OOSearcherManager(this);
+ }
+ } else {
+ indexSearcherRefMgr.needRefresh();
+ }
+ } catch (IOException e) {
+ log.error("Cannot initialized the searcher manager", e);
+ }
+ }
+
+ @Override
+ public void event(Event event) {
+ if(event instanceof IndexerEvent) {
+ if(IndexerEvent.INDEX_CREATED.equals(event.getCommand())) {
+ createIndexSearcherManager();
+ }
+ }
}
/**
@@ -277,16 +309,16 @@ public class SearchServiceImpl implements SearchService {
String[] fieldsArr = getFieldsToSearchIn();
QueryParser queryParser = new MultiFieldQueryParser(SearchService.OO_LUCENE_VERSION, fieldsArr, analyzer);
queryParser.setLowercaseExpandedTerms(false);//some add. fields are not tokenized and not lowered case
- Query multiFieldQuery = queryParser.parse(queryString.toLowerCase());
- query.add(multiFieldQuery, Occur.MUST);
+ Query multiFieldQuery = queryParser.parse(queryString.toLowerCase());
+ query.add(multiFieldQuery, Occur.MUST);
}
if(condQueries != null && !condQueries.isEmpty()) {
for(String condQueryString:condQueries) {
QueryParser condQueryParser = new QueryParser(SearchService.OO_LUCENE_VERSION, condQueryString, analyzer);
condQueryParser.setLowercaseExpandedTerms(false);
- Query condQuery = condQueryParser.parse(condQueryString);
- query.add(condQuery, Occur.MUST);
+ Query condQuery = condQueryParser.parse(condQueryString);
+ query.add(condQuery, Occur.MUST);
}
}
return query;
@@ -300,24 +332,29 @@ public class SearchServiceImpl implements SearchService {
* Delegates impl to the searchSpellChecker.
* @see org.olat.search.service.searcher.OLATSearcher#spellCheck(java.lang.String)
*/
+ @Override
public Set<String> spellCheck(String query) {
if(searchSpellChecker==null) throw new AssertException ("Try to call spellCheck() in Search.java but searchSpellChecker is null");
return searchSpellChecker.check(query);
}
+ @Override
public long getQueryCount() {
return queryCount;
}
-
+
+ @Override
public SearchServiceStatus getStatus() {
return new SearchServiceStatusImpl(indexer,this);
}
+ @Override
public void setIndexInterval(long indexInterval) {
if (indexer==null) throw new AssertException ("Try to call setIndexInterval() but indexer is null");
indexer.setIndexInterval(indexInterval);
}
-
+
+ @Override
public long getIndexInterval() {
if (indexer==null) throw new AssertException ("Try to call setIndexInterval() but indexer is null");
return indexer.getIndexInterval();
@@ -327,10 +364,12 @@ public class SearchServiceImpl implements SearchService {
*
* @return Resturn search module configuration.
*/
+ @Override
public SearchModule getSearchModuleConfig() {
return searchModuleConfig;
}
+ @Override
public void stop() {
SearchServiceStatus status = getStatus();
String statusStr = status.getStatus();
@@ -338,66 +377,113 @@ public class SearchServiceImpl implements SearchService {
stopIndexing();
}
try {
- if (searcher != null) {
- searcher.getIndexReader().close();
- searcher = null;
+ if (indexSearcherRefMgr != null) {
+ indexSearcherRefMgr.close();
+ indexSearcherRefMgr = null;
}
} catch (Exception e) {
log.error("", e);
}
}
+ @Override
public boolean isEnabled() {
return true;
}
- protected IndexSearcher getIndexSearcher() throws ServiceNotAvailableException, IOException {
- if(searcher == null) {
- synchronized (createIndexSearcherLock) {//o_clusterOK by:fj if service is only configured on one vm, which is recommended way
- if (searcher == null) {
- try {
- getIndexSearcher(indexPath, permanentIndexPath);
- } catch(IOException ioEx) {
- log.warn("Can not create searcher", ioEx);
- throw new ServiceNotAvailableException("Index is not available");
- }
- }
- }
+ protected IndexSearcher getIndexSearcher()
+ throws ServiceNotAvailableException, IOException {
+ if(indexSearcherRefMgr == null) {
+ throw new ServiceNotAvailableException("Local search not available");
}
+
+ indexSearcherRefMgr.maybeRefresh();
+ return indexSearcherRefMgr.acquire();
+ }
- return getIndexSearcher(indexPath, permanentIndexPath);
+ protected void releaseIndexSearcher(IndexSearcher s) {
+ try {
+ indexSearcherRefMgr.release(s);
+ } catch (IOException e) {
+ log.error("Error while releasing index searcher", e);
+ }
}
+
- private synchronized IndexSearcher getIndexSearcher(String path, String permanentPath)
- throws IOException {
- boolean hasChanged = false;
- if(reader == null) {
- hasChanged = true;
- reader = DirectoryReader.open(FSDirectory.open(new File(path)));
- } else {
- DirectoryReader newReader = DirectoryReader.openIfChanged(reader);
- if(newReader != null) {
- hasChanged = true;
- reader = newReader;
- }
+ private IndexSearcher newSearcher() throws IOException {
+ DirectoryReader classicReader = DirectoryReader.open(FSDirectory.open(new File(indexPath)));
+ DirectoryReader permanentReader = DirectoryReader.open(FSDirectory.open(new File(permanentIndexPath)));
+ OOMultiReader mReader = new OOMultiReader(classicReader, permanentReader);
+ return new IndexSearcher(mReader);
+ }
+
+ private static class OOMultiReader extends MultiReader {
+
+ private final DirectoryReader reader;
+ private final DirectoryReader permanentReader;
+
+ public OOMultiReader(DirectoryReader reader, DirectoryReader permanentReader) {
+ super(reader, permanentReader);
+ this.reader = reader;
+ this.permanentReader = permanentReader;
+ }
+
+ public DirectoryReader getReader() {
+ return reader;
+ }
+
+ public DirectoryReader getPermanentReader() {
+ return permanentReader;
+ }
+ }
+
+ private static class OOSearcherManager extends ReferenceManager<IndexSearcher> {
+
+ private final SearchServiceImpl factory;
+ private AtomicBoolean refresh = new AtomicBoolean(false);
+
+ public OOSearcherManager(SearchServiceImpl factory) throws IOException {
+ this.factory = factory;
+ this.current = getSearcher(factory);
}
- if(permReader == null) {
- hasChanged = true;
- permReader = DirectoryReader.open(FSDirectory.open(new File(permanentPath)));
- } else {
- DirectoryReader newReader = DirectoryReader.openIfChanged(permReader);
- if(newReader != null) {
- hasChanged = true;
- permReader = newReader;
- }
+ protected void needRefresh() {
+ refresh.getAndSet(true);
+ }
+
+ @Override
+ protected void decRef(IndexSearcher reference) throws IOException {
+ reference.getIndexReader().decRef();
+ }
+
+ @Override
+ protected IndexSearcher refreshIfNeeded(IndexSearcher referenceToRefresh)
+ throws IOException {
+ final OOMultiReader r = (OOMultiReader)referenceToRefresh.getIndexReader();
+ final IndexReader newReader = DirectoryReader.openIfChanged(r.getReader());
+ final IndexReader newPermReader = DirectoryReader.openIfChanged(r.getPermanentReader());
+ if (newReader == null && newPermReader == null) {
+ return null;
+ } else {
+ return getSearcher(factory);
+ }
+ }
+
+ @Override
+ protected boolean tryIncRef(IndexSearcher reference) throws IOException {
+ return reference.getIndexReader().tryIncRef();
+ }
+
+ @Override
+ protected int getRefCount(IndexSearcher reference) {
+ return reference.getIndexReader().getRefCount();
}
- if(hasChanged) {
- MultiReader mReader = new MultiReader(reader, permReader);
- searcher = new IndexSearcher(mReader);
+ public static IndexSearcher getSearcher(SearchServiceImpl searcherFactory)
+ throws IOException {
+ IndexSearcher searcher = searcherFactory.newSearcher();
+ return searcher;
}
- return searcher;
}
/**
@@ -442,23 +528,22 @@ public class SearchServiceImpl implements SearchService {
* @return TRUE: Starting is allowed
*/
private boolean startingFullIndexingAllowed() {
- if (searchModuleConfig.getGenerateAtStartup()) {
- Calendar calendar = Calendar.getInstance();
- calendar.setTime(new Date());
- // check day, Restart only at config day of week, 0-7 8=every day
- int dayNow = calendar.get(Calendar.DAY_OF_WEEK);
- int restartDayOfWeek = searchModuleConfig.getRestartDayOfWeek();
- if (restartDayOfWeek == 0 || (dayNow == restartDayOfWeek) ) {
- // check time, Restart only in the config time-slot e.g. 01:00 - 03:00
- int hourNow = calendar.get(Calendar.HOUR_OF_DAY);
- int restartWindowStart = searchModuleConfig.getRestartWindowStart();
- int restartWindowEnd = searchModuleConfig.getRestartWindowEnd();
- if ( (restartWindowStart <= hourNow) && (hourNow < restartWindowEnd) ) {
- return true;
- }
- }
- }
- return false;
+ if (searchModuleConfig.getGenerateAtStartup()) {
+ Calendar calendar = Calendar.getInstance();
+ calendar.setTime(new Date());
+ // check day, Restart only at config day of week, 0-7 8=every day
+ int dayNow = calendar.get(Calendar.DAY_OF_WEEK);
+ int restartDayOfWeek = searchModuleConfig.getRestartDayOfWeek();
+ if (restartDayOfWeek == 0 || (dayNow == restartDayOfWeek) ) {
+ // check time, Restart only in the config time-slot e.g. 01:00 - 03:00
+ int hourNow = calendar.get(Calendar.HOUR_OF_DAY);
+ int restartWindowStart = searchModuleConfig.getRestartWindowStart();
+ int restartWindowEnd = searchModuleConfig.getRestartWindowEnd();
+ if ( (restartWindowStart <= hourNow) && (hourNow < restartWindowEnd) ) {
+ return true;
+ }
+ }
+ }
+ return false;
}
-
}
diff --git a/src/main/java/org/olat/search/service/indexer/Index.java b/src/main/java/org/olat/search/service/indexer/Index.java
index 956c9df8555..87668bad24d 100644
--- a/src/main/java/org/olat/search/service/indexer/Index.java
+++ b/src/main/java/org/olat/search/service/indexer/Index.java
@@ -35,6 +35,7 @@ import org.olat.core.helpers.Settings;
import org.olat.core.logging.OLog;
import org.olat.core.logging.Tracing;
import org.olat.core.util.FileUtils;
+import org.olat.core.util.coordinate.CoordinatorManager;
import org.olat.search.SearchModule;
import org.olat.search.service.spell.SearchSpellChecker;
@@ -62,14 +63,15 @@ public class Index {
* @param restartInterval Restart interval of full-index in milliseconds.
* @param indexInterval Sleeping time in milliseconds between adding documents to index.
*/
- public Index(SearchModule searchModuleConfig, SearchSpellChecker spellChecker, MainIndexer mainIndexer, LifeFullIndexer lifeIndexer) {
+ public Index(SearchModule searchModuleConfig, SearchSpellChecker spellChecker, MainIndexer mainIndexer,
+ LifeFullIndexer lifeIndexer, CoordinatorManager coordinatorManager) {
this.spellChecker = spellChecker;
this.indexPath = searchModuleConfig.getFullIndexPath();
this.tempIndexPath = searchModuleConfig.getFullTempIndexPath();
this.permanentIndexPath = searchModuleConfig.getFullPermanentIndexPath();
this.lifeIndexer = lifeIndexer;
- fullIndexer = new OlatFullIndexer(this, searchModuleConfig, mainIndexer);
+ fullIndexer = new OlatFullIndexer(this, searchModuleConfig, mainIndexer, coordinatorManager);
}
/**
@@ -78,9 +80,9 @@ public class Index {
public void startFullIndex() {
// do not start search engine in test mode, some repository tests might lead to nullpointers
// since only dummy entries are generated (or fix the search service to handle those correctly)
- if ( ! Settings.isJUnitTest()) {
+ if (!Settings.isJUnitTest()) {
lifeIndexer.fullIndex();
- fullIndexer.startIndexing();
+ fullIndexer.startIndexing();
}
}
@@ -123,12 +125,7 @@ public class Index {
*/
public void indexingIsDone() {
// Full indexing is done => move tempIndex to index dir
- moveTempIndexToIndex(tempIndexPath,indexPath);
- spellChecker.createSpellIndex();
- }
-
- private void moveTempIndexToIndex(String tempIndexPath, String indexPath) {
- File indexDir = new File( indexPath );
+ File indexDir = new File(indexPath);
if (!indexDir.exists()) {
indexDir.mkdirs();
}
@@ -138,6 +135,8 @@ public class Index {
FileUtils.deleteDirsAndFiles(indexDir, true, false);
FileUtils.copyDirContentsToDir(new File(tempIndexDir, "main") , indexDir ,true, "search indexer move tmp index");
log.info("New generated Index ready to use." );
+
+ spellChecker.createSpellIndex();
}
public OlatFullIndexer getIndexer() {
diff --git a/src/main/java/org/olat/search/service/indexer/IndexWriterHolder.java b/src/main/java/org/olat/search/service/indexer/IndexWriterHolder.java
index 34e67551d33..0f11373727e 100644
--- a/src/main/java/org/olat/search/service/indexer/IndexWriterHolder.java
+++ b/src/main/java/org/olat/search/service/indexer/IndexWriterHolder.java
@@ -27,6 +27,7 @@ import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.store.Directory;
import org.olat.core.logging.OLog;
import org.olat.core.logging.Tracing;
+import org.olat.core.util.CodeHelper;
/**
*
@@ -49,27 +50,33 @@ public class IndexWriterHolder {
this.indexer = indexer;
}
- public synchronized void ensureIndexExists() {
+ /**
+ * @return true if it has created the index
+ */
+ public synchronized boolean ensureIndexExists() {
+ boolean created = false;
IndexWriter writer = null;
try {
if(!DirectoryReader.indexExists(indexPath)) {
writer = getAndLock();
+ created = true;
}
} catch (IOException e) {
log.error("", e);
} finally {
release(writer);
}
+ return created;
}
public synchronized IndexWriter getAndLock() throws IOException {
if(writerRef == null) {
- long start = System.currentTimeMillis();
+ long start = System.nanoTime();
IndexWriter indexWriter = new IndexWriter(indexPath, indexer.newIndexWriterConfig());
if(!DirectoryReader.indexExists(indexPath)) {
indexWriter.commit();//make sure it exists
}
- log.info("Opening writer takes (ms): " + (System.currentTimeMillis() - start));
+ log.info("Opening writer takes (ms): " + CodeHelper.nanoToMilliTime(start));
writerRef = indexWriter;
}
counter.incrementAndGet();
@@ -81,11 +88,11 @@ public class IndexWriterHolder {
try {
int used = counter.decrementAndGet();
if(used == 0) {
- long start = System.currentTimeMillis();
+ long start = System.nanoTime();
indexWriter.commit();
indexWriter.close();
writerRef = null;
- log.info("Close writer takes (ms): " + (System.currentTimeMillis() - start));
+ log.info("Close writer takes (ms): " + CodeHelper.nanoToMilliTime(start));
}
} catch (Exception e) {
log.error("", e);
diff --git a/src/main/java/org/olat/search/service/indexer/IndexerEvent.java b/src/main/java/org/olat/search/service/indexer/IndexerEvent.java
new file mode 100644
index 00000000000..f14fbd4bd8c
--- /dev/null
+++ b/src/main/java/org/olat/search/service/indexer/IndexerEvent.java
@@ -0,0 +1,43 @@
+/**
+ * <a href="http://www.openolat.org">
+ * OpenOLAT - Online Learning and Training</a><br>
+ * <p>
+ * Licensed under the Apache License, Version 2.0 (the "License"); <br>
+ * you may not use this file except in compliance with the License.<br>
+ * You may obtain a copy of the License at the
+ * <a href="http://www.apache.org/licenses/LICENSE-2.0">Apache homepage</a>
+ * <p>
+ * Unless required by applicable law or agreed to in writing,<br>
+ * software distributed under the License is distributed on an "AS IS" BASIS, <br>
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. <br>
+ * See the License for the specific language governing permissions and <br>
+ * limitations under the License.
+ * <p>
+ * Initial code contributed and copyrighted by<br>
+ * frentix GmbH, http://www.frentix.com
+ * <p>
+ */
+package org.olat.search.service.indexer;
+
+import org.olat.core.id.OLATResourceable;
+import org.olat.core.util.event.MultiUserEvent;
+import org.olat.core.util.resource.OresHelper;
+
+/**
+ *
+ * Initial date: 23.07.2014<br>
+ * @author srosse, stephane.rosse@frentix.com, http://www.frentix.com
+ *
+ */
+public class IndexerEvent extends MultiUserEvent {
+
+ public static final String INDEX_CREATED = "index-created";
+ public static final OLATResourceable INDEX_ORES = OresHelper.createOLATResourceableType("Indexer");
+
+ private static final long serialVersionUID = -2979077265102230245L;
+
+ public IndexerEvent(String cmd) {
+ super(cmd);
+ }
+
+}
diff --git a/src/main/java/org/olat/search/service/indexer/JmsIndexer.java b/src/main/java/org/olat/search/service/indexer/JmsIndexer.java
index fc384be96d1..fbfbc83da9d 100644
--- a/src/main/java/org/olat/search/service/indexer/JmsIndexer.java
+++ b/src/main/java/org/olat/search/service/indexer/JmsIndexer.java
@@ -54,6 +54,7 @@ import org.apache.lucene.store.FSDirectory;
import org.olat.core.commons.persistence.DBFactory;
import org.olat.core.logging.OLog;
import org.olat.core.logging.Tracing;
+import org.olat.core.util.coordinate.CoordinatorManager;
import org.olat.search.SearchModule;
import org.olat.search.SearchService;
import org.olat.search.model.AbstractOlatDocument;
@@ -78,6 +79,7 @@ public class JmsIndexer implements MessageListener, LifeFullIndexer {
private QueueConnection connection;
private SearchService searchService;
+ private CoordinatorManager coordinatorManager;
private String permanentIndexPath;
private DirectoryReader reader;
@@ -88,10 +90,11 @@ public class JmsIndexer implements MessageListener, LifeFullIndexer {
private List<LifeIndexer> indexers = new ArrayList<LifeIndexer>();
- public JmsIndexer(SearchModule searchModuleConfig) {
+ public JmsIndexer(SearchModule searchModuleConfig, CoordinatorManager coordinatorManager) {
indexingNode = searchModuleConfig.isSearchServiceEnabled();
ramBufferSizeMB = searchModuleConfig.getRAMBufferSizeMB();
permanentIndexPath = searchModuleConfig.getFullPermanentIndexPath();
+ this.coordinatorManager = coordinatorManager;
}
public Queue getJmsQueue() {
@@ -172,8 +175,12 @@ public class JmsIndexer implements MessageListener, LifeFullIndexer {
File tempIndexDir = new File(permanentIndexPath);
Directory indexPath = FSDirectory.open(tempIndexDir);
if(indexingNode) {
- permanentIndexWriter = new IndexWriterHolder (indexPath, this);
- permanentIndexWriter.ensureIndexExists();
+ permanentIndexWriter = new IndexWriterHolder(indexPath, this);
+ boolean created = permanentIndexWriter.ensureIndexExists();
+ if(created) {
+ IndexerEvent event = new IndexerEvent(IndexerEvent.INDEX_CREATED);
+ coordinatorManager.getCoordinator().getEventBus().fireEventToListenersOf(event, IndexerEvent.INDEX_ORES);
+ }
}
reader = DirectoryReader.open(indexPath);
} catch (IOException e) {
@@ -297,8 +304,8 @@ public class JmsIndexer implements MessageListener, LifeFullIndexer {
private void doIndex(JmsIndexWork workUnit) {
if(searchService instanceof SearchServiceImpl) {
String type = workUnit.getIndexType();
- List<LifeIndexer> indexers = getIndexerByType(type);
- for(LifeIndexer indexer:indexers) {
+ List<LifeIndexer> lifeIndexers = getIndexerByType(type);
+ for(LifeIndexer indexer:lifeIndexers) {
indexer.indexDocument(workUnit.getKey(), this);
}
}
@@ -307,8 +314,8 @@ public class JmsIndexer implements MessageListener, LifeFullIndexer {
private void doDelete(JmsIndexWork workUnit) {
if(searchService instanceof SearchServiceImpl) {
String type = workUnit.getIndexType();
- List<LifeIndexer> indexers = getIndexerByType(type);
- for(LifeIndexer indexer:indexers) {
+ List<LifeIndexer> lifeIndexers = getIndexerByType(type);
+ for(LifeIndexer indexer:lifeIndexers) {
indexer.deleteDocument(workUnit.getKey(), this);
}
}
@@ -360,8 +367,8 @@ public class JmsIndexer implements MessageListener, LifeFullIndexer {
String resourceUrl = document.get(AbstractOlatDocument.RESOURCEURL_FIELD_NAME);
Term uuidTerm = new Term(AbstractOlatDocument.RESOURCEURL_FIELD_NAME, resourceUrl);
- DirectoryReader reader = getReader();
- IndexSearcher searcher = new IndexSearcher(reader);
+ DirectoryReader currentReader = getReader();
+ IndexSearcher searcher = new IndexSearcher(currentReader);
TopDocs hits = searcher.search(new TermQuery(uuidTerm), 10);
writer = permanentIndexWriter.getAndLock();
if(hits.totalHits > 0) {
@@ -381,8 +388,8 @@ public class JmsIndexer implements MessageListener, LifeFullIndexer {
try {
String resourceUrl = document.get(AbstractOlatDocument.RESOURCEURL_FIELD_NAME);
Term uuidTerm = new Term(AbstractOlatDocument.RESOURCEURL_FIELD_NAME, resourceUrl);
- DirectoryReader reader = getReader();
- IndexSearcher searcher = new IndexSearcher(reader);
+ DirectoryReader currentReader = getReader();
+ IndexSearcher searcher = new IndexSearcher(currentReader);
TopDocs hits = searcher.search(new TermQuery(uuidTerm), 10);
if(hits.totalHits > 0) {
writer.updateDocument(uuidTerm, document);
diff --git a/src/main/java/org/olat/search/service/indexer/OlatFullIndexer.java b/src/main/java/org/olat/search/service/indexer/OlatFullIndexer.java
index c077e551438..418453b5dd0 100644
--- a/src/main/java/org/olat/search/service/indexer/OlatFullIndexer.java
+++ b/src/main/java/org/olat/search/service/indexer/OlatFullIndexer.java
@@ -47,6 +47,7 @@ import org.olat.core.commons.persistence.DBFactory;
import org.olat.core.logging.OLog;
import org.olat.core.logging.Tracing;
import org.olat.core.util.WorkThreadInformations;
+import org.olat.core.util.coordinate.CoordinatorManager;
import org.olat.search.SearchModule;
import org.olat.search.SearchService;
import org.olat.search.model.OlatDocument;
@@ -71,12 +72,13 @@ public class OlatFullIndexer {
*/
private Index index;
private IndexWriter indexWriter;
-
-
+
/** Flag to stop indexing. */
private boolean stopIndexing;
/** When restartIndexingWhenFinished is true, the restart interval in ms can be set. */
private long indexInterval = 500;
+
+ private double ramBufferSizeMB;
/** Current status of full-indexer. */
private FullIndexerStatus fullIndexerStatus;
@@ -99,7 +101,8 @@ public class OlatFullIndexer {
private Map<String,Integer> fileTypeCounters;
private MainIndexer mainIndexer;
- private double ramBufferSizeMB;
+ private CoordinatorManager coordinatorManager;
+
/**
*
@@ -108,18 +111,20 @@ public class OlatFullIndexer {
* @param restartInterval Restart interval in milliseconds.
* @param indexInterval Sleep time in milliseconds between adding documents.
*/
- public OlatFullIndexer(Index index, SearchModule searchModuleConfig, MainIndexer mainIndexer) {
- this.index = index;
- this.mainIndexer = mainIndexer;
- tempIndexPath = searchModuleConfig.getFullTempIndexPath();
- indexInterval = searchModuleConfig.getIndexInterval();
- numberIndexWriter = searchModuleConfig.getNumberIndexWriter();
- documentsPerInterval = searchModuleConfig.getDocumentsPerInterval();
- ramBufferSizeMB = searchModuleConfig.getRAMBufferSizeMB();
- fullIndexerStatus = new FullIndexerStatus(numberIndexWriter);
- stopIndexing = true;
- documentQueue = new Vector<Document>();
- resetDocumentCounters();
+ public OlatFullIndexer(Index index, SearchModule searchModuleConfig,
+ MainIndexer mainIndexer, CoordinatorManager coordinatorManager) {
+ this.index = index;
+ this.mainIndexer = mainIndexer;
+ this.coordinatorManager = coordinatorManager;
+ tempIndexPath = searchModuleConfig.getFullTempIndexPath();
+ indexInterval = searchModuleConfig.getIndexInterval();
+ numberIndexWriter = searchModuleConfig.getNumberIndexWriter();
+ documentsPerInterval = searchModuleConfig.getDocumentsPerInterval();
+ ramBufferSizeMB = searchModuleConfig.getRAMBufferSizeMB();
+ fullIndexerStatus = new FullIndexerStatus(numberIndexWriter);
+ stopIndexing = true;
+ documentQueue = new Vector<Document>();
+ resetDocumentCounters();
}
/**
@@ -228,6 +233,10 @@ public class OlatFullIndexer {
indexWriter.close();
indexWriter = null;
indexWriterWorkers = null;
+
+ //created because the index is deleted and copied
+ IndexerEvent event = new IndexerEvent(IndexerEvent.INDEX_CREATED);
+ coordinatorManager.getCoordinator().getEventBus().fireEventToListenersOf(event, IndexerEvent.INDEX_ORES);
} catch (IOException e) {
log.warn("Can not create IndexWriter, indexname=" + tempIndexPath, e);
} finally {
--
GitLab