From 2bb721829a22a7df1f69f77431aa882b69ad6483 Mon Sep 17 00:00:00 2001
From: srosse <stephane.rosse@frentix.com>
Date: Fri, 29 Jun 2018 18:12:44 +0200
Subject: [PATCH] OO-3562: don't reuse the support NGramProfil across request
but create a new one every time
---
.../text/impl/nutch/LanguageIdentifier.java | 5 +-
.../text/impl/nutch/NGramProfile.java | 9 +-
.../services/text/TextServiceTest.java | 97 +++++++++++++++++++
.../java/org/olat/test/AllTestsJunit4.java | 1 +
4 files changed, 105 insertions(+), 7 deletions(-)
create mode 100644 src/test/java/org/olat/core/commons/services/text/TextServiceTest.java
diff --git a/src/main/java/org/olat/core/commons/services/text/impl/nutch/LanguageIdentifier.java b/src/main/java/org/olat/core/commons/services/text/impl/nutch/LanguageIdentifier.java
index 7cd48d66dac..318baeb0024 100644
--- a/src/main/java/org/olat/core/commons/services/text/impl/nutch/LanguageIdentifier.java
+++ b/src/main/java/org/olat/core/commons/services/text/impl/nutch/LanguageIdentifier.java
@@ -66,8 +66,7 @@ public class LanguageIdentifier {
/** A global index of ngrams of all supported languages */
private Map<CharSequence, NGramEntry[]> ngramsIdx = new HashMap<CharSequence, NGramEntry[]>();
- /** The NGramProfile used for identification */
- private NGramProfile suspect = null;
+
/**
@@ -136,7 +135,6 @@ public class LanguageIdentifier {
}
log.info(list.toString());
// Create the suspect profile
- suspect = new NGramProfile("suspect", minLength, maxLength);
} catch (Exception e) {
log.error("", e);
}
@@ -172,6 +170,7 @@ public class LanguageIdentifier {
text.setLength(analyzeLength);
}
+ NGramProfile suspect = new NGramProfile("suspect", minLength, maxLength);
suspect.analyze(text);
Iterator<NGramEntry> iter = suspect.getSorted().iterator();
float topscore = Float.MIN_VALUE;
diff --git a/src/main/java/org/olat/core/commons/services/text/impl/nutch/NGramProfile.java b/src/main/java/org/olat/core/commons/services/text/impl/nutch/NGramProfile.java
index 0dd725e558f..b7bb2ce8b5d 100644
--- a/src/main/java/org/olat/core/commons/services/text/impl/nutch/NGramProfile.java
+++ b/src/main/java/org/olat/core/commons/services/text/impl/nutch/NGramProfile.java
@@ -127,7 +127,7 @@ public class NGramProfile {
*
* @param w is the word to add
*/
- public void add(StringBuffer w) {
+ public void add(StringBuilder w) {
for (int i=minLength; (i <= maxLength) && (i < w.length()); i++) {
add(w, i);
}
@@ -203,7 +203,7 @@ public class NGramProfile {
* @param w
* @param n sequence length
*/
- private void add(StringBuffer w, int n) {
+ private void add(StringBuilder w, int n) {
for (int i=0; i <= w.length()-n; i++) {
add(w.subSequence(i, i + n));
}
@@ -254,9 +254,10 @@ public class NGramProfile {
}
// Inherited JavaDoc
+ @Override
public String toString() {
- StringBuffer s = new StringBuffer().append("NGramProfile: ")
+ StringBuilder s = new StringBuilder().append("NGramProfile: ")
.append(name).append("\n");
Iterator<NGramEntry> i = getSorted().iterator();
@@ -543,7 +544,7 @@ public class NGramProfile {
* @param count is the number of occurences of this ngram
*/
public NGramEntry(String seq, int count) {
- this.seq = new StringBuffer(seq).subSequence(0, seq.length());
+ this.seq = new StringBuilder(seq).subSequence(0, seq.length());
this.count = count;
}
diff --git a/src/test/java/org/olat/core/commons/services/text/TextServiceTest.java b/src/test/java/org/olat/core/commons/services/text/TextServiceTest.java
new file mode 100644
index 00000000000..5817480152c
--- /dev/null
+++ b/src/test/java/org/olat/core/commons/services/text/TextServiceTest.java
@@ -0,0 +1,97 @@
+package org.olat.core.commons.services.text;
+
+import java.util.Locale;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.olat.core.logging.OLog;
+import org.olat.core.logging.Tracing;
+import org.olat.test.OlatTestCase;
+import org.springframework.beans.factory.annotation.Autowired;
+
+/**
+ *
+ * Initial date: 29 juin 2018<br>
+ * @author srosse, stephane.rosse@frentix.com, http://www.frentix.com
+ *
+ */
+public class TextServiceTest extends OlatTestCase {
+
+ private static final OLog log = Tracing.createLoggerFor(TextServiceTest.class);
+
+ @Autowired
+ private TextService textService;
+
+ @Test
+ public void detectLocale_french() {
+ Locale locale = textService.detectLocale("Bonjour, je parle français.");
+ Assert.assertNotNull(locale);
+ Assert.assertEquals("fr", locale.getLanguage());
+ }
+
+ @Test
+ public void detectLocale_english() {
+ Locale locale = textService.detectLocale("Hi, I speak British english.");
+ Assert.assertNotNull(locale);
+ Assert.assertEquals("en", locale.getLanguage());
+ }
+
+ @Test
+ public void concurrentDetection() throws InterruptedException {
+ int numOfThreads = 10;
+ CountDownLatch latch = new CountDownLatch(numOfThreads);
+ NGramThread[] threads = new NGramThread[numOfThreads];
+ for(int i=numOfThreads; i-->0; ) {
+ threads[i] = new NGramThread(textService, latch);
+ }
+
+ for(int i=numOfThreads; i-->0; ) {
+ threads[i].start();
+ }
+
+ latch.await(120, TimeUnit.SECONDS);
+
+ for(int i=numOfThreads; i-->0; ) {
+ if(threads[i].getException() != null) {
+ log.error("", threads[i].getException());
+ }
+ Assert.assertNull(threads[i].getException());
+ }
+ }
+
+ private static class NGramThread extends Thread {
+
+ private final TextService service;
+ private final CountDownLatch latch;
+
+ private Exception exception;
+
+ public NGramThread(TextService service, CountDownLatch latch) {
+ this.service = service;
+ this.latch = latch;
+ }
+
+ public Exception getException() {
+ return exception;
+ }
+
+ @Override
+ public void run() {
+ try {
+ Thread.sleep(100);
+ for(int i=0; i<2500; i++) {
+ Locale locale = service.detectLocale("Bonjour, je parle français.");
+ Assert.assertNotNull(locale);
+
+ }
+ } catch (Exception e) {
+ exception = e;
+ } finally {
+ latch.countDown();
+ }
+ }
+ }
+
+}
diff --git a/src/test/java/org/olat/test/AllTestsJunit4.java b/src/test/java/org/olat/test/AllTestsJunit4.java
index 8e0e2d27417..703fd0ca136 100644
--- a/src/test/java/org/olat/test/AllTestsJunit4.java
+++ b/src/test/java/org/olat/test/AllTestsJunit4.java
@@ -109,6 +109,7 @@ import org.junit.runners.Suite;
org.olat.core.commons.services.sms.manager.MessageLogDAOTest.class,
org.olat.core.commons.services.taskexecutor.PersistentTaskDAOTest.class,
org.olat.core.commons.services.taskexecutor.TaskExecutorManagerTest.class,
+ org.olat.core.commons.services.text.TextServiceTest.class,
org.olat.admin.user.delete.service.UserDeletionManagerTest.class,
org.olat.group.BusinessGroupManagedFlagsTest.class,
org.olat.group.test.BGRightManagerTest.class,
--
GitLab