From ea633a794cb25fb03030dd4f535c59680ec5e2ac Mon Sep 17 00:00:00 2001
From: reger <reger18@arcor.de>
Date: Sat, 29 Nov 2014 22:13:24 +0100
Subject: [PATCH] including small junit test case for WordTokenizer

---
 test/net/yacy/document/WordTokenizerTest.java | 33 +++++++++++++++++++
 1 file changed, 33 insertions(+)
 create mode 100644 test/net/yacy/document/WordTokenizerTest.java

diff --git a/test/net/yacy/document/WordTokenizerTest.java b/test/net/yacy/document/WordTokenizerTest.java
new file mode 100644
index 000000000..c32e71ead
--- /dev/null
+++ b/test/net/yacy/document/WordTokenizerTest.java
@@ -0,0 +1,33 @@
+package net.yacy.document;
+
+import org.junit.Test;
+import static org.junit.Assert.*;
+
+public class WordTokenizerTest {
+
+    /**
+     * Test of nextElement method, of class WordTokenizer.
+     */
+    @Test
+    public void testNextElement() {
+        // test sentences containing 10x the word "word"
+        String[] testTxtArr = new String[]{
+            "  word word..... (word) [word] . 'word word' \"word word\" word ?  word! ",
+            "word-word word . word.word@word.word ....word... word,word "
+        };
+        
+        for (String testTxt : testTxtArr) {
+            SentenceReader sr = new SentenceReader(testTxt);
+            WordTokenizer wt = new WordTokenizer(sr, null);
+            int cnt = 0;
+            while (wt.hasMoreElements()) {
+                StringBuilder sb = wt.nextElement();
+                assertEquals("word", sb.toString());
+                cnt++;
+            }
+            wt.close();
+            assertEquals(10, cnt);
+        }
+    }
+
+}