From f40efb39af852f40821290f7425aedf32527f878 Mon Sep 17 00:00:00 2001
From: Marek Otahal <markotahal@gmail.com>
Date: Fri, 6 Jan 2012 02:01:20 +0100
Subject: [PATCH] Blacklist loadList() remove duplicates by using Set

Signed-off-by: Marek Otahal <markotahal@gmail.com>
---
 source/net/yacy/repository/Blacklist.java | 10 ++++++++--
 source/net/yacy/search/index/Segment.java |  4 ++--
 2 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/source/net/yacy/repository/Blacklist.java b/source/net/yacy/repository/Blacklist.java
index 76098c2e6..32b4f30c2 100644
--- a/source/net/yacy/repository/Blacklist.java
+++ b/source/net/yacy/repository/Blacklist.java
@@ -181,6 +181,12 @@ public class Blacklist {
         }
     }
 
+    /**
+     * create a blacklist from file, entries separated by 'sep'
+     * duplicit entries are removed
+     * @param blFile
+     * @param sep 
+     */
     private void loadList(final BlacklistFile blFile, final String sep) {
         final Map<String, List<String>> blacklistMapMatch = getBlacklistMap(blFile.getType(), true);
         final Map<String, List<String>> blacklistMapNotMatch = getBlacklistMap(blFile.getType(), false);
@@ -213,8 +219,8 @@ public class Blacklist {
                         blacklistMapNotMatch.put(loadedEntry.getKey(), loadedPaths);
                     }
                 } else {
-                    // TODO check for duplicates? (refactor List -> Set)
-                    paths.addAll(loadedPaths);
+                    // check for duplicates? (refactor List -> Set)
+                    paths.addAll(new HashSet<String>(loadedPaths));
                 }
             }
         }
diff --git a/source/net/yacy/search/index/Segment.java b/source/net/yacy/search/index/Segment.java
index e63119cad..20627524e 100644
--- a/source/net/yacy/search/index/Segment.java
+++ b/source/net/yacy/search/index/Segment.java
@@ -77,9 +77,9 @@ public class Segment {
     public static final int  writeBufferSize = 4 * 1024 * 1024;
 
     // the reference factory
-    public static final ReferenceFactory<WordReference> wordReferenceFactory = new WordReferenceFactory();
+    public static final ReferenceFactory<WordReference> wordReferenceFactory = new WordReferenceFactory();
     public static final ReferenceFactory<NavigationReference> navigationReferenceFactory = new NavigationReferenceFactory();
-    public static final ByteOrder wordOrder = Base64Order.enhancedCoder;
+    public static final ByteOrder wordOrder = Base64Order.enhancedCoder;
 
     private   final Log                            log;
     protected final IndexCell<WordReference>       termIndex;