|
|
|
|
@ -11,12 +11,12 @@
|
|
|
|
|
* modify it under the terms of the GNU Lesser General Public
|
|
|
|
|
* License as published by the Free Software Foundation; either
|
|
|
|
|
* version 2.1 of the License, or (at your option) any later version.
|
|
|
|
|
*
|
|
|
|
|
*
|
|
|
|
|
* This library is distributed in the hope that it will be useful,
|
|
|
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
|
|
|
* Lesser General Public License for more details.
|
|
|
|
|
*
|
|
|
|
|
*
|
|
|
|
|
* You should have received a copy of the GNU Lesser General Public License
|
|
|
|
|
* along with this program in the file lgpl21.txt
|
|
|
|
|
* If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
|
@ -34,21 +34,21 @@ import java.util.concurrent.atomic.AtomicLong;
|
|
|
|
|
|
|
|
|
|
import org.apache.solr.common.SolrInputDocument;
|
|
|
|
|
|
|
|
|
|
public class SolrChardingSelection {
|
|
|
|
|
|
|
|
|
|
public class SolrShardingSelection {
|
|
|
|
|
|
|
|
|
|
public final static Charset charsetUTF8;
|
|
|
|
|
static {
|
|
|
|
|
charsetUTF8 = Charset.forName("UTF-8");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private final Method method; // the charding method
|
|
|
|
|
private AtomicLong chardID; // the next id that shall be given away
|
|
|
|
|
|
|
|
|
|
private final Method method; // the sharding method
|
|
|
|
|
private final AtomicLong chardID; // the next id that shall be given away
|
|
|
|
|
private final int dimension; // the number of chards
|
|
|
|
|
public enum Method {
|
|
|
|
|
MODULO_HOST_MD5, ROUND_ROBIN;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public SolrChardingSelection(Method method, int dimension) {
|
|
|
|
|
|
|
|
|
|
public SolrShardingSelection(final Method method, final int dimension) {
|
|
|
|
|
this.method = method;
|
|
|
|
|
this.dimension = dimension;
|
|
|
|
|
this.chardID = new AtomicLong(0);
|
|
|
|
|
@ -57,36 +57,36 @@ public class SolrChardingSelection {
|
|
|
|
|
private int selectRoundRobin() {
|
|
|
|
|
return (int) (this.chardID.getAndIncrement() % this.dimension);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public int select(SolrInputDocument solrdoc) throws IOException {
|
|
|
|
|
|
|
|
|
|
public int select(final SolrInputDocument solrdoc) throws IOException {
|
|
|
|
|
if (this.method == Method.MODULO_HOST_MD5) {
|
|
|
|
|
String sku = (String) solrdoc.getField("sku").getValue();
|
|
|
|
|
final String sku = (String) solrdoc.getField("sku").getValue();
|
|
|
|
|
return selectURL(sku);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// finally if no method matches use ROUND_ROBIN
|
|
|
|
|
return selectRoundRobin();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public int selectURL(String sku) throws IOException {
|
|
|
|
|
|
|
|
|
|
public int selectURL(final String sku) throws IOException {
|
|
|
|
|
if (this.method == Method.MODULO_HOST_MD5) {
|
|
|
|
|
try {
|
|
|
|
|
URL url = new URL(sku);
|
|
|
|
|
String host = url.getHost();
|
|
|
|
|
if (host == null) throw new IOException("charding - bad url, host empty: " + sku);
|
|
|
|
|
final URL url = new URL(sku);
|
|
|
|
|
final String host = url.getHost();
|
|
|
|
|
if (host == null) throw new IOException("sharding - bad url, host empty: " + sku);
|
|
|
|
|
try {
|
|
|
|
|
MessageDigest digest = MessageDigest.getInstance("MD5");
|
|
|
|
|
final MessageDigest digest = MessageDigest.getInstance("MD5");
|
|
|
|
|
digest.update(host.getBytes(charsetUTF8));
|
|
|
|
|
byte[] md5 = digest.digest();
|
|
|
|
|
final byte[] md5 = digest.digest();
|
|
|
|
|
return (0xff & md5[0]) % this.dimension;
|
|
|
|
|
} catch (NoSuchAlgorithmException e) {
|
|
|
|
|
throw new IOException("charding - no md5 available: " + e.getMessage());
|
|
|
|
|
} catch (final NoSuchAlgorithmException e) {
|
|
|
|
|
throw new IOException("sharding - no md5 available: " + e.getMessage());
|
|
|
|
|
}
|
|
|
|
|
} catch (MalformedURLException e) {
|
|
|
|
|
throw new IOException("charding - bad url: " + sku);
|
|
|
|
|
} catch (final MalformedURLException e) {
|
|
|
|
|
throw new IOException("sharding - bad url: " + sku);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// finally if no method matches use ROUND_ROBIN
|
|
|
|
|
return (int) (this.chardID.getAndIncrement() % this.dimension);
|
|
|
|
|
}
|
orbiter
13 years ago