From 63762d8f895dee92595f50d8436634787dfceefe Mon Sep 17 00:00:00 2001 From: orbiter Date: Thu, 20 Sep 2012 19:38:22 +0200 Subject: [PATCH] removed kelondro dependencies from cora --- htroot/DictionaryLoader_p.java | 4 +- htroot/api/yacydoc.java | 2 +- htroot/yacysearch.java | 2 +- htroot/yacysearch_location.java | 2 +- source/de/anomic/data/DidYouMean.java | 2 +- source/net/yacy/cora/document/RSSMessage.java | 5 +-- .../yacy/{ => cora}/document/WordCache.java | 23 ++++++----- .../geolocation => cora/geo}/GeoLocation.java | 2 +- .../geolocation => cora/geo}/GeoPoint.java | 2 +- .../geo}/GeonamesLocation.java | 16 ++++---- .../geo}/IntegerGeoPoint.java | 24 ++++++++++- .../geolocation => cora/geo}/Locations.java | 2 +- .../geo}/OpenGeoDBLocation.java | 6 +-- .../geo}/OverarchingLocation.java | 2 +- source/net/yacy/cora/lod/JenaTripleStore.java | 41 ++++++++++--------- .../net/yacy/cora/lod/vocabulary/Tagging.java | 6 +-- source/net/yacy/cora/protocol/Domains.java | 25 +++++------ .../yacy/cora/sorting/OrderedScoreMap.java | 2 +- source/net/yacy/cora/storage/KeyList.java | 5 +++ .../util}/StringBuilderComparator.java | 4 +- source/net/yacy/document/Autotagging.java | 2 +- source/net/yacy/document/Condenser.java | 1 + source/net/yacy/document/LibraryProvider.java | 7 ++-- source/net/yacy/document/WordTokenizer.java | 1 + source/net/yacy/search/Switchboard.java | 9 +++- .../net/yacy/search/query/AccessTracker.java | 2 +- source/net/yacy/search/query/QueryParams.java | 2 +- 27 files changed, 122 insertions(+), 79 deletions(-) rename source/net/yacy/{ => cora}/document/WordCache.java (96%) rename source/net/yacy/{document/geolocation => cora/geo}/GeoLocation.java (98%) rename source/net/yacy/{document/geolocation => cora/geo}/GeoPoint.java (98%) rename source/net/yacy/{document/geolocation => cora/geo}/GeonamesLocation.java (97%) rename source/net/yacy/{document/geolocation => cora/geo}/IntegerGeoPoint.java (73%) rename source/net/yacy/{document/geolocation => cora/geo}/Locations.java (98%) rename source/net/yacy/{document/geolocation => cora/geo}/OpenGeoDBLocation.java (96%) rename source/net/yacy/{document/geolocation => cora/geo}/OverarchingLocation.java (99%) rename source/net/yacy/{document => cora/util}/StringBuilderComparator.java (98%) diff --git a/htroot/DictionaryLoader_p.java b/htroot/DictionaryLoader_p.java index d14f0efdd..d27b356b9 100644 --- a/htroot/DictionaryLoader_p.java +++ b/htroot/DictionaryLoader_p.java @@ -21,11 +21,11 @@ import java.io.IOException; import java.net.MalformedURLException; +import net.yacy.cora.geo.GeonamesLocation; +import net.yacy.cora.geo.OpenGeoDBLocation; import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.services.federated.yacy.CacheStrategy; import net.yacy.document.LibraryProvider; -import net.yacy.document.geolocation.GeonamesLocation; -import net.yacy.document.geolocation.OpenGeoDBLocation; import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.util.FileUtils; diff --git a/htroot/api/yacydoc.java b/htroot/api/yacydoc.java index 03ab6523a..35c7f3e4c 100644 --- a/htroot/api/yacydoc.java +++ b/htroot/api/yacydoc.java @@ -146,7 +146,7 @@ public class yacydoc { references += r.toString()+","; } - JenaTripleStore.log.logInfo(references); + Log.logInfo("yacydoc", references); prop.put("taglinks", references); diff --git a/htroot/yacysearch.java b/htroot/yacysearch.java index e0dc5f421..5178dcfd1 100644 --- a/htroot/yacysearch.java +++ b/htroot/yacysearch.java @@ -45,6 +45,7 @@ import net.yacy.cora.document.Classification; import net.yacy.cora.document.Classification.ContentDomain; import net.yacy.cora.document.RSSMessage; import net.yacy.cora.document.UTF8; +import net.yacy.cora.geo.GeoLocation; import net.yacy.cora.lod.vocabulary.Tagging; import net.yacy.cora.protocol.Domains; import net.yacy.cora.protocol.HeaderFramework; @@ -55,7 +56,6 @@ import net.yacy.document.Condenser; import net.yacy.document.Document; import net.yacy.document.LibraryProvider; import net.yacy.document.Parser; -import net.yacy.document.geolocation.GeoLocation; import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.data.meta.URIMetadata; import net.yacy.kelondro.data.word.Word; diff --git a/htroot/yacysearch_location.java b/htroot/yacysearch_location.java index b2d702bdb..26670ed16 100644 --- a/htroot/yacysearch_location.java +++ b/htroot/yacysearch_location.java @@ -24,12 +24,12 @@ import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; import net.yacy.cora.document.RSSMessage; +import net.yacy.cora.geo.GeoLocation; import net.yacy.cora.protocol.Domains; import net.yacy.cora.protocol.HeaderFramework; import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.services.federated.opensearch.SRURSSConnector; import net.yacy.document.LibraryProvider; -import net.yacy.document.geolocation.GeoLocation; import net.yacy.search.Switchboard; import net.yacy.search.SwitchboardConstants; import de.anomic.server.serverCore; diff --git a/source/de/anomic/data/DidYouMean.java b/source/de/anomic/data/DidYouMean.java index fa8d96782..24a63cbed 100644 --- a/source/de/anomic/data/DidYouMean.java +++ b/source/de/anomic/data/DidYouMean.java @@ -10,8 +10,8 @@ import java.util.concurrent.LinkedBlockingQueue; import net.yacy.cora.sorting.ClusteredScoreMap; import net.yacy.cora.sorting.ReversibleScoreMap; +import net.yacy.cora.util.StringBuilderComparator; import net.yacy.document.LibraryProvider; -import net.yacy.document.StringBuilderComparator; import net.yacy.kelondro.logging.Log; import net.yacy.search.index.Segment; diff --git a/source/net/yacy/cora/document/RSSMessage.java b/source/net/yacy/cora/document/RSSMessage.java index 7df30535b..f8c70491a 100644 --- a/source/net/yacy/cora/document/RSSMessage.java +++ b/source/net/yacy/cora/document/RSSMessage.java @@ -38,7 +38,6 @@ import net.yacy.cora.date.ISO8601Formatter; import net.yacy.cora.lod.vocabulary.DublinCore; import net.yacy.cora.lod.vocabulary.Geo; import net.yacy.cora.protocol.HeaderFramework; -import net.yacy.kelondro.data.meta.DigestURI; public class RSSMessage implements Hit, Comparable, Comparator { @@ -110,13 +109,13 @@ public class RSSMessage implements Hit, Comparable, Comparator(); this.map.put("title", title); this.map.put("description", description); this.map.put("link", link.toNormalform(true, false)); this.map.put("pubDate", ISO8601Formatter.FORMATTER.format()); - this.map.put("guid", ASCII.String(link.hash())); + this.map.put("guid", guid); } public RSSMessage() { diff --git a/source/net/yacy/document/WordCache.java b/source/net/yacy/cora/document/WordCache.java similarity index 96% rename from source/net/yacy/document/WordCache.java rename to source/net/yacy/cora/document/WordCache.java index 491146190..84726f36a 100644 --- a/source/net/yacy/document/WordCache.java +++ b/source/net/yacy/cora/document/WordCache.java @@ -18,7 +18,7 @@ * If not, see . */ -package net.yacy.document; +package net.yacy.cora.document; import java.io.BufferedReader; import java.io.File; @@ -37,9 +37,10 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicInteger; import java.util.zip.GZIPInputStream; +import org.apache.log4j.Logger; + import net.yacy.cora.sorting.OrderedScoreMap; -import net.yacy.kelondro.logging.Log; -import net.yacy.kelondro.util.MemoryControl; +import net.yacy.cora.util.StringBuilderComparator; /** * provide a completion library for the did-you-mean class @@ -47,13 +48,12 @@ import net.yacy.kelondro.util.MemoryControl; */ public class WordCache { + private final static Logger log = Logger.getLogger(WordCache.class); + // common word cache - private static final int commonWordsMaxSize = (int) (MemoryControl.available() / 30000); // maximum size of common word cache + private static final int commonWordsMaxSize = 20000; // maximum size of common word cache private static final int commonWordsMinLength = 5; // words must have that length at minimum private static OrderedScoreMap commonWords = new OrderedScoreMap(StringBuilderComparator.CASE_INSENSITIVE_ORDER); - static { - Log.logConfig("WordCache", "commonWordsMaxSize = " + commonWordsMaxSize); - } // dictionaries private final File dictionaryPath; @@ -213,9 +213,6 @@ public class WordCache { if (word.length() < commonWordsMinLength) { return; } - if (MemoryControl.shortStatus()) { - commonWords.clear(); - } commonWords.inc(word); if (!(commonWords.sizeSmaller(commonWordsMaxSize))) { commonWords.shrinkToMaxSize(commonWordsMaxSize / 2); @@ -242,7 +239,7 @@ public class WordCache { Dictionary dict = new Dictionary(new File(this.dictionaryPath, f)); this.dictionaries.put(f.substring(0, f.length() - 6), dict); } catch (final IOException e) { - Log.logException(e); + log.warn(e); } } } @@ -323,6 +320,10 @@ public class WordCache { } return size; } + + public static void clear() { + commonWords.clear(); + } /** * a property that is used during the construction of recommendation: diff --git a/source/net/yacy/document/geolocation/GeoLocation.java b/source/net/yacy/cora/geo/GeoLocation.java similarity index 98% rename from source/net/yacy/document/geolocation/GeoLocation.java rename to source/net/yacy/cora/geo/GeoLocation.java index 3940b5d90..45cb68be5 100644 --- a/source/net/yacy/document/geolocation/GeoLocation.java +++ b/source/net/yacy/cora/geo/GeoLocation.java @@ -20,7 +20,7 @@ * If not, see . */ -package net.yacy.document.geolocation; +package net.yacy.cora.geo; import java.util.Comparator; diff --git a/source/net/yacy/document/geolocation/GeoPoint.java b/source/net/yacy/cora/geo/GeoPoint.java similarity index 98% rename from source/net/yacy/document/geolocation/GeoPoint.java rename to source/net/yacy/cora/geo/GeoPoint.java index f396beaea..5f2d59f16 100644 --- a/source/net/yacy/document/geolocation/GeoPoint.java +++ b/source/net/yacy/cora/geo/GeoPoint.java @@ -20,7 +20,7 @@ * If not, see . */ -package net.yacy.document.geolocation; +package net.yacy.cora.geo; /** * Geolocation storage may vary using different data structures for the points. diff --git a/source/net/yacy/document/geolocation/GeonamesLocation.java b/source/net/yacy/cora/geo/GeonamesLocation.java similarity index 97% rename from source/net/yacy/document/geolocation/GeonamesLocation.java rename to source/net/yacy/cora/geo/GeonamesLocation.java index 0f25639d2..d77b7ee3e 100644 --- a/source/net/yacy/document/geolocation/GeonamesLocation.java +++ b/source/net/yacy/cora/geo/GeonamesLocation.java @@ -20,7 +20,7 @@ * If not, see . */ -package net.yacy.document.geolocation; +package net.yacy.cora.geo; import java.io.BufferedReader; import java.io.File; @@ -39,9 +39,10 @@ import java.util.TreeSet; import java.util.zip.ZipEntry; import java.util.zip.ZipFile; -import net.yacy.document.StringBuilderComparator; -import net.yacy.document.WordCache; -import net.yacy.kelondro.logging.Log; +import org.apache.log4j.Logger; + +import net.yacy.cora.document.WordCache; +import net.yacy.cora.util.StringBuilderComparator; public class GeonamesLocation implements Locations { @@ -68,7 +69,8 @@ public class GeonamesLocation implements Locations { timezone : the timezone id (see file timeZone.txt) modification date : date of last modification in yyyy-MM-dd format */ - + private final static Logger log = Logger.getLogger(GeonamesLocation.class); + private final Map id2loc; private final TreeMap> name2ids; private final File file; @@ -92,7 +94,7 @@ public class GeonamesLocation implements Locations { final InputStream is = zf.getInputStream(ze); reader = new BufferedReader(new InputStreamReader(is, "UTF-8")); } catch ( final IOException e ) { - Log.logException(e); + log.warn(e); return; } @@ -153,7 +155,7 @@ public class GeonamesLocation implements Locations { } } } catch ( final IOException e ) { - Log.logException(e); + log.warn(e); } } diff --git a/source/net/yacy/document/geolocation/IntegerGeoPoint.java b/source/net/yacy/cora/geo/IntegerGeoPoint.java similarity index 73% rename from source/net/yacy/document/geolocation/IntegerGeoPoint.java rename to source/net/yacy/cora/geo/IntegerGeoPoint.java index 34151d142..f978fccae 100644 --- a/source/net/yacy/document/geolocation/IntegerGeoPoint.java +++ b/source/net/yacy/cora/geo/IntegerGeoPoint.java @@ -1,4 +1,26 @@ -package net.yacy.document.geolocation; +/** + * IntegerGeoPoint + * Copyright 2009 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany + * first published 08.10.2009 on http://yacy.net + * + * This file is part of YaCy Content Integration + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program in the file lgpl21.txt + * If not, see . + */ + +package net.yacy.cora.geo; /** * GeoPoint implementation with Integer accuracy diff --git a/source/net/yacy/document/geolocation/Locations.java b/source/net/yacy/cora/geo/Locations.java similarity index 98% rename from source/net/yacy/document/geolocation/Locations.java rename to source/net/yacy/cora/geo/Locations.java index 919c9bd5f..4d16df597 100644 --- a/source/net/yacy/document/geolocation/Locations.java +++ b/source/net/yacy/cora/geo/Locations.java @@ -21,7 +21,7 @@ */ -package net.yacy.document.geolocation; +package net.yacy.cora.geo; import java.util.Set; import java.util.TreeSet; diff --git a/source/net/yacy/document/geolocation/OpenGeoDBLocation.java b/source/net/yacy/cora/geo/OpenGeoDBLocation.java similarity index 96% rename from source/net/yacy/document/geolocation/OpenGeoDBLocation.java rename to source/net/yacy/cora/geo/OpenGeoDBLocation.java index 844f9091f..92f820646 100644 --- a/source/net/yacy/document/geolocation/OpenGeoDBLocation.java +++ b/source/net/yacy/cora/geo/OpenGeoDBLocation.java @@ -20,7 +20,7 @@ * If not, see . */ -package net.yacy.document.geolocation; +package net.yacy.cora.geo; import java.io.BufferedReader; import java.io.File; @@ -39,8 +39,8 @@ import java.util.TreeMap; import java.util.TreeSet; import java.util.zip.GZIPInputStream; -import net.yacy.document.StringBuilderComparator; -import net.yacy.document.WordCache; +import net.yacy.cora.document.WordCache; +import net.yacy.cora.util.StringBuilderComparator; import net.yacy.kelondro.logging.Log; /** diff --git a/source/net/yacy/document/geolocation/OverarchingLocation.java b/source/net/yacy/cora/geo/OverarchingLocation.java similarity index 99% rename from source/net/yacy/document/geolocation/OverarchingLocation.java rename to source/net/yacy/cora/geo/OverarchingLocation.java index ce400cccf..c06064ff9 100644 --- a/source/net/yacy/document/geolocation/OverarchingLocation.java +++ b/source/net/yacy/cora/geo/OverarchingLocation.java @@ -20,7 +20,7 @@ * If not, see . */ -package net.yacy.document.geolocation; +package net.yacy.cora.geo; import java.util.HashMap; import java.util.HashSet; diff --git a/source/net/yacy/cora/lod/JenaTripleStore.java b/source/net/yacy/cora/lod/JenaTripleStore.java index c7e7e6222..4e727232c 100644 --- a/source/net/yacy/cora/lod/JenaTripleStore.java +++ b/source/net/yacy/cora/lod/JenaTripleStore.java @@ -20,9 +20,11 @@ import net.yacy.cora.lod.vocabulary.Geo; import net.yacy.cora.lod.vocabulary.HttpHeader; import net.yacy.cora.lod.vocabulary.Tagging; import net.yacy.cora.lod.vocabulary.YaCyMetadata; -import net.yacy.kelondro.logging.Log; import net.yacy.search.Switchboard; +import org.apache.log4j.Logger; +import org.mortbay.log.Log; + import com.hp.hpl.jena.rdf.model.Model; import com.hp.hpl.jena.rdf.model.ModelFactory; import com.hp.hpl.jena.rdf.model.Property; @@ -37,7 +39,8 @@ import com.hp.hpl.jena.util.FileManager; public class JenaTripleStore { - public static Log log = new Log("TRIPLESTORE"); + private final static Logger log = Logger.getLogger(JenaTripleStore.class); + public static Model model = ModelFactory.createDefaultModel(); static { init(model); @@ -67,12 +70,12 @@ public class JenaTripleStore { public static void loadRDF(String fileNameOrUri) throws IOException { Model tmp = ModelFactory.createDefaultModel(); - log.logInfo("Loading from " + fileNameOrUri); + log.info("Loading from " + fileNameOrUri); InputStream is = FileManager.get().open(fileNameOrUri); if (is != null) { // read the RDF/XML file tmp.read(is, null); - log.logInfo("loaded " + tmp.size() + " triples from " + fileNameOrUri); + log.info("loaded " + tmp.size() + " triples from " + fileNameOrUri); model = model.union(tmp); } else { throw new IOException("cannot read " + fileNameOrUri); @@ -80,7 +83,7 @@ public class JenaTripleStore { } public static void LoadNTriples(String fileNameOrUri) throws IOException { - log.logInfo("Loading N-Triples from " + fileNameOrUri); + log.info("Loading N-Triples from " + fileNameOrUri); InputStream is = FileManager.get().open(fileNameOrUri); LoadNTriples(is); } @@ -89,7 +92,7 @@ public class JenaTripleStore { Model tmp = ModelFactory.createDefaultModel(); if (is != null) { tmp.read(is, null, "N-TRIPLE"); - log.logInfo("loaded " + tmp.size() + " triples"); + log.info("loaded " + tmp.size() + " triples"); model = model.union(tmp); //model.write(System.out, "TURTLE"); } else { @@ -118,10 +121,10 @@ public class JenaTripleStore { File ftmp = new File(filename + "." + System.currentTimeMillis()); if (model.isEmpty() && !f.exists()) { // we don't store zero-size models if they did not exist before - log.logInfo("NOT saving triplestore with " + model.size() + " triples to " + filename); + log.info("NOT saving triplestore with " + model.size() + " triples to " + filename); return; } - log.logInfo("Saving triplestore with " + model.size() + " triples to " + filename); + log.info("Saving triplestore with " + model.size() + " triples to " + filename); OutputStream fout; try { fout = new BufferedOutputStream(new FileOutputStream(ftmp)); @@ -133,9 +136,9 @@ public class JenaTripleStore { if (!f.exists()) { ftmp.renameTo(f); } - log.logInfo("Saved triplestore with " + model.size() + " triples to " + filename); + log.info("Saved triplestore with " + model.size() + " triples to " + filename); } catch (Exception e) { - log.logWarning("Saving to " + filename+" failed"); + log.warn("Saving to " + filename+" failed"); } } @@ -176,14 +179,14 @@ public class JenaTripleStore { Resource r = model.getResource(subject); Property pr = model.getProperty(predicate); r.addProperty(pr, object); - log.logInfo("ADD " + subject + " - " + predicate + " - " + object); + log.info("ADD " + subject + " - " + predicate + " - " + object); } public static String getObject(final String subject, final String predicate) { Iterator ni = JenaTripleStore.getObjects(subject, predicate); String object = ""; if (ni.hasNext()) object = ni.next().toString(); - log.logInfo("GET " + subject + " - " + predicate + " - " + object); + log.info("GET " + subject + " - " + predicate + " - " + object); return object; } @@ -196,7 +199,7 @@ public class JenaTripleStore { Iterator ni = JenaTripleStore.getPrivateObjects(subject, predicate, username); String object = ""; if (ni.hasNext()) object = ni.next().toString(); - log.logInfo("GET (" + username + ") " + subject + " - " + predicate + " - " + object); + log.info("GET (" + username + ") " + subject + " - " + predicate + " - " + object); return object; } @@ -271,7 +274,7 @@ public class JenaTripleStore { public static void initPrivateStores() { Switchboard switchboard = Switchboard.getSwitchboard(); - log.logInfo("Init private stores"); + log.info("Init private stores"); if (privatestorage == null) privatestorage = new ConcurrentHashMap(); if (privatestorage != null) privatestorage.clear(); @@ -282,17 +285,17 @@ public class JenaTripleStore { String username = e.getUserName(); File triplestore = new File(switchboard.getConfig("triplestore", new File(switchboard.getDataPath(), "DATA/TRIPLESTORE").getAbsolutePath())); File currentuserfile = new File(triplestore, "private_store_"+username+".rdf"); - log.logInfo("Init " + username + " from "+currentuserfile.getAbsolutePath()); + log.info("Init " + username + " from "+currentuserfile.getAbsolutePath()); Model tmp = ModelFactory.createDefaultModel(); init (tmp); if (currentuserfile.exists()) { - log.logInfo("Loading from " + currentuserfile.getAbsolutePath()); + log.info("Loading from " + currentuserfile.getAbsolutePath()); InputStream is = FileManager.get().open(currentuserfile.getAbsolutePath()); if (is != null) { // read the RDF/XML file tmp.read(is, null); - log.logInfo("loaded " + tmp.size() + " triples from " + currentuserfile.getAbsolutePath()); + log.info("loaded " + tmp.size() + " triples from " + currentuserfile.getAbsolutePath()); } else { throw new IOException("cannot read " + currentuserfile.getAbsolutePath()); } @@ -303,13 +306,13 @@ public class JenaTripleStore { } } } catch (Exception anyex) { - Log.logException(anyex); + Log.warn(anyex); } } public static void savePrivateStores() { Switchboard switchboard = Switchboard.getSwitchboard(); - log.logInfo("Saving user triplestores"); + log.info("Saving user triplestores"); if (privatestorage == null) return; for (Entry s : privatestorage.entrySet()) { File triplestore = new File(switchboard.getConfig("triplestore", new File(switchboard.getDataPath(), "DATA/TRIPLESTORE").getAbsolutePath())); diff --git a/source/net/yacy/cora/lod/vocabulary/Tagging.java b/source/net/yacy/cora/lod/vocabulary/Tagging.java index c9acb0878..d3ab280e7 100644 --- a/source/net/yacy/cora/lod/vocabulary/Tagging.java +++ b/source/net/yacy/cora/lod/vocabulary/Tagging.java @@ -34,10 +34,10 @@ import java.util.concurrent.BlockingQueue; import java.util.concurrent.ConcurrentHashMap; import java.util.regex.Pattern; +import net.yacy.cora.document.WordCache.Dictionary; +import net.yacy.cora.geo.GeoLocation; +import net.yacy.cora.geo.Locations; import net.yacy.cora.storage.Files; -import net.yacy.document.WordCache.Dictionary; -import net.yacy.document.geolocation.GeoLocation; -import net.yacy.document.geolocation.Locations; public class Tagging { diff --git a/source/net/yacy/cora/protocol/Domains.java b/source/net/yacy/cora/protocol/Domains.java index 13ddf758b..224157e29 100644 --- a/source/net/yacy/cora/protocol/Domains.java +++ b/source/net/yacy/cora/protocol/Domains.java @@ -50,19 +50,21 @@ import java.util.concurrent.TimeUnit; import java.util.regex.Pattern; import java.util.regex.PatternSyntaxException; +import org.apache.log4j.Logger; + import net.yacy.cora.plugin.ClassProvider; import net.yacy.cora.storage.ARC; import net.yacy.cora.storage.ConcurrentARC; import net.yacy.cora.storage.KeyList; -import net.yacy.kelondro.logging.Log; -import net.yacy.kelondro.util.MemoryControl; import com.google.common.net.InetAddresses; import com.google.common.util.concurrent.SimpleTimeLimiter; import com.google.common.util.concurrent.TimeLimiter; public class Domains { - + + private final static Logger log = Logger.getLogger(Domains.class); + public static final String LOCALHOST = "127.0.0.1"; // replace with IPv6 0:0:0:0:0:0:0:1 ? private static String LOCALHOST_NAME = LOCALHOST; // this will be replaced with the actual name of the local host @@ -602,7 +604,7 @@ public class Domains { globalHosts = null; } else try { globalHosts = new KeyList(globalHostsnameCache); - Log.logInfo("Domains", "loaded globalHosts cache of hostnames, size = " + globalHosts.size()); + log.info("loaded globalHosts cache of hostnames, size = " + globalHosts.size()); } catch (final IOException e) { globalHosts = null; } @@ -621,7 +623,7 @@ public class Domains { } public static synchronized void close() { - if (globalHosts != null) try {globalHosts.close();} catch (final IOException e) {Log.logException(e);} + if (globalHosts != null) try {globalHosts.close();} catch (final IOException e) {log.warn(e);} } /** @@ -769,7 +771,7 @@ public class Domains { if (InetAddresses.isInetAddress(host)) { try { ip = InetAddresses.forString(host); - Log.logInfo("Domains", "using guava for host resolution:" + host); + log.info("using guava for host resolution:" + host); } catch (IllegalArgumentException e) { ip = null; } @@ -811,12 +813,7 @@ public class Domains { localHostNames.add(host); } else { if (globalHosts != null) try { - if (MemoryControl.shortStatus()) { - globalHosts.close(); - globalHosts = null; - } else { - globalHosts.add(host); - } + globalHosts.add(host); } catch (final IOException e) {} } } @@ -825,6 +822,10 @@ public class Domains { } } + public static void clear() { + try {globalHosts.clear();} catch (IOException e) {} + } + private final static Pattern dotPattern = Pattern.compile("\\."); public static final InetAddress parseInetAddress(String ip) { diff --git a/source/net/yacy/cora/sorting/OrderedScoreMap.java b/source/net/yacy/cora/sorting/OrderedScoreMap.java index 93a13489a..292080b59 100644 --- a/source/net/yacy/cora/sorting/OrderedScoreMap.java +++ b/source/net/yacy/cora/sorting/OrderedScoreMap.java @@ -38,7 +38,7 @@ import java.util.TreeMap; import java.util.TreeSet; import java.util.concurrent.atomic.AtomicInteger; -import net.yacy.document.StringBuilderComparator; +import net.yacy.cora.util.StringBuilderComparator; public class OrderedScoreMap extends AbstractScoreMap implements ScoreMap { diff --git a/source/net/yacy/cora/storage/KeyList.java b/source/net/yacy/cora/storage/KeyList.java index 5d95b2800..c259e4e0b 100644 --- a/source/net/yacy/cora/storage/KeyList.java +++ b/source/net/yacy/cora/storage/KeyList.java @@ -75,6 +75,11 @@ public class KeyList implements Iterable { this.raf = new RandomAccessFile(file, "rw"); } + + public void clear() throws IOException { + this.raf.setLength(0); + this.keys.clear(); + } public int size() { return this.keys.size(); diff --git a/source/net/yacy/document/StringBuilderComparator.java b/source/net/yacy/cora/util/StringBuilderComparator.java similarity index 98% rename from source/net/yacy/document/StringBuilderComparator.java rename to source/net/yacy/cora/util/StringBuilderComparator.java index 6eea342c6..430506c1f 100644 --- a/source/net/yacy/document/StringBuilderComparator.java +++ b/source/net/yacy/cora/util/StringBuilderComparator.java @@ -1,5 +1,5 @@ /** - * CaseInsensitiveStringBuilderComparator.java + * StringBuilderComparator.java * Copyright 2011 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany * First released 09.11.2011 at http://yacy.net * @@ -18,7 +18,7 @@ * If not, see . */ -package net.yacy.document; +package net.yacy.cora.util; import java.util.ArrayList; import java.util.Comparator; diff --git a/source/net/yacy/document/Autotagging.java b/source/net/yacy/document/Autotagging.java index 4930ba8ed..2d2e19d8a 100644 --- a/source/net/yacy/document/Autotagging.java +++ b/source/net/yacy/document/Autotagging.java @@ -27,8 +27,8 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import net.yacy.cora.geo.Locations; import net.yacy.cora.lod.vocabulary.Tagging; -import net.yacy.document.geolocation.Locations; import net.yacy.kelondro.logging.Log; /** diff --git a/source/net/yacy/document/Condenser.java b/source/net/yacy/document/Condenser.java index 2ebe25eb2..4f83bc2dd 100644 --- a/source/net/yacy/document/Condenser.java +++ b/source/net/yacy/document/Condenser.java @@ -35,6 +35,7 @@ import java.util.SortedSet; import java.util.TreeMap; import net.yacy.cora.document.ASCII; +import net.yacy.cora.document.WordCache; import net.yacy.cora.document.Classification.ContentDomain; import net.yacy.cora.document.MultiProtocolURI; import net.yacy.cora.lod.vocabulary.Tagging; diff --git a/source/net/yacy/document/LibraryProvider.java b/source/net/yacy/document/LibraryProvider.java index ee5830126..368f186e3 100644 --- a/source/net/yacy/document/LibraryProvider.java +++ b/source/net/yacy/document/LibraryProvider.java @@ -43,13 +43,14 @@ import java.util.zip.ZipException; import java.util.zip.ZipFile; import net.yacy.cora.document.MultiProtocolURI; +import net.yacy.cora.document.WordCache; +import net.yacy.cora.geo.GeonamesLocation; +import net.yacy.cora.geo.OpenGeoDBLocation; +import net.yacy.cora.geo.OverarchingLocation; import net.yacy.cora.lod.JenaTripleStore; import net.yacy.cora.lod.vocabulary.Tagging; import net.yacy.cora.lod.vocabulary.Tagging.SOTuple; import net.yacy.cora.storage.Files; -import net.yacy.document.geolocation.GeonamesLocation; -import net.yacy.document.geolocation.OpenGeoDBLocation; -import net.yacy.document.geolocation.OverarchingLocation; import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.util.FileUtils; diff --git a/source/net/yacy/document/WordTokenizer.java b/source/net/yacy/document/WordTokenizer.java index 1644a0046..664e9fa37 100644 --- a/source/net/yacy/document/WordTokenizer.java +++ b/source/net/yacy/document/WordTokenizer.java @@ -30,6 +30,7 @@ import java.util.List; import java.util.SortedMap; import java.util.TreeMap; +import net.yacy.cora.document.WordCache; import net.yacy.kelondro.data.word.Word; import net.yacy.kelondro.order.Base64Order; diff --git a/source/net/yacy/search/Switchboard.java b/source/net/yacy/search/Switchboard.java index 7bd1c994f..54d899881 100644 --- a/source/net/yacy/search/Switchboard.java +++ b/source/net/yacy/search/Switchboard.java @@ -84,6 +84,7 @@ import net.yacy.cora.document.RSSFeed; import net.yacy.cora.document.RSSMessage; import net.yacy.cora.document.RSSReader; import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.WordCache; import net.yacy.cora.lod.JenaTripleStore; import net.yacy.cora.protocol.ClientIdentification; import net.yacy.cora.protocol.ConnectionInfo; @@ -1914,6 +1915,12 @@ public final class Switchboard extends serverSwitch } public boolean cleanupJob() { + + if (MemoryControl.shortStatus()) { + WordCache.clear(); + Domains.clear(); + } + try { // flush the document compressor cache Cache.commit(); @@ -2556,7 +2563,7 @@ public final class Switchboard extends serverSwitch ASCII.getBytes(this.peers.mySeed().hash)) ? EventChannel.LOCALINDEXING : EventChannel.REMOTEINDEXING); - feed.addMessage(new RSSMessage("Indexed web page", dc_title, queueEntry.url())); + feed.addMessage(new RSSMessage("Indexed web page", dc_title, queueEntry.url(), ASCII.String(queueEntry.url().hash()))); } catch ( final IOException e ) { //if (this.log.isFine()) log.logFine("Not Indexed Resource '" + queueEntry.url().toNormalform(false, true) + "': process case=" + processCase); addURLtoErrorDB( diff --git a/source/net/yacy/search/query/AccessTracker.java b/source/net/yacy/search/query/AccessTracker.java index 286451d9f..2799aed29 100644 --- a/source/net/yacy/search/query/AccessTracker.java +++ b/source/net/yacy/search/query/AccessTracker.java @@ -36,7 +36,7 @@ import java.util.LinkedList; import net.yacy.cora.date.GenericFormatter; import net.yacy.cora.document.UTF8; -import net.yacy.document.WordCache; +import net.yacy.cora.document.WordCache; import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.util.MemoryControl; diff --git a/source/net/yacy/search/query/QueryParams.java b/source/net/yacy/search/query/QueryParams.java index 3e4664e00..abb995558 100644 --- a/source/net/yacy/search/query/QueryParams.java +++ b/source/net/yacy/search/query/QueryParams.java @@ -43,12 +43,12 @@ import net.yacy.cora.document.Classification; import net.yacy.cora.document.Classification.ContentDomain; import net.yacy.cora.document.MultiProtocolURI; import net.yacy.cora.document.UTF8; +import net.yacy.cora.geo.GeoLocation; import net.yacy.cora.lod.vocabulary.Tagging; import net.yacy.cora.services.federated.yacy.CacheStrategy; import net.yacy.cora.storage.HandleSet; import net.yacy.cora.util.SpaceExceededException; import net.yacy.document.Condenser; -import net.yacy.document.geolocation.GeoLocation; import net.yacy.document.parser.html.AbstractScraper; import net.yacy.document.parser.html.CharacterCoding; import net.yacy.kelondro.data.meta.DigestURI;