boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

*) changes for better code readability

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6797 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
low012 15 years ago
parent dc93cec3a8
commit 2bc459252e
19 changed files with 1212 additions and 1161 deletions
Show Stats Download Patch File Download Diff File

15
htroot/Blog.java
Unescape View File

@ -32,7 +32,6 @@
import java.io.UnsupportedEncodingException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
@ -49,6 +48,8 @@ import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.yacy.yacyNewsPool;
import de.anomic.yacy.yacyNewsRecord;
import java.util.List;
import java.util.Map;
public class Blog {
@ -127,9 +128,8 @@ public class Blog {
if(hasRights && post.containsKey("delete") && post.get("delete").equals("sure")) {
page = sb.blogDB.readBlogEntry(pagename);
final Iterator<String> i = page.getComments().iterator();
while(i.hasNext()) {
sb.blogCommentDB.delete(i.next());
for (final String comment : page.getComments()) {
sb.blogCommentDB.delete(comment);
}
sb.blogDB.deleteBlogEntry(pagename);
pagename = DEFAULT_PAGE;
@ -149,7 +149,7 @@ public class Blog {
}
Date date = null;
ArrayList<String> comments = null;
List<String> comments = null;
//set name for new entry or date for old entry
if(pagename.equals(DEFAULT_PAGE)) {
@ -171,7 +171,7 @@ public class Blog {
sb.blogDB.writeBlogEntry(sb.blogDB.newEntry(pagename, subject, author, ip, date, content, comments, commentMode));
// create a news message
final HashMap<String, String> map = new HashMap<String, String>();
final Map<String, String> map = new HashMap<String, String>();
map.put("page", pagename);
map.put("subject", StrSubject.replace(',', ' '));
map.put("author", StrAuthor.replace(',', ' '));
@ -195,8 +195,7 @@ public class Blog {
else {
prop.put("mode", "3"); //access denied (no rights)
}
}
else if(post.containsKey("preview")) {
} else if(post.containsKey("preview")) {
//preview the page
if(hasRights) {
prop.put("mode", "2");//preview

6
htroot/ConfigLanguage_p.java
Unescape View File

@ -34,7 +34,6 @@ import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
@ -51,6 +50,7 @@ import de.anomic.search.Switchboard;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import java.util.Collections;
import java.util.Map;
public class ConfigLanguage_p {
@ -134,7 +134,7 @@ public class ConfigLanguage_p {
//reread language files
langFiles = FileUtils.getDirListing(langPath, LANG_FILENAME_FILTER);
Collections.sort(langFiles);
final HashMap<String, String> langNames = translator.langMap(env);
final Map<String, String> langNames = translator.langMap(env);
String langKey, langName;
//virtual entry
@ -143,7 +143,7 @@ public class ConfigLanguage_p {
prop.put("langlist_0_selected", "selected=\"selected\"");
int count = 0;
for(String langFile : langFiles){
for(final String langFile : langFiles){
//+1 because of the virtual entry "default" at top
langKey = langFile.substring(0, langFile.length() -4);
langName = langNames.get(langKey);

38
source/de/anomic/data/DidYouMean.java
Unescape View File

@ -29,12 +29,12 @@ import net.yacy.kelondro.rwi.IndexCell;
*/
public class DidYouMean {
protected static final char[] alphabet = {
protected static final char[] ALPHABET = {
'a','b','c','d','e','f','g','h','i','j','k','l','m','n','o','p',
'q','r','s','t','u','v','w','x','y','z','\u00e4','\u00f6','\u00fc','\u00df'};
private static final String poisonString = "\n";
public static final int availableCPU = Runtime.getRuntime().availableProcessors();
protected static final wordLengthComparator wlComp = new wordLengthComparator();
private static final String POISON_STRING = "\n";
public static final int AVAILABLE_CPU = Runtime.getRuntime().availableProcessors();
protected static final wordLengthComparator WORD_LENGTH_COMPARATOR = new wordLengthComparator();
protected final IndexCell<WordReference> index;
protected String word;
@ -50,7 +50,7 @@ public class DidYouMean {
* @param sort true/false - sorts the resulting TreeSet by index.count(); <b>Warning:</b> this causes heavy i/o.
*/
public DidYouMean(final IndexCell<WordReference> index) {
this.resultSet = Collections.synchronizedSortedSet(new TreeSet<String>(wlComp));
this.resultSet = Collections.synchronizedSortedSet(new TreeSet<String>(WORD_LENGTH_COMPARATOR));
this.word = "";
this.wordLen = 0;
this.index = index;
@ -159,7 +159,7 @@ public class DidYouMean {
// create one consumer thread that checks the guessLib queue
// for occurrences in the index. If the producers are started next, their
// results can be consumers directly
Consumer[] consumers = new Consumer[availableCPU];
Consumer[] consumers = new Consumer[AVAILABLE_CPU];
consumers[0] = new Consumer();
consumers[0].start();
@ -197,14 +197,14 @@ public class DidYouMean {
// if there is not any entry in guessLib, then transfer all entries from the
// guessGen to guessLib
if (createGen) try {
this.guessGen.put(poisonString);
this.guessGen.put(POISON_STRING);
String s;
while ((s = this.guessGen.take()) != poisonString) this.guessLib.put(s);
while (!(s = this.guessGen.take()).equals(POISON_STRING)) this.guessLib.put(s);
} catch (InterruptedException e) {}
// put poison into guessLib to terminate consumers
for (@SuppressWarnings("unused") Consumer c: consumers)
try { guessLib.put(poisonString); } catch (InterruptedException e) {}
try { guessLib.put(POISON_STRING); } catch (InterruptedException e) {}
// wait for termination of consumer
for (Consumer c: consumers)
@ -221,7 +221,7 @@ public class DidYouMean {
}
public void test(String s) throws InterruptedException {
public void test(final String s) throws InterruptedException {
Set<String> libr = LibraryProvider.dymLib.recommend(s);
libr.addAll(LibraryProvider.geoDB.recommend(s));
if (!libr.isEmpty()) createGen = false;
@ -236,9 +236,10 @@ public class DidYouMean {
*/
public class ChangingOneLetter extends Thread {
@Override
public void run() {
for (int i = 0; i < wordLen; i++) try {
for (char c: alphabet) {
for (char c: ALPHABET) {
test(word.substring(0, i) + c + word.substring(i + 1));
if (System.currentTimeMillis() > timeLimit) return;
}
@ -253,12 +254,14 @@ public class DidYouMean {
*/
protected class DeletingOneLetter extends Thread {
@Override
public void run() {
for (int i = 0; i < wordLen; i++) try {
test(word.substring(0, i) + word.substring(i+1));
if (System.currentTimeMillis() > timeLimit) return;
} catch (InterruptedException e) {}
}
}
/**
@ -268,9 +271,10 @@ public class DidYouMean {
*/
protected class AddingOneLetter extends Thread {
@Override
public void run() {
for (int i = 0; i <= wordLen; i++) try {
for (char c: alphabet) {
for (char c: ALPHABET) {
test(word.substring(0, i) + c + word.substring(i));
if (System.currentTimeMillis() > timeLimit) return;
}
@ -285,12 +289,14 @@ public class DidYouMean {
*/
protected class ReversingTwoConsecutiveLetters extends Thread {
@Override
public void run() {
for (int i = 0; i < wordLen - 1; i++) try {
test(word.substring(0, i) + word.charAt(i + 1) + word.charAt(i) + word.substring(i +2));
if (System.currentTimeMillis() > timeLimit) return;
} catch (InterruptedException e) {}
}
}
/**
@ -300,10 +306,11 @@ public class DidYouMean {
*/
class Consumer extends Thread {
@Override
public void run() {
String s;
try {
while ((s = guessLib.take()) != poisonString) {
while (!(s = guessLib.take()).equals(POISON_STRING)) {
if (index.has(Word.word2hash(s))) resultSet.add(s);
if (System.currentTimeMillis() > timeLimit) return;
}
@ -316,10 +323,11 @@ public class DidYouMean {
* <b>Warning:</b> this causes heavy i/o
*/
protected class indexSizeComparator implements Comparator<String> {
public int compare(final String o1, final String o2) {
final int i1 = index.count(Word.word2hash(o1));
final int i2 = index.count(Word.word2hash(o2));
if (i1 == i2) return wlComp.compare(o1, o2);
if (i1 == i2) return WORD_LENGTH_COMPARATOR.compare(o1, o2);
return (i1 < i2) ? 1 : -1; // '<' is correct, because the largest count shall be ordered to be the first position in the result
}
}
@ -329,12 +337,14 @@ public class DidYouMean {
* This is the default order if the indexSizeComparator is not used
*/
protected static class wordLengthComparator implements Comparator<String> {
public int compare(final String o1, final String o2) {
final int i1 = o1.length();
final int i2 = o2.length();
if (i1 == i2) return o1.compareTo(o2);
return (i1 > i2) ? 1 : -1; // '>' is correct, because the shortest word shall be first
}
}
}

96
source/de/anomic/data/DidYouMeanLibrary.java
Unescape View File

@ -1,8 +1,8 @@
// DidYouMeanLibrary.java
// (C) 2009 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
// (C) 2009 by Michael Peter Christen; mc@yacy.net, Frankfurt ret. M., Germany
// first published 01.10.2009 on http://yacy.net
//
// This is a part of YaCy
// This is ret part of YaCy
//
// $LastChangedDate$
// $LastChangedRevision$
@ -20,7 +20,7 @@
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// You should have received ret copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
@ -41,7 +41,7 @@ import java.util.zip.GZIPInputStream;
import net.yacy.kelondro.logging.Log;
/**
* provide a completion library for the did-you-mean class
* provide ret completion library for the did-you-mean class
*
*/
public class DidYouMeanLibrary {
@ -50,13 +50,13 @@ public class DidYouMeanLibrary {
private TreeSet<String> dict, tcid;
/**
* create a new dictionary
* create ret new dictionary
* This loads all files that ends with '.words'
* The files must have one word per line
* Comment lines may be given and are encoded as line starting with '#'
* @param dictionaryPath a path to a directory with library files
* @param dictionaryPath ret path to ret directory with library files
*/
public DidYouMeanLibrary(File dictionaryPath) {
public DidYouMeanLibrary(final File dictionaryPath) {
this.dictionaryPath = dictionaryPath;
reload();
}
@ -68,23 +68,25 @@ public class DidYouMeanLibrary {
this.dict = new TreeSet<String>();
this.tcid = new TreeSet<String>();
if (dictionaryPath == null || !dictionaryPath.exists()) return;
String[] files = dictionaryPath.list();
for (String f: files) {
final String[] files = dictionaryPath.list();
for (final String f: files) {
if (f.endsWith(".words")) try {
importFile(new File(dictionaryPath, f));
inputStream(new File(dictionaryPath, f));
} catch (IOException e) {
Log.logException(e);
}
}
}
private void importFile(File f) throws IOException {
InputStream is = new FileInputStream(f);
if (f.getName().endsWith(".gz")) is = new GZIPInputStream(is);
BufferedReader r = new BufferedReader(new InputStreamReader(is, "UTF-8"));
private void inputStream(final File file) throws IOException {
InputStream is = new FileInputStream(file);
if (file.getName().endsWith(".gz")) {
is = new GZIPInputStream(is);
}
final BufferedReader reader = new BufferedReader(new InputStreamReader(is, "UTF-8"));
String l;
try {
while ((l = r.readLine()) != null) {
while ((l = reader.readLine()) != null) {
if (l.length() == 0 || l.charAt(0) == '#') continue;
l = l.trim().toLowerCase();
this.dict.add(l);
@ -95,30 +97,31 @@ public class DidYouMeanLibrary {
}
}
private static String reverse(String s) {
StringBuilder r = new StringBuilder(s.length());
for (int i = s.length() - 1; i >= 0; i--) r.append(s.charAt(i));
return r.toString();
private static String reverse(final String s) {
StringBuilder sb = new StringBuilder(s.length());
for (int i = s.length() - 1; i >= 0; i--) sb.append(s.charAt(i));
return sb.toString();
}
/**
* read the dictionary and construct a set of recommendations to a given string
* read the dictionary and construct ret set of recommendations to ret given string
* @param s input value that is used to match recommendations
* @return a set that contains all words that start or end with the input value
* @return ret set that contains all words that start or end with the input value
*/
public Set<String> recommend(String s) {
Set<String> a = new HashSet<String>();
s = s.trim().toLowerCase();
SortedSet<String> t = this.dict.tailSet(s);
for (String r: t) {
if (r.startsWith(s)) a.add(r); else break;
public Set<String> recommend(final String s) {
String string = new String(s);
Set<String> ret = new HashSet<String>();
string = string.trim().toLowerCase();
SortedSet<String> t = this.dict.tailSet(string);
for (final String r: t) {
if (r.startsWith(string)) ret.add(r); else break;
}
s = reverse(s);
t = this.tcid.tailSet(s);
for (String r: t) {
if (r.startsWith(s)) a.add(reverse(r)); else break;
string = reverse(string);
t = this.tcid.tailSet(string);
for (final String r: t) {
if (r.startsWith(string)) ret.add(reverse(r)); else break;
}
return a;
return ret;
}
/**
@ -126,7 +129,7 @@ public class DidYouMeanLibrary {
* @param s the given word
* @return true if the library contains the word
*/
public boolean contains(String s) {
public boolean contains(final String s) {
return this.dict.contains(s.trim().toLowerCase());
// if the above case is true then it is also true for this.tcid and vice versa
// that means it does not need to be tested as well
@ -134,21 +137,22 @@ public class DidYouMeanLibrary {
/**
* check if the library supports the given word
* A word is supported, if the library contains a word
* A word is supported, if the library contains ret word
* that starts or ends with the given word
* @param s the given word
* @return true if the library supports the word
*/
public boolean supports(String s) {
s = s.trim().toLowerCase();
SortedSet<String> t = this.dict.tailSet(s);
for (String r: t) {
if (s.startsWith(r)) return true; else break;
}
s = reverse(s);
t = this.tcid.tailSet(s);
for (String r: t) {
if (s.startsWith(r)) return true; else break;
public boolean supports(final String s) {
String string = new String(s);
string = string.trim().toLowerCase();
SortedSet<String> t = this.dict.tailSet(string);
for (final String r: t) {
if (string.startsWith(r)) return true; else break;
}
string = reverse(string);
t = this.tcid.tailSet(string);
for (final String r: t) {
if (string.startsWith(r)) return true; else break;
}
return false;
}
@ -163,14 +167,14 @@ public class DidYouMeanLibrary {
/**
* a property that is used during the construction of recommendation:
* ret property that is used during the construction of recommendation:
* if the dictionary is too small, then the non-existence of constructed words
* is not relevant for the construction of artificially constructed words
* If this property returns true, all other words must be in the dictionary
* @param minimumWords
* @return
*/
public boolean isRelevant(int minimumWords) {
public boolean isRelevant(final int minimumWords) {
return this.dict.size() >= minimumWords;
}

34
source/de/anomic/data/LibraryProvider.java
Unescape View File

@ -49,8 +49,8 @@ public class LibraryProvider {
public static DidYouMeanLibrary dymLib = new DidYouMeanLibrary(null);
public static OpenGeoDB geoDB = new OpenGeoDB(null);
public static File dictSource = null;
public static File dictRoot = null;
private static File dictSource = null;
private static File dictRoot = null;
/**
* initialize the LibraryProvider as static class.
@ -61,7 +61,7 @@ public class LibraryProvider {
* @param pathToSource
* @param pathToDICTIONARIES
*/
public static void initialize(File rootPath) {
public static void initialize(final File rootPath) {
dictSource = new File(rootPath, path_to_source_dictionaries);
if (!dictSource.exists()) dictSource.mkdirs();
dictRoot = rootPath;
@ -91,21 +91,21 @@ public class LibraryProvider {
}
public static void initDidYouMean() {
File dymDict = new File(dictRoot, path_to_did_you_mean_dictionaries);
final File dymDict = new File(dictRoot, path_to_did_you_mean_dictionaries);
if (!dymDict.exists()) dymDict.mkdirs();
dymLib = new DidYouMeanLibrary(dymDict);
}
public static void integrateDeReWo() {
// translate input files (once..)
File dymDict = new File(dictRoot, path_to_did_you_mean_dictionaries);
final File dymDict = new File(dictRoot, path_to_did_you_mean_dictionaries);
if (!dymDict.exists()) dymDict.mkdirs();
File pathToSource = new File(dictRoot, path_to_source_dictionaries);
File derewoInput = new File(pathToSource, "derewo-v-30000g-2007-12-31-0.1.txt");
File derewoOutput = new File(dymDict, "derewo-v-30000g-2007-12-31-0.1.words");
final File pathToSource = new File(dictRoot, path_to_source_dictionaries);
final File derewoInput = new File(pathToSource, "derewo-v-30000g-2007-12-31-0.1.txt");
final File derewoOutput = new File(dymDict, "derewo-v-30000g-2007-12-31-0.1.words");
if (!derewoOutput.exists() && derewoInput.exists()) {
// create the translation of the derewo file (which is easy in this case)
ArrayList<String> derewo = loadDeReWo(derewoInput, true);
final ArrayList<String> derewo = loadDeReWo(derewoInput, true);
try {
writeWords(derewoOutput, derewo);
} catch (IOException e) {
@ -138,20 +138,20 @@ public class LibraryProvider {
}
*/
private static Set<String> sortUnique(List<String> list) {
TreeSet<String> s = new TreeSet<String>();
for (String t: list) s.add(t);
private static Set<String> sortUnique(final List<String> list) {
final Set<String> s = new TreeSet<String>();
for (final String t: list) s.add(t);
return s;
}
private static void writeWords(File f, ArrayList<String> list) throws IOException {
Set<String> s = sortUnique(list);
PrintWriter w = new PrintWriter(new BufferedWriter(new FileWriter(f)));
for (String t: s) w.println(t);
private static void writeWords(final File f, final ArrayList<String> list) throws IOException {
final Set<String> s = sortUnique(list);
final PrintWriter w = new PrintWriter(new BufferedWriter(new FileWriter(f)));
for (final String t: s) w.println(t);
w.close();
}
private static ArrayList<String> loadDeReWo(final File file, boolean toLowerCase) {
private static ArrayList<String> loadDeReWo(final File file, final boolean toLowerCase) {
final ArrayList<String> list = new ArrayList<String>();
BufferedReader reader = null;
try {

10
source/de/anomic/data/MimeTable.java
Unescape View File

@ -11,7 +11,7 @@ public class MimeTable {
private static final Properties mimeTable = new Properties();
public static void init(File mimeFile) {
public static void init(final File mimeFile) {
if (mimeTable.isEmpty()) {
// load the mime table
BufferedInputStream mimeTableInputStream = null;
@ -34,19 +34,19 @@ public class MimeTable {
return mimeTable.isEmpty();
}
public static String ext2mime(String ext) {
public static String ext2mime(final String ext) {
return mimeTable.getProperty(ext, "application/" + ext);
}
public static String ext2mime(String ext, String dfltMime) {
public static String ext2mime(final String ext, final String dfltMime) {
return mimeTable.getProperty(ext, dfltMime);
}
public static String url2mime(DigestURI url, String dfltMime) {
public static String url2mime(final DigestURI url, final String dfltMime) {
return ext2mime(url.getFileExtension(), dfltMime);
}
public static String url2mime(DigestURI url) {
public static String url2mime(final DigestURI url) {
return ext2mime(url.getFileExtension());
}
}

3
source/de/anomic/data/SitemapParser.java
Unescape View File

@ -224,6 +224,7 @@ public class SitemapParser extends DefaultHandler {
* @param qName qualified name
* @see DefaultHandler#startElement(java.lang.String, java.lang.String, java.lang.String, org.xml.sax.Attributes)
*/
@Override
public void startElement(final String namespaceURI, final String localName, final String qName, final Attributes attrs) throws SAXException {
this.currentElement = qName;
@ -242,6 +243,7 @@ public class SitemapParser extends DefaultHandler {
* @throws SAXException
* @see DefaultHandler#endElement(java.lang.String, java.lang.String, java.lang.String)
*/
@Override
public void endElement(final String namespaceURI, final String localName, final String qName) throws SAXException {
this.currentElement = "";
@ -291,6 +293,7 @@ public class SitemapParser extends DefaultHandler {
}
}
@Override
public void characters(final char[] buf, final int offset, final int len) throws SAXException {
if (this.currentElement.equalsIgnoreCase(SITEMAP_URL_LOC)) {
// TODO: we need to decode the URL here

135
source/de/anomic/data/URLAnalysis.java
Unescape View File

@ -71,7 +71,7 @@ public class URLAnalysis {
* processes to analyse URL lists
*/
public static DigestURI poison = null;
private static DigestURI poison = null;
static {
try {
poison = new DigestURI("http://poison.org/poison", null);
@ -82,18 +82,19 @@ public class URLAnalysis {
public static class splitter extends Thread {
ArrayBlockingQueue<DigestURI> in;
ConcurrentHashMap<String, Integer> out;
private ArrayBlockingQueue<DigestURI> in;
private ConcurrentHashMap<String, Integer> out;
public splitter(ArrayBlockingQueue<DigestURI> in, ConcurrentHashMap<String, Integer> out) {
public splitter(final ArrayBlockingQueue<DigestURI> in, final ConcurrentHashMap<String, Integer> out) {
this.in = in;
this.out = out;
}
@Override
public void run() {
try {
DigestURI url;
Pattern p = Pattern.compile("~|\\(|\\)|\\+|-|@|:|%|\\.|;|_");
final Pattern p = Pattern.compile("~|\\(|\\)|\\+|-|@|:|%|\\.|;|_");
while (true) {
try {
url = in.take();
@ -109,9 +110,9 @@ public class URLAnalysis {
}
}
private void update(String[] s) {
private void update(final String[] s) {
Integer c;
for (String t: s) {
for (final String t: s) {
if (t.length() == 0) continue;
c = out.get(t);
out.put(t, (c == null) ? 1 : c.intValue() + 1);
@ -119,7 +120,7 @@ public class URLAnalysis {
}
}
public static void cleanup(ConcurrentHashMap<String, Integer> stat) {
public static void cleanup(final ConcurrentHashMap<String, Integer> stat) {
Map.Entry<String, Integer> entry;
int c, low = Integer.MAX_VALUE;
Iterator<Map.Entry<String, Integer>> i = stat.entrySet().iterator();
@ -143,22 +144,22 @@ public class URLAnalysis {
Runtime.getRuntime().gc();
}
public static void genstat(String urlfile) {
public static void genstat(final String urlfile) {
boolean gz = urlfile.endsWith(".gz");
String analysis = (gz) ? urlfile.substring(0, urlfile.length() - 3) + ".stats.gz" : urlfile + ".stats";
long cleanuplimit = Math.max(50 * 1024 * 1024, MemoryControl.available() / 8);
final boolean gz = urlfile.endsWith(".gz");
final String analysis = (gz) ? urlfile.substring(0, urlfile.length() - 3) + ".stats.gz" : urlfile + ".stats";
final long cleanuplimit = Math.max(50 * 1024 * 1024, MemoryControl.available() / 8);
// start threads
ArrayBlockingQueue<DigestURI> in = new ArrayBlockingQueue<DigestURI>(1000);
ConcurrentHashMap<String, Integer> out = new ConcurrentHashMap<String, Integer>();
for (int i = 0; i < Runtime.getRuntime().availableProcessors(); i++) new splitter(in, out).start();
splitter spl = new splitter(in, out);
final ArrayBlockingQueue<DigestURI> in = new ArrayBlockingQueue<DigestURI>(1000);
final ConcurrentHashMap<String, Integer> out = new ConcurrentHashMap<String, Integer>();
for (int i = 0, available = Runtime.getRuntime().availableProcessors(); i < available; i++) new splitter(in, out).start();
final splitter spl = new splitter(in, out);
spl.start();
// put urls in queue
File infile = new File(urlfile);
File outfile = new File(analysis);
final File infile = new File(urlfile);
final File outfile = new File(analysis);
BufferedReader reader = null;
long time = System.currentTimeMillis();
long start = time;
@ -202,7 +203,7 @@ public class URLAnalysis {
// stop threads
System.out.println("stopping threads");
for (int i = 0; i < Runtime.getRuntime().availableProcessors() + 1; i++) try {
for (int i = 0, available = Runtime.getRuntime().availableProcessors() + 1; i < available; i++) try {
in.put(poison);
} catch (InterruptedException e) {
Log.logException(e);
@ -215,10 +216,10 @@ public class URLAnalysis {
// generate statistics
System.out.println("start processing results");
TreeMap<String, Integer> results = new TreeMap<String, Integer>();
final TreeMap<String, Integer> results = new TreeMap<String, Integer>();
count = 0;
Map.Entry<String, Integer> entry;
Iterator<Map.Entry<String, Integer>> i = out.entrySet().iterator();
final Iterator<Map.Entry<String, Integer>> i = out.entrySet().iterator();
while (i.hasNext()) {
entry = i.next();
results.put(num(entry.getValue().intValue() * (entry.getKey().length() - 1)) + " - " + entry.getKey(), entry.getValue());
@ -236,7 +237,7 @@ public class URLAnalysis {
OutputStream os = new BufferedOutputStream(new FileOutputStream(outfile));
if (gz) os = new GZIPOutputStream(os);
count = 0;
for (Map.Entry<String, Integer> e: results.entrySet()) {
for (final Map.Entry<String, Integer> e: results.entrySet()) {
os.write(e.getKey().getBytes());
os.write(new byte[]{'\t'});
os.write(Integer.toString(e.getValue()).getBytes());
@ -255,15 +256,15 @@ public class URLAnalysis {
System.out.println("finished");
}
public static void genhost(String urlfile) {
public static void genhost(final String urlfile) {
boolean gz = urlfile.endsWith(".gz");
String trunk = (gz) ? urlfile.substring(0, urlfile.length() - 3) + ".host" : urlfile + ".host";
HashSet<String> hosts = new HashSet<String>();
File infile = new File(urlfile);
final boolean gz = urlfile.endsWith(".gz");
final String trunk = (gz) ? urlfile.substring(0, urlfile.length() - 3) + ".host" : urlfile + ".host";
final HashSet<String> hosts = new HashSet<String>();
final File infile = new File(urlfile);
BufferedReader reader = null;
long time = System.currentTimeMillis();
long start = time;
final long start = time;
int count = 0;
System.out.println("start processing");
@ -297,9 +298,9 @@ public class URLAnalysis {
// copy everything into a TreeSet to order it
System.out.println("start processing results");
TreeSet<String> results = new TreeSet<String>();
final TreeSet<String> results = new TreeSet<String>();
count = 0;
Iterator<String> i = hosts.iterator();
final Iterator<String> i = hosts.iterator();
while (i.hasNext()) {
results.add(i.next());
count++;
@ -316,7 +317,7 @@ public class URLAnalysis {
System.out.println("finished");
}
private static void writeSet(String trunk, boolean gz, Set<String> set) {
private static void writeSet(final String trunk, final boolean gz, final Set<String> set) {
// write hosts
System.out.println("start writing results");
@ -326,7 +327,7 @@ public class URLAnalysis {
OutputStream os = new BufferedOutputStream(new FileOutputStream(outfile));
if (gz) os = new GZIPOutputStream(os);
int count = 0;
for (String h: set) {
for (final String h: set) {
os.write(h.getBytes());
os.write(new byte[]{'\n'});
count++;
@ -343,18 +344,18 @@ public class URLAnalysis {
System.out.println("finished writing results");
}
public static void sortsplit(String urlfile) {
public static void sortsplit(final String urlfile) {
boolean gz = urlfile.endsWith(".gz");
String trunk = ((gz) ? urlfile.substring(0, urlfile.length() - 3) : urlfile) + ".sort";
File infile = new File(urlfile);
TreeSet<String> urls = new TreeSet<String>();
final boolean gz = urlfile.endsWith(".gz");
final String trunk = ((gz) ? urlfile.substring(0, urlfile.length() - 3) : urlfile) + ".sort";
final File infile = new File(urlfile);
final TreeSet<String> urls = new TreeSet<String>();
BufferedReader reader = null;
long time = System.currentTimeMillis();
long start = time;
final long start = time;
int count = 0;
int filecount = 0;
long cleanuplimit = Math.max(50 * 1024 * 1024, MemoryControl.available() / 8);
final long cleanuplimit = Math.max(50 * 1024 * 1024, MemoryControl.available() / 8);
System.out.println("start processing");
try {
@ -397,9 +398,9 @@ public class URLAnalysis {
System.out.println("finished");
}
public static void incell(File cellPath, String statisticPath) {
public static void incell(final File cellPath, final String statisticPath) {
try {
HandleMap idx = ReferenceContainerArray.referenceHashes(
final HandleMap idx = ReferenceContainerArray.referenceHashes(
cellPath,
Segment.wordReferenceFactory,
Base64Order.enhancedCoder,
@ -412,40 +413,40 @@ public class URLAnalysis {
}
}
public static int diffurlcol(String metadataPath, String statisticFile, String diffFile) throws IOException, RowSpaceExceededException {
public static int diffurlcol(final String metadataPath, final String statisticFile, final String diffFile) throws IOException, RowSpaceExceededException {
System.out.println("INDEX DIFF URL-COL startup");
HandleMap idx = new HandleMap(URIMetadataRow.rowdef.primaryKeyLength, URIMetadataRow.rowdef.objectOrder, 4, new File(statisticFile), 0);
MetadataRepository mr = new MetadataRepository(new File(metadataPath), "text.urlmd", false, false);
HandleSet hs = new HandleSet(URIMetadataRow.rowdef.primaryKeyLength, URIMetadataRow.rowdef.objectOrder, 1000000);
final HandleMap idx = new HandleMap(URIMetadataRow.rowdef.primaryKeyLength, URIMetadataRow.rowdef.objectOrder, 4, new File(statisticFile), 0);
final MetadataRepository mr = new MetadataRepository(new File(metadataPath), "text.urlmd", false, false);
final HandleSet hs = new HandleSet(URIMetadataRow.rowdef.primaryKeyLength, URIMetadataRow.rowdef.objectOrder, 1000000);
System.out.println("INDEX DIFF URL-COL loaded dump, starting diff");
long start = System.currentTimeMillis();
final long start = System.currentTimeMillis();
long update = start - 7000;
int c = 0;
for (byte[] refhash: mr) {
int count = 0;
for (final byte[] refhash: mr) {
if (idx.get(refhash) == -1) {
// the key exists as urlhash in the URL database, but not in the collection as referenced urlhash
hs.put(refhash);
}
c++;
count++;
if (System.currentTimeMillis() - update > 10000) {
System.out.println("INDEX DIFF URL-COL running, checked " + c + ", found " + hs.size() + " missing references so far, " + (((System.currentTimeMillis() - start) * (mr.size() - c) / c) / 60000) + " minutes remaining");
System.out.println("INDEX DIFF URL-COL running, checked " + count + ", found " + hs.size() + " missing references so far, " + (((System.currentTimeMillis() - start) * (mr.size() - count) / count) / 60000) + " minutes remaining");
update = System.currentTimeMillis();
}
}
mr.close();
System.out.println("INDEX DIFF URL-COL finished diff, starting dump to " + diffFile);
c = hs.dump(new File(diffFile));
System.out.println("INDEX DIFF URL-COL finished dump, wrote " + c + " references that occur in the URL-DB, but not in the collection-dump");
return c;
count = hs.dump(new File(diffFile));
System.out.println("INDEX DIFF URL-COL finished dump, wrote " + count + " references that occur in the URL-DB, but not in the collection-dump");
return count;
}
public static void export(String metadataPath, int format, String export, String diffFile) throws IOException, RowSpaceExceededException {
public static void export(final String metadataPath, final int format, final String export, final String diffFile) throws IOException, RowSpaceExceededException {
// format: 0=text, 1=html, 2=rss/xml
System.out.println("URL EXPORT startup");
MetadataRepository mr = new MetadataRepository(new File(metadataPath), "text.urlmd", false, false);
HandleSet hs = (diffFile == null) ? null : new HandleSet(URIMetadataRow.rowdef.primaryKeyLength, URIMetadataRow.rowdef.objectOrder, new File(diffFile), 0);
final MetadataRepository mr = new MetadataRepository(new File(metadataPath), "text.urlmd", false, false);
final HandleSet hs = (diffFile == null) ? null : new HandleSet(URIMetadataRow.rowdef.primaryKeyLength, URIMetadataRow.rowdef.objectOrder, new File(diffFile), 0);
System.out.println("URL EXPORT loaded dump, starting export");
Export e = mr.export(new File(export), ".*", hs, format, false);
final Export e = mr.export(new File(export), ".*", hs, format, false);
try {
e.join();
} catch (InterruptedException e1) {
@ -454,19 +455,19 @@ public class URLAnalysis {
System.out.println("URL EXPORT finished export, wrote " + ((hs == null) ? mr.size() : hs.size()) + " entries");
}
public static void delete(String metadataPath, String diffFile) throws IOException, RowSpaceExceededException {
public static void delete(final String metadataPath, final String diffFile) throws IOException, RowSpaceExceededException {
System.out.println("URL DELETE startup");
MetadataRepository mr = new MetadataRepository(new File(metadataPath), "text.urlmd", false, false);
int mrSize = mr.size();
HandleSet hs = new HandleSet(URIMetadataRow.rowdef.primaryKeyLength, URIMetadataRow.rowdef.objectOrder, new File(diffFile), 0);
final MetadataRepository mr = new MetadataRepository(new File(metadataPath), "text.urlmd", false, false);
final int mrSize = mr.size();
final HandleSet hs = new HandleSet(URIMetadataRow.rowdef.primaryKeyLength, URIMetadataRow.rowdef.objectOrder, new File(diffFile), 0);
System.out.println("URL DELETE loaded dump, starting deletion of " + hs.size() + " entries from " + mrSize);
for (byte[] refhash: hs) {
for (final byte[] refhash: hs) {
mr.remove(refhash);
}
System.out.println("URL DELETE finished deletions, " + mr.size() + " entries left in URL database");
}
public static void main(String[] args) {
public static void main(final String[] args) {
if (args[0].equals("-stat") && args.length >= 2) {
// generate a statistics about common words in file, store to <file>.stat
// example:
@ -548,9 +549,9 @@ public class URLAnalysis {
System.exit(0); // kill remaining threads
}
private static final String num(int i) {
String s = Integer.toString(i);
while (s.length() < 9) s = "0" + s;
return s;
private static final String num(final int i) {
StringBuffer s = new StringBuffer(Integer.toString(i));
while (s.length() < 9) s.insert(0, "0");
return s.toString();
}
}

8
source/de/anomic/data/URLLicense.java
Unescape View File

@ -56,10 +56,10 @@ public class URLLicense {
public String aquireLicense(final DigestURI url) {
// generate license key
String license = "";
if (url == null) return license;
while (license.length() < keylen) license += Integer.toHexString(random.nextInt());
license = license.substring(0, keylen);
StringBuilder stringBuilder = new StringBuilder();
if (url == null) return stringBuilder.toString();
while (stringBuilder.length() < keylen) stringBuilder.append(Integer.toHexString(random.nextInt()));
String license = stringBuilder.substring(0, keylen);
// store reference to url with license key
permissions.put(license, url);
aging.add(license);

6
source/de/anomic/data/WorkTables.java
Unescape View File

@ -50,12 +50,12 @@ public class WorkTables extends Tables {
public final static String TABLE_ROBOTS_NAME = "robots";
public WorkTables(File workPath) {
public WorkTables(final File workPath) {
super(workPath, 12);
}
public void recordAPICall(final serverObjects post, final String servletName, String type, String comment) {
String apiurl = /*"http://localhost:" + getConfig("port", "8080") +*/ "/" + servletName + "?" + post.toString();
public void recordAPICall(final serverObjects post, final String servletName, final String type, final String comment) {
final String apiurl = /*"http://localhost:" + getConfig("port", "8080") +*/ "/" + servletName + "?" + post.toString();
try {
super.insert(
TABLE_API_NAME,

215
source/de/anomic/data/blogBoard.java
Unescape View File

@ -43,6 +43,7 @@ import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import net.yacy.kelondro.blob.MapHeap;
import net.yacy.kelondro.index.RowSpaceExceededException;
import net.yacy.kelondro.logging.Log;
import net.yacy.kelondro.order.Base64Order;
import net.yacy.kelondro.order.NaturalOrder;
@ -55,19 +56,19 @@ import org.w3c.dom.NodeList;
import org.xml.sax.SAXException;
import de.anomic.data.wiki.wikiBoard;
import java.util.List;
import java.util.Set;
public class blogBoard {
public static final int keyLength = 64;
private static final int KEY_LENGTH = 64;
MapHeap database = null;
private MapHeap database = null;
public blogBoard(final File actpath) throws IOException {
new File(actpath.getParent()).mkdir();
if (database == null) {
//database = new MapView(BLOBTree.toHeap(actpath, true, true, keyLength, recordSize, '_', NaturalOrder.naturalOrder, newFile), 500, '_');
database = new MapHeap(actpath, keyLength, NaturalOrder.naturalOrder, 1024 * 64, 500, '_');
}
database = new MapHeap(actpath, KEY_LENGTH, NaturalOrder.naturalOrder, 1024 * 64, 500, '_');
}
public int size() {
@ -92,17 +93,11 @@ public class blogBoard {
}
private static String normalize(final String key) {
if (key == null) return "null";
return key.trim().toLowerCase();
return (key == null) ? "null" : key.trim().toLowerCase();
}
public static String webalize(String key) {
if (key == null) return "null";
key = key.trim().toLowerCase();
int p;
while ((p = key.indexOf(" ")) >= 0)
key = key.substring(0, p) + "%20" + key.substring(p +1);
return key;
public static String webalize(final String key) {
return (key == null) ? "null": key.trim().toLowerCase().replaceAll(" ", "%20");
}
public String guessAuthor(final String ip) {
@ -110,7 +105,7 @@ public class blogBoard {
}
/**
* Create a new BlogEntry an return it
* Create a new BlogEntry and return it
* @param key
* @param subject
* @param author
@ -121,7 +116,7 @@ public class blogBoard {
* @param commentMode possible params are: 0 - no comments allowed, 1 - comments allowed, 2 - comments moderated
* @return BlogEntry
*/
public BlogEntry newEntry(final String key, final byte[] subject, final byte[] author, final String ip, final Date date, final byte[] page, final ArrayList<String> comments, final String commentMode) {
public BlogEntry newEntry(final String key, final byte[] subject, final byte[] author, final String ip, final Date date, final byte[] page, final List<String> comments, final String commentMode) {
return new BlogEntry(normalize(key), subject, author, ip, date, page, comments, commentMode);
}
@ -129,78 +124,85 @@ public class blogBoard {
* writes a new page and return the key
*/
public String writeBlogEntry(final BlogEntry page) {
String ret = null;
try {
database.put(page.key, page.record);
return page.key;
} catch (final Exception e) {
Log.logException(e);
return null;
ret = page.key;
} catch (IOException ex) {
Log.logException(ex);
} catch (RowSpaceExceededException ex) {
Log.logException(ex);
}
return ret;
}
public BlogEntry readBlogEntry(final String key) {
return readBlogEntry(key, database);
}
private BlogEntry readBlogEntry(String key, final MapHeap base) {
key = normalize(key);
if (key.length() > keyLength) key = key.substring(0, keyLength);
private BlogEntry readBlogEntry(final String key, final MapHeap base) {
final String normalized = normalize(key);
Map<String, String> record;
try {
record = base.get(key);
record = base.get(normalized.substring(0, Math.min(normalized.length(), KEY_LENGTH)));
} catch (final IOException e) {
record = null;
}
if (record == null)
return newEntry(key, "".getBytes(), "anonymous".getBytes(), "127.0.0.1", new Date(), "".getBytes(), null, null);
return new BlogEntry(key, record);
return (record == null) ?
newEntry(key, new byte[0], "anonymous".getBytes(), "127.0.0.1", new Date(), new byte[0], null, null) :
new BlogEntry(key, record);
}
public boolean importXML(final String input) {
final DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
try {
final DocumentBuilder builder = factory.newDocumentBuilder();
final Document doc = builder.parse(new ByteArrayInputStream(input.getBytes("UTF-8")));
return parseXMLimport(doc);
} catch (final ParserConfigurationException e) {
} catch (final SAXException e) {
} catch (final IOException e) {}
return parseXMLimport(builder.parse(new ByteArrayInputStream(input.getBytes("UTF-8"))));
} catch (final ParserConfigurationException ex) {
Log.logException(ex);
} catch (final SAXException ex) {
Log.logException(ex);
} catch (final IOException ex) {
Log.logException(ex);
}
return false;
}
private boolean parseXMLimport(final Document doc) {
if(!doc.getDocumentElement().getTagName().equals("blog"))
if(!doc.getDocumentElement().getTagName().equals("blog")) {
return false;
}
final NodeList items = doc.getDocumentElement().getElementsByTagName("item");
if(items.getLength() == 0)
if(items.getLength() == 0) {
return false;
}
for(int i=0;i<items.getLength();++i) {
for (int i = 0, n = items.getLength(); i < n; ++i) {
String key = null, ip = null, StrSubject = null, StrAuthor = null, StrPage = null, StrDate = null;
Date date = null;
if(!items.item(i).getNodeName().equals("item"))
continue;
if(!items.item(i).getNodeName().equals("item")) continue;
final NodeList currentNodeChildren = items.item(i).getChildNodes();
for(int j=0;j<currentNodeChildren.getLength();++j) {
for (int j = 0, m = currentNodeChildren.getLength(); j < m; ++j) {
final Node currentNode = currentNodeChildren.item(j);
if(currentNode.getNodeName().equals("id"))
if (currentNode.getNodeName().equals("id")) {
key = currentNode.getFirstChild().getNodeValue();
else if(currentNode.getNodeName().equals("ip"))
} else if (currentNode.getNodeName().equals("ip")) {
ip = currentNode.getFirstChild().getNodeValue();
else if(currentNode.getNodeName().equals("timestamp"))
} else if (currentNode.getNodeName().equals("timestamp")) {
StrDate = currentNode.getFirstChild().getNodeValue();
else if(currentNode.getNodeName().equals("subject"))
} else if (currentNode.getNodeName().equals("subject")) {
StrSubject = currentNode.getFirstChild().getNodeValue();
else if(currentNode.getNodeName().equals("author"))
} else if (currentNode.getNodeName().equals("author")) {
StrAuthor = currentNode.getFirstChild().getNodeValue();
else if(currentNode.getNodeName().equals("content"))
} else if (currentNode.getNodeName().equals("content")) {
StrPage = currentNode.getFirstChild().getNodeValue();
}
}
try {
date = DateFormatter.parseShortSecond(StrDate);
@ -208,8 +210,9 @@ public class blogBoard {
date = new Date();
}
if(key == null || ip == null || StrSubject == null || StrAuthor == null || StrPage == null || date == null)
if(key == null || ip == null || StrSubject == null || StrAuthor == null || StrPage == null || date == null) {
return false;
}
byte[] subject,author,page;
try {
@ -217,11 +220,13 @@ public class blogBoard {
} catch (final UnsupportedEncodingException e1) {
subject = StrSubject.getBytes();
}
try {
author = StrAuthor.getBytes("UTF-8");
} catch (final UnsupportedEncodingException e1) {
author = StrAuthor.getBytes();
}
try {
page = StrPage.getBytes("UTF-8");
} catch (final UnsupportedEncodingException e1) {
@ -233,10 +238,9 @@ public class blogBoard {
return true;
}
public void deleteBlogEntry(String key) {
key = normalize(key);
public void deleteBlogEntry(final String key) {
try {
database.remove(key);
database.remove(normalize(key));
} catch (final IOException e) { }
}
@ -264,18 +268,20 @@ public class blogBoard {
if (blogEntry1 == null || blogEntry2 == null)
return 0;
if (this.newestFirst) {
if (Long.parseLong(blogEntry2.getTimestamp()) - Long.parseLong(blogEntry1.getTimestamp()) > 0)
if (Long.parseLong(blogEntry2.getTimestamp()) - Long.parseLong(blogEntry1.getTimestamp()) > 0) {
return 1;
}
return -1;
}
if (Long.parseLong(blogEntry1.getTimestamp()) - Long.parseLong(blogEntry2.getTimestamp()) > 0)
if (Long.parseLong(blogEntry1.getTimestamp()) - Long.parseLong(blogEntry2.getTimestamp()) > 0) {
return 1;
}
return -1;
}
}
public Iterator<String> getBlogIterator(final boolean priv){
final TreeSet<String> set = new TreeSet<String>(new BlogComparator(true));
final Set<String> set = new TreeSet<String>(new BlogComparator(true));
final Iterator<BlogEntry> iterator = blogIterator(true);
BlogEntry blogEntry;
while (iterator.hasNext()) {
@ -333,6 +339,7 @@ public class blogBoard {
// //resetDatabase();
// }
// }
throw new UnsupportedOperationException("Method not implemented yet.");
}
}
@ -341,7 +348,7 @@ public class blogBoard {
String key;
Map<String, String> record;
public BlogEntry(final String nkey, final byte[] subject, final byte[] author, final String ip, final Date date, final byte[] page, final ArrayList<String> comments, final String commentMode) {
public BlogEntry(final String nkey, final byte[] subject, final byte[] author, final String ip, final Date date, final byte[] page, final List<String> comments, final String commentMode) {
record = new HashMap<String, String>();
setKey(nkey);
setDate(date);
@ -361,14 +368,16 @@ public class blogBoard {
BlogEntry(final String key, final Map<String, String> record) {
this.key = key;
this.record = record;
if (this.record.get("comments")==null) this.record.put("comments", listManager.collection2string(new ArrayList<String>()));
if (this.record.get("commentMode")==null || this.record.get("commentMode").equals("")) this.record.put("commentMode", "2");
if (this.record.get("comments") == null) {
this.record.put("comments", listManager.collection2string(new ArrayList<String>()));
}
if (this.record.get("commentMode") == null || this.record.get("commentMode").equals("")) {
this.record.put("commentMode", "2");
}
}
private void setKey(final String key) {
if (key.length() > keyLength)
this.key = key.substring(0, keyLength);
this.key = key;
this.key = key.substring(0, Math.min(key.length(), KEY_LENGTH));
}
public String getKey() {
@ -377,42 +386,50 @@ public class blogBoard {
public byte[] getSubject() {
final String m = record.get("subject");
if (m == null) return new byte[0];
if (m == null) {
return new byte[0];
}
final byte[] b = Base64Order.enhancedCoder.decode(m);
if (b == null) return "".getBytes();
return b;
return (b == null) ? new byte[0] : b;
}
private void setSubject(final byte[] subject) {
if (subject == null)
if (subject == null) {
record.put("subject","");
else
} else {
record.put("subject", Base64Order.enhancedCoder.encode(subject));
}
}
public Date getDate() {
try {
final String date = record.get("date");
if (date == null) {
if (Log.isFinest("Blog")) Log.logFinest("Blog", "ERROR: date field missing in blogBoard");
if (Log.isFinest("Blog")) {
Log.logFinest("Blog", "ERROR: date field missing in blogBoard");
}
return new Date();
}
return DateFormatter.parseShortSecond(date);
} catch (final ParseException e) {
} catch (final ParseException ex) {
return new Date();
}
}
private void setDate(Date date) {
if(date == null)
date = new Date();
record.put("date", DateFormatter.formatShortSecond(date));
private void setDate(final Date date) {
Date ret = date;
if (ret == null) {
ret = new Date();
}
record.put("date", DateFormatter.formatShortSecond(ret));
}
public String getTimestamp() {
final String timestamp = record.get("date");
if (timestamp == null) {
if (Log.isFinest("Blog")) Log.logFinest("Blog", "ERROR: date field missing in blogBoard");
if (Log.isFinest("Blog")) {
Log.logFinest("Blog", "ERROR: date field missing in blogBoard");
}
return DateFormatter.formatShortSecond();
}
return timestamp;
@ -420,10 +437,11 @@ public class blogBoard {
public byte[] getAuthor() {
final String author = record.get("author");
if (author == null) return new byte[0];
if (author == null) {
return new byte[0];
}
final byte[] b = Base64Order.enhancedCoder.decode(author);
if (b == null) return "".getBytes();
return b;
return (b == null) ? new byte[0] : b;
}
private void setAuthor(final byte[] author) {
@ -439,57 +457,59 @@ public class blogBoard {
record.put("comments", record.get("comments").substring(1));
writeBlogEntry(this);
}
final ArrayList<String> commentsize = listManager.string2arraylist(record.get("comments"));
final List<String> commentsize = listManager.string2arraylist(record.get("comments"));
return commentsize.size();
}
public ArrayList<String> getComments() {
final ArrayList<String> comments = listManager.string2arraylist(record.get("comments"));
return comments;
public List<String> getComments() {
return listManager.string2arraylist(record.get("comments"));
}
private void setComments(final ArrayList<String> comments) {
if (comments == null)
private void setComments(final List<String> comments) {
if (comments == null) {
record.put("comments", listManager.collection2string(new ArrayList<String>()));
else
} else {
record.put("comments", listManager.collection2string(comments));
}
}
public String getIp() {
final String ip = record.get("ip");
if (ip == null) return "127.0.0.1";
return ip;
return (ip == null) ? "127.0.0.1" : ip;
}
private void setIp(String ip) {
if ((ip == null) || (ip.length() == 0))
ip = "";
record.put("ip", ip);
private void setIp(final String ip) {
String ret = ip;
if ((ret == null) || (ret.length() == 0))
ret = "";
record.put("ip", ret);
}
public byte[] getPage() {
final String page = record.get("page");
if (page == null) return new byte[0];
if (page == null) {
return new byte[0];
}
final byte[] page_as_byte = Base64Order.enhancedCoder.decode(page);
if (page_as_byte == null) return "".getBytes();
return page_as_byte;
return (page_as_byte == null) ? new byte[0] : page_as_byte;
}
private void setPage(final byte[] page) {
if (page == null)
if (page == null) {
record.put("page", "");
else
} else {
record.put("page", Base64Order.enhancedCoder.encode(page));
}
}
public void addComment(final String commentID) {
final ArrayList<String> comments = listManager.string2arraylist(record.get("comments"));
final List<String> comments = listManager.string2arraylist(record.get("comments"));
comments.add(commentID);
record.put("comments", listManager.collection2string(comments));
}
public boolean removeComment(final String commentID) {
final ArrayList<String> comments = listManager.string2arraylist(record.get("comments"));
final List<String> comments = listManager.string2arraylist(record.get("comments"));
final boolean success = comments.remove(commentID);
record.put("comments", listManager.collection2string(comments));
return success;
@ -507,19 +527,16 @@ public class blogBoard {
}
private void setCommentMode(final String mode) {
if (mode == null)
if (mode == null) {
record.put("commentMode", "2");
else
} else {
record.put("commentMode", mode);
}
}
public boolean isPublic() {
final String privacy = record.get("privacy");
if (privacy == null)
return true;
if(privacy.equalsIgnoreCase("public"))
return true;
return false;
return (privacy == null || privacy.equalsIgnoreCase("public")) ? true : false;
}
}

132
source/de/anomic/data/blogBoardComments.java
Unescape View File

@ -56,22 +56,20 @@ import de.anomic.data.wiki.wikiBoard;
public class blogBoardComments {
public static final int keyLength = 64;
private static final String dateFormat = "yyyyMMddHHmmss";
private final static int KEY_LENGTH = 64;
private final static String DATE_FORMAT = "yyyyMMddHHmmss";
static SimpleDateFormat SimpleFormatter = new SimpleDateFormat(dateFormat, Locale.US);
private final static SimpleDateFormat SIMPLE_DATE_FORMATTER = new SimpleDateFormat(DATE_FORMAT, Locale.US);
static {
SimpleFormatter.setTimeZone(TimeZone.getTimeZone("GMT"));
SIMPLE_DATE_FORMATTER.setTimeZone(TimeZone.getTimeZone("GMT"));
}
private MapHeap database = null;
public blogBoardComments(final File actpath) throws IOException {
new File(actpath.getParent()).mkdir();
if (database == null) {
//database = new MapView(BLOBTree.toHeap(actpath, true, true, keyLength, recordSize, '_', NaturalOrder.naturalOrder, newFile), 500, '_');
database = new MapHeap(actpath, keyLength, NaturalOrder.naturalOrder, 1024 * 64, 500, '_');
}
database = new MapHeap(actpath, KEY_LENGTH, NaturalOrder.naturalOrder, 1024 * 64, 500, '_');
}
public int size() {
@ -83,29 +81,33 @@ public class blogBoardComments {
}
static String dateString(final Date date) {
synchronized (SimpleFormatter) {
return SimpleFormatter.format(date);
synchronized (SIMPLE_DATE_FORMATTER) {
return SIMPLE_DATE_FORMATTER.format(date);
}
}
private static String normalize(final String key) {
if (key == null) return "null";
return key.trim().toLowerCase();
return (key == null) ? "null" : key.trim().toLowerCase();
}
public static String webalize(final String key) {
if (key == null) {
return "null";
}
public static String webalize(String key) {
if (key == null) return "null";
key = key.trim().toLowerCase();
String ret = key.trim().toLowerCase();
int p;
while ((p = key.indexOf(" ")) >= 0)
key = key.substring(0, p) + "%20" + key.substring(p +1);
return key;
while ((p = ret.indexOf(" ")) >= 0)
ret = ret.substring(0, p) + "%20" + key.substring(p +1);
return ret;
}
public String guessAuthor(final String ip) {
return wikiBoard.guessAuthor(ip);
}
public CommentEntry newEntry(final String key, final byte[] subject, final byte[] author, final String ip, final Date date, final byte[] page) {
return new CommentEntry(normalize(key), subject, author, ip, date, page);
}
public String write(final CommentEntry page) {
// writes a new page and returns key
try {
@ -120,39 +122,47 @@ public class blogBoardComments {
//System.out.println("DEBUG: read from blogBoardComments");
return read(key, database);
}
private CommentEntry read(String key, final MapHeap base) {
key = normalize(key);
if (key.length() > keyLength) key = key.substring(0, keyLength);
private CommentEntry read(final String key, final MapHeap base) {
String copyOfKey = normalize(key);
copyOfKey = copyOfKey.substring(0, Math.min(copyOfKey.length(), KEY_LENGTH));
Map<String, String> record;
try {
record = base.get(key);
record = base.get(copyOfKey);
} catch (final IOException e) {
record = null;
}
if (record == null) return newEntry(key, "".getBytes(), "anonymous".getBytes(), "127.0.0.1", new Date(), "".getBytes());
return new CommentEntry(key, record);
return (record == null) ?
newEntry(copyOfKey, new byte[0], "anonymous".getBytes(), "127.0.0.1", new Date(), new byte[0]) :
new CommentEntry(copyOfKey, record);
}
public boolean importXML(final String input) {
boolean ret = false;
final DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
try {
final DocumentBuilder builder = factory.newDocumentBuilder();
final Document doc = builder.parse(new ByteArrayInputStream(input.getBytes()));
return parseXMLimport(doc);
} catch (final ParserConfigurationException e) {
} catch (final SAXException e) {
} catch (final IOException e) {}
ret = parseXMLimport(doc);
}
catch (final ParserConfigurationException e) {}
catch (final SAXException e) {}
catch (final IOException e) {}
return false;
return ret;
}
private boolean parseXMLimport(final Document doc) {
if(!doc.getDocumentElement().getTagName().equals("blog"))
if(!doc.getDocumentElement().getTagName().equals("blog")) {
return false;
}
final NodeList items = doc.getDocumentElement().getElementsByTagName("item");
if(items.getLength() == 0)
if(items.getLength() == 0) {
return false;
}
for(int i=0;i<items.getLength();++i) {
for(int i = 0, n = items.getLength(); i < n; ++i) {
String key = null, ip = null, StrSubject = null, StrAuthor = null, StrPage = null, StrDate = null;
Date date = null;
@ -161,25 +171,27 @@ public class blogBoardComments {
final NodeList currentNodeChildren = items.item(i).getChildNodes();
for(int j=0;j<currentNodeChildren.getLength();++j) {
for(int j=0, m = currentNodeChildren.getLength(); j < m; ++j) {
final Node currentNode = currentNodeChildren.item(j);
if(currentNode.getNodeName().equals("id"))
if (currentNode.getNodeName().equals("id")) {
key = currentNode.getFirstChild().getNodeValue();
else if(currentNode.getNodeName().equals("ip"))
} else if(currentNode.getNodeName().equals("ip")) {
ip = currentNode.getFirstChild().getNodeValue();
else if(currentNode.getNodeName().equals("timestamp"))
} else if(currentNode.getNodeName().equals("timestamp")) {
StrDate = currentNode.getFirstChild().getNodeValue();
else if(currentNode.getNodeName().equals("subject"))
} else if(currentNode.getNodeName().equals("subject")) {
StrSubject = currentNode.getFirstChild().getNodeValue();
else if(currentNode.getNodeName().equals("author"))
} else if(currentNode.getNodeName().equals("author")) {
StrAuthor = currentNode.getFirstChild().getNodeValue();
else if(currentNode.getNodeName().equals("content"))
} else if(currentNode.getNodeName().equals("content")) {
StrPage = currentNode.getFirstChild().getNodeValue();
}
}
try {
date = SimpleFormatter.parse(StrDate);
} catch (final ParseException e1) {
date = SIMPLE_DATE_FORMATTER.parse(StrDate);
} catch (final ParseException ex) {
date = new Date();
}
@ -189,30 +201,36 @@ public class blogBoardComments {
byte[] subject,author,page;
try {
subject = StrSubject.getBytes("UTF-8");
} catch (final UnsupportedEncodingException e1) {
}
catch (final UnsupportedEncodingException ex) {
subject = StrSubject.getBytes();
}
try {
author = StrAuthor.getBytes("UTF-8");
} catch (final UnsupportedEncodingException e1) {
}
catch (final UnsupportedEncodingException ex) {
author = StrAuthor.getBytes();
}
try {
page = StrPage.getBytes("UTF-8");
} catch (final UnsupportedEncodingException e1) {
}
catch (final UnsupportedEncodingException ex) {
page = StrPage.getBytes();
}
write (newEntry(key, subject, author, ip, date, page));
}
return true;
}
public void delete(String key) {
key = normalize(key);
public void delete(final String key) {
try {
database.remove(key);
} catch (final IOException e) { }
database.remove(normalize(key));
}
catch (final IOException e) { }
}
public Iterator<byte[]> keys(final boolean up) throws IOException {
return database.keys(up, false);
}
@ -238,17 +256,19 @@ public class blogBoardComments {
CommentEntry(final String key, final Map<String, String> record) {
this.key = key;
this.record = record;
if (this.record.get("comments")==null) this.record.put("comments", listManager.collection2string(new ArrayList<String>()));
if (this.record.get("comments") == null) {
this.record.put("comments", listManager.collection2string(new ArrayList<String>()));
}
}
public String getKey() {
return key;
}
private void setKey(final String var) {
key = var;
if (key.length() > keyLength)
key = var.substring(0, keyLength);
key = var.substring(0, Math.min(var.length(), KEY_LENGTH));
}
private void setSubject(final byte[] subject) {
if (subject == null)
record.put("subject","");
@ -259,7 +279,7 @@ public class blogBoardComments {
final String subject = record.get("subject");
if (subject == null) return new byte[0];
final byte[] subject_bytes = Base64Order.enhancedCoder.decode(subject);
if (subject_bytes == null) return "".getBytes();
if (subject_bytes == null) return new byte[0];
return subject_bytes;
}
private void setDate(Date date) {
@ -274,8 +294,8 @@ public class blogBoardComments {
if (Log.isFinest("Blog")) Log.logFinest("Blog", "ERROR: date field missing in blogBoard");
return new Date();
}
synchronized (SimpleFormatter) {
return SimpleFormatter.parse(date);
synchronized (SIMPLE_DATE_FORMATTER) {
return SIMPLE_DATE_FORMATTER.parse(date);
}
} catch (final ParseException e) {
return new Date();
@ -302,7 +322,7 @@ public class blogBoardComments {
return new byte[0];
final byte[] author_byte = Base64Order.enhancedCoder.decode(author);
if (author_byte == null)
return "".getBytes();
return new byte[0];
return author_byte;
}
private void setIp(String ip) {
@ -328,7 +348,7 @@ public class blogBoardComments {
return new byte[0];
final byte[] page_byte = Base64Order.enhancedCoder.decode(page);
if (page_byte == null)
return "".getBytes();
return new byte[0];
return page_byte;
}
/**

205
source/de/anomic/data/bookmarksDB.java
Unescape View File

@ -65,21 +65,21 @@ public class bookmarksDB {
// Declaration of Class-Attributes
// ------------------------------------
final static int SORT_ALPHA = 1;
final static int SORT_SIZE = 2;
final static int SHOW_ALL = -1;
final static String SLEEP_TIME = "3600000"; // default sleepTime: check for recrawls every hour
//final static int SORT_ALPHA = 1;
private final static int SORT_SIZE = 2;
private final static int SHOW_ALL = -1;
private final static String SLEEP_TIME = "3600000"; // default sleepTime: check for recrawls every hour
// bookmarks
MapHeap bookmarks;
private MapHeap bookmarks;
// tags
ConcurrentHashMap<String, Tag> tags;
private ConcurrentHashMap<String, Tag> tags;
// autoReCrawl
private final BusyThread autoReCrawl;
BookmarkDate dates;
private BookmarkDate dates;
// ------------------------------------
// bookmarksDB's class constructor
@ -99,15 +99,15 @@ public class bookmarksDB {
final Iterator<Bookmark> it = new bookmarkIterator(true);
Bookmark bookmark;
Tag tag;
String[] tags;
String[] tagArray;
while(it.hasNext()){
bookmark = it.next();
tags = BookmarkHelper.cleanTagsString(bookmark.getTagsString() + bookmark.getFoldersString()).split(",");
tagArray = BookmarkHelper.cleanTagsString(bookmark.getTagsString() + bookmark.getFoldersString()).split(",");
tag = null;
for (int i = 0; i < tags.length; i++) {
tag = getTag(BookmarkHelper.tagHash(tags[i]));
for (final String element : tagArray) {
tag = getTag(BookmarkHelper.tagHash(element));
if (tag == null) {
tag = new Tag(tags[i]);
tag = new Tag(element);
}
tag.addUrl(bookmark.getUrlHash());
putTag(tag);
@ -122,9 +122,9 @@ public class bookmarksDB {
if (!datesExisted) this.dates.init(new bookmarkIterator(true));
// autoReCrawl
Switchboard sb = Switchboard.getSwitchboard();
final Switchboard sb = Switchboard.getSwitchboard();
this.autoReCrawl = new InstantBusyThread(this, "autoReCrawl", null, null, Long.MIN_VALUE, Long.MAX_VALUE, Long.MIN_VALUE, Long.MAX_VALUE);
long sleepTime = Long.parseLong(sb.getConfig("autoReCrawl_idlesleep" , SLEEP_TIME));
final long sleepTime = Long.parseLong(sb.getConfig("autoReCrawl_idlesleep" , SLEEP_TIME));
sb.deployThread("autoReCrawl", "autoReCrawl Scheduler", "simple scheduler for automatic re-crawls of bookmarked urls", null, autoReCrawl, 120000,
sleepTime, sleepTime, Long.parseLong(sb.getConfig("autoReCrawl_memprereq" , "-1"))
);
@ -148,14 +148,14 @@ public class bookmarksDB {
public boolean autoReCrawl() {
// read crontab
File f = new File (Switchboard.getSwitchboard().getRootPath(),"DATA/SETTINGS/autoReCrawl.conf");
final File file = new File (Switchboard.getSwitchboard().getRootPath(),"DATA/SETTINGS/autoReCrawl.conf");
String s;
try {
BufferedReader in = new BufferedReader(new InputStreamReader(new FileInputStream(f)));
Log.logInfo("BOOKMARKS", "autoReCrawl - reading schedules from " + f);
final BufferedReader in = new BufferedReader(new InputStreamReader(new FileInputStream(file)));
Log.logInfo("BOOKMARKS", "autoReCrawl - reading schedules from " + file);
while( null != (s = in.readLine()) ) {
if (s.length() > 0 && s.charAt(0) != '#') {
String parser[] = s.split("\t");
final String parser[] = s.split("\t");
if (parser.length == 13) {
folderReCrawl(Long.parseLong(parser[0]), parser[1], parser[2], Integer.parseInt(parser[3]), Long.parseLong(parser[4]),
Integer.parseInt(parser[5]), Integer.parseInt(parser[6]), Boolean.parseBoolean(parser[7]),
@ -178,13 +178,14 @@ public class bookmarksDB {
} catch( FileNotFoundException ex ) {
try {
Log.logInfo("BOOKMARKS", "autoReCrawl - creating new autoReCrawl.conf");
File inputFile = new File(Switchboard.getSwitchboard().getRootPath(),"defaults/autoReCrawl.conf");
File outputFile = new File(Switchboard.getSwitchboard().getRootPath(),"DATA/SETTINGS/autoReCrawl.conf");
FileReader i = new FileReader(inputFile);
FileWriter o = new FileWriter(outputFile);
final File inputFile = new File(Switchboard.getSwitchboard().getRootPath(),"defaults/autoReCrawl.conf");
final File outputFile = new File(Switchboard.getSwitchboard().getRootPath(),"DATA/SETTINGS/autoReCrawl.conf");
final FileReader i = new FileReader(inputFile);
final FileWriter o = new FileWriter(outputFile);
int c;
while ((c = i.read()) != -1)
while ((c = i.read()) != -1) {
o.write(c);
}
i.close();
o.close();
autoReCrawl();
@ -197,7 +198,7 @@ public class bookmarksDB {
return false;
}
} catch( Exception ex ) {
Log.logSevere("BOOKMARKS", "autoReCrawl - error reading " + f, ex);
Log.logSevere("BOOKMARKS", "autoReCrawl - error reading " + file, ex);
return false;
}
return true;
@ -207,30 +208,30 @@ public class bookmarksDB {
int crawlingDomFilterDepth, int crawlingDomMaxPages, boolean crawlingQ, boolean indexText, boolean indexMedia,
boolean crawlOrder, boolean xsstopw, boolean storeHTCache, int cacheStrategy) {
Switchboard sb = Switchboard.getSwitchboard();
Iterator<String> bit=getBookmarksIterator(folder, true);
final Switchboard sb = Switchboard.getSwitchboard();
final Iterator<String> bit = getBookmarksIterator(folder, true);
Log.logInfo("BOOKMARKS", "autoReCrawl - processing: "+folder);
boolean xdstopw = xsstopw;
boolean xpstopw = xsstopw;
final boolean xdstopw = xsstopw;
final boolean xpstopw = xsstopw;
while(bit.hasNext()) {
Bookmark bm = getBookmark(bit.next());
long sleepTime = Long.parseLong(sb.getConfig("autoReCrawl_idlesleep" , SLEEP_TIME));
long interTime = (System.currentTimeMillis()-bm.getTimeStamp())%schedule;
final Bookmark bm = getBookmark(bit.next());
final long sleepTime = Long.parseLong(sb.getConfig("autoReCrawl_idlesleep" , SLEEP_TIME));
final long interTime = (System.currentTimeMillis()-bm.getTimeStamp())%schedule;
Date date=new Date(bm.getTimeStamp());
final Date date = new Date(bm.getTimeStamp());
Log.logInfo("BOOKMARKS", "autoReCrawl - checking schedule for: "+"["+DateFormatter.formatISO8601(date)+"] "+bm.getUrl());
if (interTime >= 0 && interTime < sleepTime) {
try {
int pos = 0;
// set crawlingStart to BookmarkUrl
String crawlingStart = bm.getUrl();
final String crawlingStart = bm.getUrl();
String newcrawlingMustMatch = crawlingfilter;
DigestURI crawlingStartURL = new DigestURI(crawlingStart, null);
final DigestURI crawlingStartURL = new DigestURI(crawlingStart, null);
// set the crawling filter
if (newcrawlingMustMatch.length() < 2) newcrawlingMustMatch = ".*"; // avoid that all urls are filtered out if bad value was submitted
@ -245,14 +246,15 @@ public class bookmarksDB {
// check if the crawl filter works correctly
Pattern.compile(newcrawlingMustMatch);
byte[] urlhash = crawlingStartURL.hash();
final byte[] urlhash = crawlingStartURL.hash();
sb.indexSegments.urlMetadata(Segments.Process.LOCALCRAWLING).remove(urlhash);
sb.crawlQueues.noticeURL.removeByURLHash(urlhash);
sb.crawlQueues.errorURL.remove(urlhash);
// stack url
sb.crawler.profilesPassiveCrawls.removeEntry(crawlingStartURL.hash()); // if there is an old entry, delete it
CrawlProfile.entry pe = sb.crawler.profilesActiveCrawls.newEntry(
final CrawlProfile.entry pe = sb.crawler.profilesActiveCrawls.newEntry(
folder+"/"+crawlingStartURL, crawlingStartURL,
newcrawlingMustMatch,
CrawlProfile.MATCH_BAD_URL,
@ -295,7 +297,6 @@ public class bookmarksDB {
} catch (MalformedURLException e1) {}
} // if
} // while(bit.hasNext())
return;
} // } autoReCrawl()
// -----------------------------------------------------------
@ -323,7 +324,7 @@ public class bookmarksDB {
}
}
public String addBookmark(final Bookmark bookmark){
saveBookmark(bookmark);
this.saveBookmark(bookmark);
return bookmark.getUrlHash();
}
@ -331,8 +332,7 @@ public class bookmarksDB {
public Bookmark getBookmark(final String urlHash){
try {
final Map<String, String> map = bookmarks.get(urlHash);
if (map == null) return null;
return new Bookmark(map);
return (map == null) ? new Bookmark(map) : null;
} catch (final IOException e) {
return null;
}
@ -341,9 +341,9 @@ public class bookmarksDB {
public boolean removeBookmark(final String urlHash){
final Bookmark bookmark = getBookmark(urlHash);
if (bookmark == null) return false; //does not exist
final Set<String> tags = bookmark.getTags();
final Set<String> tagSet = bookmark.getTags();
bookmarksDB.Tag tag=null;
final Iterator<String> it=tags.iterator();
final Iterator<String> it=tagSet.iterator();
while(it.hasNext()){
tag=getTag(BookmarkHelper.tagHash(it.next()));
if(tag!=null){
@ -442,7 +442,7 @@ public class bookmarksDB {
}
private Iterator<Tag> getTagIterator(final boolean priv, final int c) {
final TreeSet<Tag> set=new TreeSet<Tag>((c == SORT_SIZE) ? tagSizeComparator : tagComparator);
final Set<Tag> set = new TreeSet<Tag>((c == SORT_SIZE) ? tagSizeComparator : tagComparator);
final Iterator<Tag> it = this.tags.values().iterator();
Tag tag;
while (it.hasNext()) {
@ -474,11 +474,11 @@ public class bookmarksDB {
final Iterator<String> bit=getBookmarksIterator(tagName, priv);
Bookmark bm;
Tag tag;
Set<String> tags;
Set<String> tagSet;
while(bit.hasNext()){
bm=getBookmark(bit.next());
tags = bm.getTags();
it = tags.iterator();
tagSet = bm.getTags();
it = tagSet.iterator();
while (it.hasNext()) {
tag=getTag(BookmarkHelper.tagHash(it.next()) );
if(priv ||tag.hasPublicItems()){
@ -512,14 +512,14 @@ public class bookmarksDB {
if (oldTag != null) {
final Set<String> urlHashes = oldTag.getUrlHashes(); // preserve urlHashes of oldTag
removeTag(BookmarkHelper.tagHash(oldName)); // remove oldHash from TagsDB
final Iterator<String> it = urlHashes.iterator();
Bookmark bookmark;
Set<String> tags = new TreeSet<String>(String.CASE_INSENSITIVE_ORDER);
while (it.hasNext()) { // looping through all bookmarks which were tagged with oldName
bookmark = getBookmark(it.next());
tags = bookmark.getTags();
tags.remove(oldName);
bookmark.setTags(tags, true); // might not be needed, but doesn't hurt
Set<String> tagSet = new TreeSet<String>(String.CASE_INSENSITIVE_ORDER);
for (String urlHash : urlHashes) { // looping through all bookmarks which were tagged with oldName
bookmark = getBookmark(urlHash);
tagSet = bookmark.getTags();
tagSet.remove(oldName);
bookmark.setTags(tagSet, true); // might not be needed, but doesn't hurt
if(!newName.equals("")) bookmark.addTag(newName);
saveBookmark(bookmark);
}
@ -529,10 +529,10 @@ public class bookmarksDB {
}
public void addTag(final String selectTag, final String newTag) {
final Iterator<String> it = getTag(BookmarkHelper.tagHash(selectTag)).getUrlHashes().iterator(); // get urlHashes for selectTag
Bookmark bookmark;
while (it.hasNext()) { // looping through all bookmarks which were tagged with selectTag
bookmark = getBookmark(it.next());
for (final String urlHash : getTag(BookmarkHelper.tagHash(selectTag)).getUrlHashes()) { // looping through all bookmarks which were tagged with selectTag
bookmark = getBookmark(urlHash);
bookmark.addTag(newTag);
saveBookmark(bookmark);
}
@ -555,11 +555,12 @@ public class bookmarksDB {
public Tag(final String hash, final Map<String, String> map){
tagHash = hash;
mem = map;
if(mem.containsKey(URL_HASHES))
if (mem.containsKey(URL_HASHES)) {
urlHashes = listManager.string2set(mem.get(URL_HASHES));
else
} else {
urlHashes = new HashSet<String>();
}
}
public Tag(final String name, final HashSet<String> entries){
tagHash = BookmarkHelper.tagHash(name);
@ -570,12 +571,9 @@ public class bookmarksDB {
}
public Tag(final String name){
tagHash=BookmarkHelper.tagHash(name);
mem=new HashMap<String, String>();
//mem.put(URL_HASHES, "");
urlHashes=new HashSet<String>();
mem.put(TAG_NAME, name);
this(name, new HashSet<String>());
}
public Map<String, String> getMap(){
mem.put(URL_HASHES, listManager.collection2string(this.urlHashes));
return mem;
@ -615,11 +613,7 @@ public class bookmarksDB {
}
public boolean hasPublicItems(){
final Iterator<String> it=getBookmarksIterator(this.getTagName(), false);
if(it.hasNext()){
return true;
}
return false;
return getBookmarksIterator(this.getTagName(), false).hasNext();
}
public void addUrl(final String urlHash){
@ -651,16 +645,28 @@ public class bookmarksDB {
private String urlHash;
private Set<String> tagNames;
private long timestamp;
Map<String, String> entry;
private Map<String, String> entry;
public Bookmark(final String urlHash, final Map<String, String> map) {
this.entry = map;
entry = map;
this.urlHash = urlHash;
tagNames = new TreeSet<String>(String.CASE_INSENSITIVE_ORDER);
if (map.containsKey(BOOKMARK_TAGS)) tagNames.addAll(listManager.string2set(map.get(BOOKMARK_TAGS)));
loadTimestamp();
}
public Bookmark(final String urlHash, final String url) {
entry = new HashMap<String, String>();
this.urlHash = urlHash;
entry.put(BOOKMARK_URL, url);
tagNames = new HashSet<String>();
timestamp = System.currentTimeMillis();
}
public Bookmark(final String urlHash, final DigestURI url) {
this(urlHash, url.toNormalform(false, true));
}
public Bookmark(String url){
entry = new HashMap<String, String>();
if (!url.toLowerCase().startsWith("http://") && !url.toLowerCase().startsWith("https://")) {
@ -687,22 +693,6 @@ public class bookmarksDB {
removeBookmark(this.urlHash); //prevent empty tags
}
public Bookmark(final String urlHash, final DigestURI url) {
entry = new HashMap<String, String>();
this.urlHash=urlHash;
entry.put(BOOKMARK_URL, url.toNormalform(false, true));
tagNames=new HashSet<String>();
timestamp=System.currentTimeMillis();
}
public Bookmark(final String urlHash, final String url) {
entry = new HashMap<String, String>();
this.urlHash=urlHash;
entry.put(BOOKMARK_URL, url);
tagNames=new HashSet<String>();
timestamp=System.currentTimeMillis();
}
public Bookmark(final Map<String, String> map) throws MalformedURLException {
this(new String((new DigestURI(map.get(BOOKMARK_URL), null)).hash()), map);
}
@ -732,24 +722,26 @@ public class bookmarksDB {
public String getTagsString() {
final String s[] = listManager.collection2string(getTags()).split(",");
String tagsString="";
for (int i=0; i<s.length; i++){
if(!s[i].startsWith("/")){
tagsString += s[i]+",";
final StringBuilder stringBuilder = new StringBuilder();
for (final String element : s){
if(!element.startsWith("/")){
stringBuilder.append(element);
stringBuilder.append(",");
}
}
return tagsString;
return stringBuilder.toString();
}
public String getFoldersString(){
final String s[] = listManager.collection2string(getTags()).split(",");
String foldersString="";
for (int i=0; i<s.length; i++){
if(s[i].startsWith("/")){
foldersString += s[i]+",";
final StringBuilder stringBuilder = new StringBuilder();
for (final String element : s){
if(!element.startsWith("/")){
stringBuilder.append(element);
stringBuilder.append(",");
}
}
return foldersString;
return stringBuilder.toString();
}
public String getDescription(){
@ -834,9 +826,7 @@ public class bookmarksDB {
public void setTags(final Set<String> tags2, final boolean local){
tagNames = tags2; // TODO: check if this is safe
// tags.addAll(tags2); // in order for renameTag() to work I had to change this form 'add' to 'set'
final Iterator<String> it=tagNames.iterator();
while(it.hasNext()){
final String tagName=it.next();
for (final String tagName : tagNames) {
Tag tag = getTag(BookmarkHelper.tagHash(tagName));
if (tag == null) {
tag = new Tag(tagName);
@ -882,10 +872,10 @@ public class bookmarksDB {
}
public Bookmark next() {
try {
if (hasNext()) {
String s = new String(this.bookmarkIter.next());
return getBookmark(s);
} catch (final kelondroException e) {
} else {
//resetDatabase();
return null;
}
@ -913,8 +903,9 @@ public class bookmarksDB {
public int compare(final String obj1, final String obj2) {
final Bookmark bm1 = getBookmark(obj1);
final Bookmark bm2 = getBookmark(obj2);
if(bm1==null || bm2==null)
if (bm1 == null || bm2 == null) {
return 0; //XXX: i think this should not happen? maybe this needs further tracing of the bug
}
if (this.newestFirst){
if (bm2.getTimeStamp() - bm1.getTimeStamp() >0) return 1;
return -1;
@ -951,9 +942,13 @@ public class bookmarksDB {
private static final long serialVersionUID = 4149185397646373251L;
public int compare(final Tag obj1, final Tag obj2) {
if (obj1.size() < obj2.size()) return 1;
else if (obj1.getTagName().equals(obj2.getTagName())) return 0;
else return -1;
if (obj1.size() < obj2.size()) {
return 1;
} else if (obj1.getTagName().equals(obj2.getTagName())) {
return 0;
} else {
return -1;
}
}
}

82
source/de/anomic/data/diff.java
Unescape View File

@ -39,42 +39,42 @@ import net.yacy.document.parser.html.CharacterCoding;
public class diff {
private final ArrayList <Part> parts = new ArrayList<Part>();
protected final Object[] o;
protected final Object[] n;
protected final Object[] original;
protected final Object[] changed;
/**
* @param o the original <code>String</code>
* @param n the new <code>String</code>
* @param original the original <code>String</code>
* @param changed the new <code>String</code>
* @throws NullPointerException if one of the arguments is <code>null</code>
*/
public diff(final String o, final String n) {
this(o, n, 1);
public diff(final String original, final String changed) {
this(original, changed, 1);
}
/**
* @param o the original <code>String</code>
* @param n the new <code>String</code>
* @param original the original <code>String</code>
* @param changed the new <code>String</code>
* @param minConsecutive the minimum number of consecutive equal characters in
* both Strings. Smaller seperations will only be performed on the end of either
* String if needed
* @throws NullPointerException if <code>o</code> or <code>n</code> is
* @throws NullPointerException if <code>original</code> or <code>changed</code> is
* <code>null</code>
*/
public diff(final String o, final String n, final int minConsecutive) {
if (o == null || n == null) throw new NullPointerException("neither o nor n must be null");
this.o = new Comparable[o.length()];
for (int i=0; i<o.length(); i++)
this.o[i] = Character.valueOf(o.charAt(i));
this.n = new Comparable[n.length()];
for (int i=0; i<n.length(); i++)
this.n[i] = Character.valueOf(n.charAt(i));
public diff(final String original, final String changed, final int minConsecutive) {
if (original == null || changed == null) throw new NullPointerException("input Strings must be null");
this.original = new Comparable[original.length()];
for (int i=0; i<original.length(); i++)
this.original[i] = Character.valueOf(original.charAt(i));
this.changed = new Comparable[changed.length()];
for (int i=0; i<changed.length(); i++)
this.changed[i] = Character.valueOf(changed.charAt(i));
parse((minConsecutive > 0) ? minConsecutive : 1);
}
public diff(final Object[] o, final Object[] n, final int minConsecutive) {
if (o == null || n == null) throw new NullPointerException("neither o nor n must be null");
this.o = o;
this.n = n;
public diff(final Object[] original, final Object[] changed, final int minConsecutive) {
if (original == null || changed == null) throw new NullPointerException("input Objects must be null");
this.original = original;
this.changed = changed;
parse((minConsecutive > 0) ? minConsecutive : 1);
}
@ -102,10 +102,10 @@ public class diff {
* C| | | | | | | | | | | | | | | | |#| |
* E| | |#| | | | | | | | |#| | |#| | |#|
*/
final boolean[][] matrix = new boolean[this.n.length][this.o.length];
for (int y=0; y<this.n.length; y++)
for (int x=0; x<this.o.length; x++)
matrix[y][x] = this.o[x].equals(this.n[y]);
final boolean[][] matrix = new boolean[this.changed.length][this.original.length];
for (int y=0; y<this.changed.length; y++)
for (int x=0; x<this.original.length; x++)
matrix[y][x] = this.original[x].equals(this.changed[y]);
int s = 0, t = 0;
int[] tmp;
@ -114,7 +114,7 @@ public class diff {
this.parts.add(new Part(Part.UNCHANGED, tmp[0], s = tmp[0] + tmp[2]));
t = tmp[1] + tmp[2];
}
addReplacementParts(s, t, this.o.length, this.n.length);
addReplacementParts(s, t, this.original.length, this.changed.length);
}
private void addReplacementParts(final int startx, final int starty, final int endx, final int endy) {
@ -161,12 +161,12 @@ public class diff {
/**
* @return the original <code>Object[]</code> passed to this class on instantiation
*/
public Object[] getOriginal() { return this.o; }
public Object[] getOriginal() { return this.original; }
/**
* @return the new <code>Object[]</code> passed to this class on instantiation
*/
public Object[] getNew() { return this.n; }
public Object[] getNew() { return this.changed; }
/**
* A diff is composed of different parts. Each of these parts stands for an
@ -177,11 +177,12 @@ public class diff {
*/
public Part[] getParts() { return this.parts.toArray(new Part[this.parts.size()]); }
@Override
public String toString() {
final StringBuilder sb = new StringBuilder(this.parts.size() * 20);
for (int j=0; j<this.parts.size(); j++)
sb.append(this.parts.get(j).toString()).append("\n");
return new String(sb);
for (final Part part :parts)
sb.append(part.toString()).append("\n");
return sb.toString();
}
/**
@ -221,12 +222,12 @@ public class diff {
final StringBuilder sb = new StringBuilder(this.posNew - this.posOld);
if (this.action == ADDED) {
for (int i = this.posOld; i < this.posNew; i++)
sb.append(diff.this.n[i]);
sb.append(diff.this.changed[i]);
} else {
for (int i = this.posOld; i < this.posNew; i++)
sb.append(diff.this.o[i]);
sb.append(diff.this.original[i]);
}
return new String(sb);
return sb.toString();
}
/**
@ -237,6 +238,7 @@ public class diff {
* <dt>deleted</dt><dd>"<code>- STRING</code>"</dd>
* </dl>
*/
@Override
public String toString() {
return ((this.action == UNCHANGED) ? " " :
(this.action == ADDED) ? "+" : "-") + " " + getString();
@ -246,21 +248,21 @@ public class diff {
public static String toHTML(final diff[] diffs) {
final StringBuilder sb = new StringBuilder(diffs.length * 60);
diff.Part[] ps;
for (int i=0; i<diffs.length; i++) {
for (diff d : diffs) {
sb.append("<p class=\"diff\">\n");
ps = diffs[i].getParts();
for (int j=0; j<ps.length; j++) {
ps = d.getParts();
for (diff.Part part :ps) {
sb.append("<span\nclass=\"");
switch (ps[j].getAction()) {
switch (part.getAction()) {
case diff.Part.UNCHANGED: sb.append("unchanged"); break;
case diff.Part.ADDED: sb.append("added"); break;
case diff.Part.DELETED: sb.append("deleted"); break;
}
sb.append("\">").append(CharacterCoding.unicode2html(ps[j].getString(), true).replaceAll("\n", "<br />"));
sb.append("\">").append(CharacterCoding.unicode2html(part.getString(), true).replaceAll("\n", "<br />"));
sb.append("</span>");
}
sb.append("</p>");
}
return new String(sb);
return sb.toString();
}
}

24
source/de/anomic/data/listManager.java
Unescape View File

@ -39,6 +39,7 @@ import net.yacy.repository.BlacklistFile;
import de.anomic.search.SearchEventCache;
import de.anomic.search.Switchboard;
import java.util.List;
// The Naming of the functions is a bit strange...
@ -101,17 +102,14 @@ public class listManager {
* "listName", <code>false</code> otherwise.
*/
public static boolean listSetContains(final String setName, final String listName) {
final Set<String> Lists = getListSet(setName);
return Lists.contains(listName);
return getListSet(setName).contains(listName);
}
//================general Lists==================
public static String getListString(final String filename, final boolean withcomments) {
final File listFile = new File(listsPath ,filename);
return FileUtils.getListString(listFile, withcomments);
return FileUtils.getListString(new File(listsPath ,filename), withcomments);
}
//================Helper functions for collection conversion==================
@ -142,15 +140,15 @@ public class listManager {
* @see listManager#string2vector(String)
*/
public static ArrayList<String> string2arraylist(final String string){
ArrayList<String> l;
ArrayList<String> list;
if (string != null && string.length() > 0) {
l = new ArrayList<String>(Arrays.asList(string.split(",")));
list = new ArrayList<String>(Arrays.asList(string.split(",")));
} else {
l = new ArrayList<String>();
list = new ArrayList<String>();
}
return l;
return list;
}
/**
@ -199,12 +197,12 @@ public class listManager {
final String supportedBlacklistTypesStr = Blacklist.BLACKLIST_TYPES_STRING;
final String[] supportedBlacklistTypes = supportedBlacklistTypesStr.split(",");
final ArrayList<BlacklistFile> blacklistFiles = new ArrayList<BlacklistFile>(supportedBlacklistTypes.length);
for (int i=0; i < supportedBlacklistTypes.length; i++) {
final List<BlacklistFile> blacklistFiles = new ArrayList<BlacklistFile>(supportedBlacklistTypes.length);
for (String supportedBlacklistType : supportedBlacklistTypes) {
final BlacklistFile blFile = new BlacklistFile(
switchboard.getConfig(
supportedBlacklistTypes[i] + ".BlackLists", switchboard.getConfig("BlackLists.DefaultList", "url.default.black")),
supportedBlacklistTypes[i]);
supportedBlacklistType + ".BlackLists", switchboard.getConfig("BlackLists.DefaultList", "url.default.black")),
supportedBlacklistType);
blacklistFiles.add(blFile);
}

32
source/de/anomic/data/messageBoard.java
Unescape View File

@ -43,13 +43,13 @@ public class messageBoard {
private static final int categoryLength = 12;
private static final String dateFormat = "yyyyMMddHHmmss";
static SimpleDateFormat SimpleFormatter = new SimpleDateFormat(dateFormat, Locale.US);
private static final SimpleDateFormat SimpleFormatter = new SimpleDateFormat(dateFormat, Locale.US);
static {
SimpleFormatter.setTimeZone(TimeZone.getTimeZone("GMT"));
}
MapHeap database = null;
private MapHeap database = null;
private int sn = 0;
public messageBoard(final File path) throws IOException {
@ -104,20 +104,12 @@ public class messageBoard {
if (key.length() > categoryLength) key = key.substring(0, categoryLength);
while (key.length() < categoryLength) key += "_";
key += dateString() + snString();
if ((authorName == null) || (authorName.length() == 0)) authorName = "anonymous";
record.put("author", authorName);
if ((recName == null) || (recName.length() == 0)) recName = "anonymous";
record.put("recipient", recName);
if (authorHash == null) authorHash = "";
record.put("ahash", authorHash);
if (recHash == null) recHash = "";
record.put("rhash", recHash);
if (subject == null) subject = "";
record.put("subject", subject);
if (message == null)
record.put("message", "");
else
record.put("message", Base64Order.enhancedCoder.encode(message));
record.put("author", ((authorName == null) || (authorName.length() == 0)) ? authorName : "anonymous");
record.put("recipient", ((recName == null) || (recName.length() == 0)) ? recName : "anonymous");
record.put("ahash", (authorHash == null) ? authorHash : "");
record.put("rhash", (recHash == null) ? recHash : "");
record.put("subject", (subject == null) ? subject : "");
record.put("message", (message == null) ? "" : Base64Order.enhancedCoder.encode(message));
record.put("read", "false");
}
@ -158,13 +150,11 @@ public class messageBoard {
public String authorHash() {
final String a = record.get("ahash");
if (a == null) return null;
return a;
}
public String recipientHash() {
final String a = record.get("rhash");
if (a == null) return null;
return a;
}
@ -224,9 +214,9 @@ public class messageBoard {
public class catIter implements Iterator<String> {
Iterator<byte[]> allIter = null;
String nextKey = null;
String category = "";
private Iterator<byte[]> allIter = null;
private String nextKey = null;
private String category = "";
public catIter(final String category, final boolean up) throws IOException {
this.allIter = database.keys(up, false);

128
source/de/anomic/data/translator.java
Unescape View File

@ -39,12 +39,9 @@ import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.Hashtable;
import java.util.Iterator;
import java.util.Vector;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@ -53,6 +50,9 @@ import net.yacy.kelondro.util.FileUtils;
import net.yacy.kelondro.util.Formatter;
import de.anomic.server.serverSwitch;
import java.util.List;
import java.util.Map;
import java.util.Set;
/**
* Wordlist based translator
@ -60,16 +60,14 @@ import de.anomic.server.serverSwitch;
* Uses a Property like file with phrases or single words to translate a string or a file
* */
public class translator {
public static String translate(final String source, final Hashtable<String, String> translationList){
final Enumeration<String> keys = translationList.keys();
public static String translate(final String source, final Map<String, String> translationTable){
final Set<String> keys = translationTable.keySet();
String result = source;
String key = "";
while(keys.hasMoreElements()){
key = keys.nextElement();
for (String key : keys){
final Pattern pattern = Pattern.compile(key);
final Matcher matcher = pattern.matcher(result);
if (matcher.find()) {
result = matcher.replaceAll(translationList.get(key));
result = matcher.replaceAll(translationTable.get(key));
} else {
//Filename not available, but it will be printed in Log
//after all untranslated Strings as "Translated file: "
@ -78,27 +76,24 @@ public class translator {
}
return result;
}
/**
* Load multiple translationLists from one File. Each List starts with #File: relative/path/to/file
* @param translationFile the File, which contains the Lists
* @return a Hashtable, which contains for each File a Hashtable with translations.
*/
public static Hashtable<String, Hashtable<String, String>> loadTranslationsLists(final File translationFile){
final Hashtable<String, Hashtable<String, String>> lists = new Hashtable<String, Hashtable<String, String>>(); //list of translationLists for different files.
Hashtable<String, String> translationList = new Hashtable<String, String>(); //current Translation Table
public static Map<String, Map<String, String>> loadTranslationsLists(final File translationFile){
final Map<String, Map<String, String>> lists = new Hashtable<String, Map<String, String>>(); //list of translationLists for different files.
Map<String, String> translationList = new Hashtable<String, String>(); //current Translation Table
final ArrayList<String> list = FileUtils.getListArray(translationFile);
final Iterator<String> it = list.iterator();
String line = "";
String[] splitted;
final List<String> list = FileUtils.getListArray(translationFile);
String forFile = "";
while(it.hasNext()){
line = it.next();
for (final String line : list){
if (line.length() == 0 || line.charAt(0) != '#'){
splitted = line.split("==", 2);
if (splitted.length == 2) {
translationList.put(splitted[0], splitted[1]);
String[] split = line.split("==", 2);
if (split.length == 2) {
translationList.put(split[0], split[1]);
//}else{ //Invalid line
}
} else if (line.startsWith("#File: ")) {
@ -122,122 +117,123 @@ public class translator {
}
public static boolean translateFile(final File sourceFile, final File destFile, final File translationFile){
final Hashtable<String, String> translationList = loadTranslationsLists(translationFile).get(sourceFile.getName());
return translateFile(sourceFile, destFile, translationList);
return translateFile(sourceFile, destFile, loadTranslationsLists(translationFile).get(sourceFile.getName()));
}
public static boolean translateFile(final File sourceFile, final File destFile, final Hashtable<String, String> translationList){
public static boolean translateFile(final File sourceFile, final File destFile, final Map<String, String> translationList){
String content = "";
String line = "";
StringBuilder content = new StringBuilder();
BufferedReader br = null;
try{
br = new BufferedReader(new InputStreamReader(new FileInputStream(sourceFile),"UTF-8"));
String line = null;
while( (line = br.readLine()) != null){
content += line + de.anomic.server.serverCore.CRLF_STRING;
content.append(line).append(de.anomic.server.serverCore.CRLF_STRING);
}
br.close();
} catch(final IOException e) {
return false;
} finally {
if (br!=null)try{br.close();}catch(final Exception e){}
if (br !=null) {
try {
br.close();
} catch (final Exception e) {}
}
}
content = translate(content, translationList);
content = new StringBuilder(translate(content.toString(), translationList));
BufferedWriter bw = null;
try{
bw = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(destFile),"UTF-8"));
bw.write(content);
bw.write(content.toString());
bw.close();
}catch(final IOException e){
return false;
} finally {
if(bw!=null)try{bw.close();}catch(final Exception e){}
if(bw != null) {
try {
bw.close();
} catch (final Exception e) {}
}
}
return true;
}
public static boolean translateFiles(final File sourceDir, final File destDir, final File baseDir, final File translationFile, final String extensions){
final Hashtable<String, Hashtable<String, String>> translationLists = loadTranslationsLists(translationFile);
return translateFiles(sourceDir, destDir, baseDir, translationLists, extensions);
return translateFiles(sourceDir, destDir, baseDir, loadTranslationsLists(translationFile), extensions);
}
public static boolean translateFiles(final File sourceDir, final File destDir, final File baseDir, final Hashtable<String, Hashtable<String, String>> translationLists, final String extensions){
public static boolean translateFiles(final File sourceDir, final File destDir, final File baseDir, final Map<String, Map<String, String>> translationLists, final String extensions){
destDir.mkdirs();
final File[] sourceFiles = sourceDir.listFiles();
final Vector<String> exts=listManager.string2vector(extensions);
final List<String> exts = listManager.string2vector(extensions);
boolean rightExtension;
Iterator<String> it;
String relativePath;
for(int i=0;i<sourceFiles.length;i++){
it=exts.iterator();
for(final File sourceFile : sourceFiles){
rightExtension=false;
while(it.hasNext()){
if(sourceFiles[i].getName().endsWith(it.next())){
for(final String ext : exts){
if(sourceFile.getName().endsWith(ext)){
rightExtension=true;
break;
}
}
if (rightExtension) {
try {
relativePath=sourceFiles[i].getAbsolutePath().substring(baseDir.getAbsolutePath().length()+1); //+1 to get the "/"
relativePath=sourceFile.getAbsolutePath().substring(baseDir.getAbsolutePath().length()+1); //+1 to get the "/"
relativePath = relativePath.replace(File.separatorChar, '/');
} catch (final IndexOutOfBoundsException e){
Log.logSevere("TRANSLATOR", "Error creating relative Path for "+sourceFiles[i].getAbsolutePath());
Log.logSevere("TRANSLATOR", "Error creating relative Path for "+sourceFile.getAbsolutePath());
relativePath = "wrong path"; //not in translationLists
}
if (translationLists.containsKey(relativePath)) {
Log.logInfo("TRANSLATOR", "Translating file: "+ relativePath);
if(!translateFile(
sourceFiles[i]
, new File(destDir, sourceFiles[i].getName().replace('/', File.separatorChar))
, translationLists.get(relativePath))){
sourceFile,
new File(destDir, sourceFile.getName().replace('/', File.separatorChar)),
translationLists.get(relativePath)))
{
Log.logSevere("TRANSLATOR", "File error while translating file "+relativePath);
}
//}else{
//serverLog.logInfo("TRANSLATOR", "No translation for file: "+relativePath);
}
}
}
return true;
}
public static boolean translateFilesRecursive(final File sourceDir, final File destDir, final File translationFile, final String extensions, final String notdir){
final ArrayList<File> dirList=FileUtils.getDirsRecursive(sourceDir, notdir);
final List<File> dirList=FileUtils.getDirsRecursive(sourceDir, notdir);
dirList.add(sourceDir);
final Iterator<File> it=dirList.iterator();
File file=null;
File file2=null;
while(it.hasNext()){
file=it.next();
//cuts the sourcePath and prepends the destPath
file2=new File(destDir, file.getPath().substring(sourceDir.getPath().length()));
for (final File file : dirList){
if(file.isDirectory() && !file.getName().equals(notdir)){
//cuts the sourcePath and prepends the destPath
File file2 = new File(destDir, file.getPath().substring(sourceDir.getPath().length()));
translateFiles(file, file2, sourceDir, translationFile, extensions);
}
}
return true;
}
public static HashMap<String, String> langMap(final serverSwitch env) {
public static Map<String, String> langMap(final serverSwitch env) {
final String[] ms = env.getConfig("locale.lang", "").split(",");
final HashMap<String, String> map = new HashMap<String, String>();
int p;
for (int i = 0; i < ms.length; i++) {
p = ms[i].indexOf('/');
final Map<String, String> map = new HashMap<String, String>();
for (final String element : ms) {
int p = element.indexOf('/');
if (p > 0)
map.put(ms[i].substring(0, p), ms[i].substring(p + 1));
map.put(element.substring(0, p), element.substring(p + 1));
}
return map;
}
public static boolean changeLang(final serverSwitch env, final String langPath, final String lang) {
boolean ret = false;
if ((lang.equals("default")) || (lang.equals("default.lng"))) {
env.setConfig("locale.language", "default");
return true;
}
ret = true;
} else {
final String htRootPath = env.getConfig("htRootPath", "htroot");
final File sourceDir = new File(env.getRootPath(), htRootPath);
final File destDir = new File(env.getConfigPath("locale.translated_html","DATA/LOCALE/htroot"), lang.substring(0, lang.length() - 4));// cut
@ -247,8 +243,7 @@ public class translator {
final File translationFile = new File(langPath, lang);
//if (translator.translateFiles(sourceDir, destDir, translationFile, "html")) {
if(translator.translateFilesRecursive(sourceDir, destDir,
translationFile, "html,template,inc", "locale")){
if (translator.translateFilesRecursive(sourceDir, destDir, translationFile, "html,template,inc", "locale")) {
env.setConfig("locale.language", lang.substring(0, lang.length() - 4));
Formatter.setLocale(env.getConfig("locale.language", "en"));
try {
@ -258,8 +253,9 @@ public class translator {
} catch (final IOException e) {
// Error
}
return true;
ret = true;
}
return false;
}
return ret;
}
}

100
source/de/anomic/data/userDB.java
Unescape View File

@ -49,13 +49,13 @@ import de.anomic.http.server.RequestHeader;
public final class userDB {
public static final int USERNAME_MAX_LENGTH = 128;
public static final int USERNAME_MIN_LENGTH = 4;
private static final int USERNAME_MAX_LENGTH = 128;
private static final int USERNAME_MIN_LENGTH = 4;
MapHeap userTable;
private MapHeap userTable;
private final File userTableFile;
HashMap<String, String> ipUsers = new HashMap<String, String>();
HashMap<String, Object> cookieUsers = new HashMap<String, Object>();
private final Map<String, String> ipUsers = new HashMap<String, String>();
private final Map<String, Object> cookieUsers = new HashMap<String, Object>();
public userDB(final File userTableFile) throws IOException {
this.userTableFile = userTableFile;
@ -104,7 +104,7 @@ public final class userDB {
return new Entry(userName, record);
}
public Entry createEntry(final String userName, final HashMap<String, String> userProps) throws IllegalArgumentException{
public Entry createEntry(final String userName, final Map<String, String> userProps) throws IllegalArgumentException{
final Entry entry = new Entry(userName,userProps);
return entry;
}
@ -123,39 +123,31 @@ public final class userDB {
* use a ProxyAuth String to authenticate a user
* @param auth a base64 Encoded String, which contains "username:pw".
*/
public Entry proxyAuth(String auth) {
if(auth==null)
public Entry proxyAuth(final String auth) {
if(auth == null) {
return null;
}
Entry entry = null;
auth=auth.trim().substring(6);
try{
auth=Base64Order.standardCoder.decodeString(auth);
}catch(final RuntimeException e){} //no valid Base64
final String[] tmp=auth.split(":");
final String[] tmp= Base64Order.standardCoder.decodeString(auth.trim().substring(6)).split(":");
if(tmp.length == 2){
entry = this.passwordAuth(tmp[0], tmp[1]);
// if(entry != null){
//return entry;
// }else{ //wrong/no auth, so auth is removed from browser
/*FIXME: This cannot work
try{
entry.setProperty(Entry.LOGGED_OUT, "false");
}catch(IOException e){}*/
// }
return entry;
}
return null;
return entry;
}
public Entry getUser(final RequestHeader header){
return getUser(header.get(RequestHeader.AUTHORIZATION), header.getHeaderCookies());
}
public Entry getUser(final String auth, final String cookies){
Entry entry=null;
if(auth != null)
if(auth != null) {
entry=proxyAuth(auth);
if(entry == null)
}
if(entry == null) {
entry=cookieAuth(cookies);
}
return entry;
}
/**
@ -166,13 +158,15 @@ public final class userDB {
*/
public boolean hasAdminRight(final String auth, final String cookies) {
final Entry entry = getUser(auth, cookies);
if (entry != null)
return entry.hasAdminRight();
if (entry != null) {
return entry.hasRight(Entry.ADMIN_RIGHT);
}
// else if(entry != null && cookieAdminAuth(cookies))
// return entry.hasAdminRight();
else
else {
return false;
}
}
/**
* use a ProxyAuth String to authenticate a user and save the ip/username for ipAuth
@ -188,6 +182,7 @@ public final class userDB {
this.ipUsers.put(ip, entry.getUserName());
return entry;
}
/**
* authenticate a user by ip, if he had used proxyAuth in the last 10 Minutes
* @param ip the IP of the User
@ -221,6 +216,7 @@ public final class userDB {
}
return null;
}
public Entry passwordAuth(final String user, final String password, final String ip){
final Entry entry=passwordAuth(user, password);
if(entry == null){
@ -230,6 +226,7 @@ public final class userDB {
this.ipUsers.put(ip, entry.getUserName()); //XXX: This is insecure. TODO: use cookieauth
return entry;
}
public Entry md5Auth(final String user, final String md5) {
final Entry entry=this.getEntry(user);
if (entry != null && entry.getMD5EncodedUserPwd().equals(md5)) {
@ -245,6 +242,7 @@ public final class userDB {
}
return null;
}
public Entry cookieAuth(final String cookieString){
final String token = getLoginToken(cookieString);
if (cookieUsers.containsKey(token)) {
@ -254,21 +252,25 @@ public final class userDB {
}
return null;
}
public boolean cookieAdminAuth(final String cookieString){
final String token = getLoginToken(cookieString);
if (cookieUsers.containsKey(token)) {
final Object entry = cookieUsers.get(token);
if(entry instanceof String && entry.equals("admin"))
if (entry instanceof String && entry.equals("admin")) {
return true;
}
}
return false;
}
public String getCookie(final Entry entry){
final Random r = new Random();
final String token = Long.toString(Math.abs(r.nextLong()), 36);
cookieUsers.put(token, entry);
return token;
}
public String getAdminCookie(){
final Random r = new Random();
final String token = Long.toString(Math.abs(r.nextLong()), 36);
@ -278,15 +280,15 @@ public final class userDB {
public static String getLoginToken(final String cookies){
final String[] cookie = cookies.split(";"); //TODO: Mozilla uses "; "
String[] pair;
for(int i=0;i<cookie.length;i++){
pair=cookie[i].split("=");
for (String c :cookie) {
String[] pair = c.split("=");
if (pair[0].trim().equals("login")) {
return pair[1].trim();
}
}
return "";
}
public void adminLogout(final String logintoken){
if(cookieUsers.containsKey(logintoken)){
//XXX: We could check, if its == "admin", but we want to logout anyway.
@ -327,8 +329,8 @@ public final class userDB {
public static final int PROXY_TIMELIMIT_REACHED = 3;
// this is a simple record structure that hold all properties of a user
Map<String, String> mem;
String userName;
private Map<String, String> mem;
private String userName;
private final Calendar oldDate, newDate;
public Entry(final String userName, final Map<String, String> mem) throws IllegalArgumentException {
@ -340,12 +342,14 @@ public final class userDB {
this.userName = userName.trim();
if (this.userName.length() < USERNAME_MIN_LENGTH)
throw new IllegalArgumentException("Username to short. Length should be >= " + USERNAME_MIN_LENGTH);
throw new IllegalArgumentException("Username too short. Length should be >= " + USERNAME_MIN_LENGTH);
if (mem == null) this.mem = new HashMap<String, String>();
else this.mem = mem;
if (mem == null || !mem.containsKey(AUTHENTICATION_METHOD))this.mem.put(AUTHENTICATION_METHOD,"yacy");
if (mem == null || !mem.containsKey(AUTHENTICATION_METHOD)) {
this.mem.put(AUTHENTICATION_METHOD,"yacy");
}
this.oldDate=Calendar.getInstance();
this.newDate=Calendar.getInstance();
}
@ -433,7 +437,7 @@ public final class userDB {
public int surfRight(){
final long timeUsed=this.updateLastAccess(true);
if (!this.hasProxyRight())
if (!this.hasRight(Entry.PROXY_RIGHT))
return PROXY_NORIGHT;
if(! (this.getTimeLimit() <= 0 || ( timeUsed < this.getTimeLimit())) ){ //no timelimit or timelimit not reached
@ -441,13 +445,15 @@ public final class userDB {
}
return PROXY_ALLOK;
}
public boolean canSurf(){
if(this.surfRight()==PROXY_ALLOK) return true;
return false;
return (this.surfRight() == PROXY_ALLOK) ? true : false;
}
public long updateLastAccess(final boolean incrementTimeUsed) {
return updateLastAccess(System.currentTimeMillis(), incrementTimeUsed);
}
public long updateLastAccess(final long timeStamp, final boolean incrementTimeUsed) {
if (timeStamp < 0) throw new IllegalArgumentException();
@ -554,15 +560,18 @@ public final class userDB {
public boolean hasBookmarkRight() {
return this.hasRight(BOOKMARK_RIGHT);
}
public boolean isLoggedOut(){
return (this.mem.containsKey(LOGGED_OUT)?this.mem.get(LOGGED_OUT).equals("true"):false);
}
public void logout(final String ip, final String logintoken){
logout(ip);
if(cookieUsers.containsKey(logintoken)){
cookieUsers.remove(logintoken);
}
}
public void logout(final String ip) {
try {
setProperty(LOGGED_OUT, "true");
@ -573,19 +582,22 @@ public final class userDB {
Log.logException(e);
}
}
public void logout(){
logout("xxxxxx");
}
@Override
public String toString() {
final StringBuilder str = new StringBuilder();
str.append((this.userName==null)?"null":this.userName)
.append(": ");
str.append((this.userName==null)?"null":this.userName).append(": ");
if (this.mem != null) {
str.append(this.mem.toString());
}
return new String(str);
return str.toString();
}
}
@ -609,6 +621,7 @@ public final class userDB {
this.userIter = userDB.this.userTable.keys(up, false);
//this.nextEntry = null;
}
public boolean hasNext() {
try {
return this.userIter.hasNext();
@ -617,6 +630,7 @@ public final class userDB {
return false;
}
}
public Entry next() {
try {
return getEntry(new String(this.userIter.next()));
@ -625,6 +639,7 @@ public final class userDB {
return null;
}
}
public void remove() {
// if (this.nextEntry != null) {
// try {
@ -634,6 +649,7 @@ public final class userDB {
// resetDatabase();
// }
// }
throw new UnsupportedOperationException("Method not implemented yet.");
}
}

Write Preview
Loading…
Cancel
Save