diff --git a/htroot/Supporter.java b/htroot/Supporter.java index 60ac7ca5f..8e1433444 100644 --- a/htroot/Supporter.java +++ b/htroot/Supporter.java @@ -9,7 +9,7 @@ // $LastChangedBy$ // // LICENSE -// +// // This program is free software; you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by // the Free Software Foundation; either version 2 of the License, or @@ -45,7 +45,6 @@ import net.yacy.peers.yacyNewsPool; import net.yacy.peers.yacySeed; import net.yacy.repository.Blacklist; import net.yacy.search.Switchboard; - import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; import de.anomic.tools.crypt; @@ -56,19 +55,19 @@ public class Supporter { public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) { final Switchboard sb = (Switchboard) env; final serverObjects prop = new serverObjects(); - + final boolean authenticated = sb.adminAuthenticated(header) >= 2; final int display = ((post == null) || (!authenticated)) ? 0 : post.getInt("display", 0); prop.put("display", display); - + final boolean showScore = ((post != null) && (post.containsKey("score"))); - + // access control final boolean publicPage = sb.getConfigBool("publicSurftips", true); final boolean authorizedAccess = sb.verifyAuthentication(header, false); - + if ((publicPage) || (authorizedAccess)) { - + // read voting String hash; if ((post != null) && ((hash = post.get("voteNegative", null)) != null)) { @@ -101,7 +100,7 @@ public class Supporter { map.put("comment", post.get("comment", "")); sb.peers.newsPool.publishMyNews(sb.peers.mySeed(), yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, map); } - + // create Supporter final HashMap negativeHashes = new HashMap(); // a mapping from an url hash to Integer (count of votes) final HashMap positiveHashes = new HashMap(); // a mapping from an url hash to Integer (count of votes) @@ -114,7 +113,7 @@ public class Supporter { accumulateSupporter(sb, Supporter, ranking, rowdef, negativeHashes, positiveHashes, yacyNewsPool.INCOMING_DB); //accumulateSupporter(Supporter, ranking, rowdef, negativeHashes, positiveHashes, yacyNewsPool.OUTGOING_DB); //accumulateSupporter(Supporter, ranking, rowdef, negativeHashes, positiveHashes, yacyNewsPool.PUBLISHED_DB); - + // read out surftipp array and create property entries final Iterator k = ranking.keys(false); int i = 0; @@ -124,19 +123,19 @@ public class Supporter { while (k.hasNext()) { urlhash = k.next(); if (urlhash == null) continue; - + row = Supporter.get(urlhash); if (row == null) continue; - - url = row.getColString(0); + + url = row.getPrimaryKeyUTF8(); try { if (Switchboard.urlBlacklist.isListed(Blacklist.BLACKLIST_SURFTIPS, new DigestURI(url, urlhash.getBytes()))) continue; } catch(final MalformedURLException e) {continue;} - title = row.getColString(1); - description = row.getColString(2); + title = row.getColUTF8(1); + description = row.getColUTF8(2); if ((url == null) || (title == null) || (description == null)) continue; - refid = row.getColString(3); - voted = (sb.peers.newsPool.getSpecific(yacyNewsPool.OUTGOING_DB, yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, "refid", refid) != null) || + refid = row.getColUTF8(3); + voted = (sb.peers.newsPool.getSpecific(yacyNewsPool.OUTGOING_DB, yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, "refid", refid) != null) || (sb.peers.newsPool.getSpecific(yacyNewsPool.PUBLISHED_DB, yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, "refid", refid) != null); prop.put("supporter_results_" + i + "_authorized", authenticated ? "1" : "0"); prop.put("supporter_results_" + i + "_authorized_recommend", voted ? "0" : "1"); @@ -156,7 +155,7 @@ public class Supporter { prop.putHTML("supporter_results_" + i + "_title", (showScore) ? ("(" + ranking.get(urlhash) + ") " + title) : title); prop.putHTML("supporter_results_" + i + "_description", description); i++; - + if (i >= 50) break; } prop.put("supporter_results", i); @@ -164,14 +163,14 @@ public class Supporter { } else { prop.put("supporter", "0"); } - + return prop; } private static int timeFactor(final Date created) { return (int) Math.max(0, 10 - ((System.currentTimeMillis() - created.getTime()) / 24 / 60 / 60 / 1000)); } - + private static void accumulateVotes(final Switchboard sb, final HashMap negativeHashes, final HashMap positiveHashes, final int dbtype) { final int maxCount = Math.min(1000, sb.peers.newsPool.size(dbtype)); yacyNewsDB.Record record; @@ -180,7 +179,7 @@ public class Supporter { while ((recordIterator.hasNext()) && (j++ < maxCount)) { record = recordIterator.next(); if (record == null) continue; - + if (record.category().equals(yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD)) { final String urlhash = record.attribute("urlhash", ""); final String vote = record.attribute("vote", ""); @@ -198,7 +197,7 @@ public class Supporter { } } } - + private static void accumulateSupporter( final Switchboard sb, final HashMap Supporter, final ScoreMap ranking, final Row rowdef, @@ -215,7 +214,7 @@ public class Supporter { while ((recordIterator.hasNext()) && (j++ < maxCount)) { record = recordIterator.next(); if (record == null) continue; - + entry = null; if ((record.category().equals(yacyNewsPool.CATEGORY_PROFILE_UPDATE)) && ((seed = sb.peers.getConnected(record.originator())) != null)) { diff --git a/htroot/Surftips.java b/htroot/Surftips.java index 4188b5c43..5d93eb3c9 100644 --- a/htroot/Surftips.java +++ b/htroot/Surftips.java @@ -9,7 +9,7 @@ // $LastChangedBy$ // // LICENSE -// +// // This program is free software; you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by // the Free Software Foundation; either version 2 of the License, or @@ -44,7 +44,6 @@ import net.yacy.peers.yacyNewsPool; import net.yacy.peers.yacySeed; import net.yacy.repository.Blacklist; import net.yacy.search.Switchboard; - import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; import de.anomic.tools.crypt; @@ -55,13 +54,13 @@ public class Surftips { public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) { final Switchboard sb = (Switchboard) env; final serverObjects prop = new serverObjects(); - + final boolean authenticated = sb.adminAuthenticated(header) >= 2; final int display = ((post == null) || (!authenticated)) ? 0 : post.getInt("display", 0); prop.put("display", display); - + final boolean showScore = ((post != null) && (post.containsKey("score"))); - + // access control boolean publicPage = sb.getConfigBool("publicSurftips", true); final boolean authorizedAccess = sb.verifyAuthentication(header, false); @@ -73,9 +72,9 @@ public class Surftips { publicPage = post.get("publicPage", "0").equals("1"); sb.setConfig("publicSurftips", publicPage); } - + if ((publicPage) || (authorizedAccess)) { - + // read voting String hash; if ((post != null) && ((hash = post.get("voteNegative", null)) != null)) { @@ -108,7 +107,7 @@ public class Surftips { map.put("comment", post.get("comment", "")); sb.peers.newsPool.publishMyNews(sb.peers.mySeed(), yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, map); } - + // create surftips final HashMap negativeHashes = new HashMap(); // a mapping from an url hash to Integer (count of votes) final HashMap positiveHashes = new HashMap(); // a mapping from an url hash to Integer (count of votes) @@ -121,7 +120,7 @@ public class Surftips { accumulateSurftips(sb, surftips, ranking, rowdef, negativeHashes, positiveHashes, yacyNewsPool.INCOMING_DB); //accumulateSurftips(surftips, ranking, rowdef, negativeHashes, positiveHashes, yacyNewsPool.OUTGOING_DB); //accumulateSurftips(surftips, ranking, rowdef, negativeHashes, positiveHashes, yacyNewsPool.PUBLISHED_DB); - + // read out surftipp array and create property entries final Iterator k = ranking.keys(false); int i = 0; @@ -131,20 +130,20 @@ public class Surftips { while (k.hasNext()) { urlhash = k.next(); if (urlhash == null) continue; - + row = surftips.get(urlhash); if (row == null) continue; - - url = row.getColString(0); + + url = row.getPrimaryKeyUTF8(); try{ if(Switchboard.urlBlacklist.isListed(Blacklist.BLACKLIST_SURFTIPS ,new DigestURI(url))) continue; }catch(final MalformedURLException e){continue;}; - title = row.getColString(1); - description = row.getColString(2); + title = row.getColUTF8(1); + description = row.getColUTF8(2); if ((url == null) || (title == null) || (description == null)) continue; - refid = row.getColString(3); - voted = (sb.peers.newsPool.getSpecific(yacyNewsPool.OUTGOING_DB, yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, "refid", refid) != null) || + refid = row.getColUTF8(3); + voted = (sb.peers.newsPool.getSpecific(yacyNewsPool.OUTGOING_DB, yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, "refid", refid) != null) || (sb.peers.newsPool.getSpecific(yacyNewsPool.PUBLISHED_DB, yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, "refid", refid) != null); prop.put("surftips_results_" + i + "_authorized", (authenticated) ? "1" : "0"); prop.put("surftips_results_" + i + "_authorized_recommend", (voted) ? "0" : "1"); @@ -164,7 +163,7 @@ public class Surftips { prop.putXML("surftips_results_" + i + "_title", (showScore) ? ("(" + ranking.get(urlhash) + ") " + title) : title); prop.putHTML("surftips_results_" + i + "_description", description); i++; - + if (i >= 50) break; } prop.put("surftips_results", i); @@ -172,14 +171,14 @@ public class Surftips { } else { prop.put("surftips", "0"); } - + return prop; } private static int timeFactor(final Date created) { return (int) Math.max(0, 10 - ((System.currentTimeMillis() - created.getTime()) / 24 / 60 / 60 / 1000)); } - + private static void accumulateVotes(final Switchboard sb, final HashMap negativeHashes, final HashMap positiveHashes, final int dbtype) { final int maxCount = Math.min(1000, sb.peers.newsPool.size(dbtype)); yacyNewsDB.Record record; @@ -188,7 +187,7 @@ public class Surftips { while ((recordIterator.hasNext()) && (j++ < maxCount)) { record = recordIterator.next(); if (record == null) continue; - + if (record.category().equals(yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD)) { final String urlhash = record.attribute("urlhash", ""); final String vote = record.attribute("vote", ""); @@ -206,7 +205,7 @@ public class Surftips { } } } - + private static void accumulateSurftips( final Switchboard sb, final HashMap surftips, final ScoreMap ranking, final Row rowdef, @@ -222,7 +221,7 @@ public class Surftips { while ((recordIterator.hasNext()) && (j++ < maxCount)) { record = recordIterator.next(); if (record == null) continue; - + entry = null; if (record.category().equals(yacyNewsPool.CATEGORY_CRAWL_START)) { final String intention = record.attribute("intention", ""); @@ -236,7 +235,7 @@ public class Surftips { }); score = 2 + Math.min(10, intention.length() / 4) + timeFactor(record.created()); } - + if (record.category().equals(yacyNewsPool.CATEGORY_BOOKMARK_ADD)) { url = record.attribute("url", ""); if (url.length() < 12) continue; @@ -248,7 +247,7 @@ public class Surftips { }); score = 8 + timeFactor(record.created()); } - + if (record.category().equals(yacyNewsPool.CATEGORY_SURFTIPP_ADD)) { url = record.attribute("url", ""); if (url.length() < 12) continue; @@ -260,7 +259,7 @@ public class Surftips { }); score = 5 + timeFactor(record.created()); } - + if (record.category().equals(yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD)) { if (!(record.attribute("vote", "negative").equals("positive"))) continue; url = record.attribute("url", ""); @@ -273,7 +272,7 @@ public class Surftips { }); score = 5 + timeFactor(record.created()); } - + if (record.category().equals(yacyNewsPool.CATEGORY_WIKI_UPDATE)) { yacySeed seed = sb.peers.getConnected(record.originator()); if (seed == null) seed = sb.peers.getDisconnected(record.originator()); @@ -288,7 +287,7 @@ public class Surftips { score = 4 + timeFactor(record.created()); } } - + if (record.category().equals(yacyNewsPool.CATEGORY_BLOG_ADD)) { yacySeed seed = sb.peers.getConnected(record.originator()); if (seed == null) seed = sb.peers.getDisconnected(record.originator()); diff --git a/source/de/anomic/crawler/ZURL.java b/source/de/anomic/crawler/ZURL.java index 352686fd5..bdcb6d78e 100755 --- a/source/de/anomic/crawler/ZURL.java +++ b/source/de/anomic/crawler/ZURL.java @@ -278,7 +278,7 @@ public class ZURL implements Iterable { this.executor = entry.getColBytes(1, true); this.workdate = new Date(entry.getColLong(2)); this.workcount = (int) entry.getColLong(3); - this.anycause = entry.getColString(4); + this.anycause = entry.getColUTF8(4); this.bentry = new Request(Request.rowdef.newEntry(entry.getColBytes(5, false))); assert (Base64Order.enhancedCoder.equal(entry.getPrimaryKeyBytes(), this.bentry.url().hash())); this.stored = true; @@ -337,7 +337,7 @@ public class ZURL implements Iterable { try { return new Entry(e); } catch (final IOException ex) { - throw new RuntimeException("error '" + ex.getMessage() + "' for hash " + e.getColString(0)); + throw new RuntimeException("error '" + ex.getMessage() + "' for hash " + e.getPrimaryKeyASCII()); } } diff --git a/source/de/anomic/crawler/retrieval/Request.java b/source/de/anomic/crawler/retrieval/Request.java index 03e298175..6769ca80b 100755 --- a/source/de/anomic/crawler/retrieval/Request.java +++ b/source/de/anomic/crawler/retrieval/Request.java @@ -9,7 +9,7 @@ // $LastChangedBy$ // // LICENSE -// +// // This program is free software; you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by // the Free Software Foundation; either version 2 of the License, or @@ -39,7 +39,7 @@ import net.yacy.kelondro.order.NaturalOrder; import net.yacy.kelondro.workflow.WorkflowJob; public class Request extends WorkflowJob { - + // row definition for balancer-related NURL-entries public final static Row rowdef = new Row( "String urlhash-" + Word.commonHashLength + ", " + // the url's hash @@ -59,22 +59,22 @@ public class Request extends WorkflowJob { "Cardinal size-8 {b256}", // size of resource in bytes (if known) or 0 if not known Base64Order.enhancedCoder ); - + private byte[] initiator; // the initiator hash, is NULL or "" if it is the own proxy; // if this is generated by a crawl, the own peer hash in entered private byte[] refhash; // the url's referrer hash private DigestURI url; // the url as string - private String name; // the name of the url, from anchor tag name + private String name; // the name of the url, from anchor tag name private long appdate; // the time when the url was first time appeared. private String profileHandle; // the name of the fetch profile private int depth; // the prefetch depth so far, starts at 0 private int anchors; // number of anchors of the parent private int forkfactor; // sum of anchors of all ancestors - private Bitfield flags; + private Bitfield flags; private long size; // size of resource in bytes (if known) or 0 if not known private String statusMessage; private int initialHash; // to provide a object hash that does not change even if the url changes because of redirection - + /** * convenience method for 'full' request object * @param url @@ -83,30 +83,30 @@ public class Request extends WorkflowJob { public Request(final DigestURI url, final byte[] referrerhash) { this(null, url, referrerhash, null, null, null, 0, 0, 0, 0); } - + /** * A Request Entry is a object that is created to provide * all information to load a specific resource. - * + * * @param initiator the hash of the initiator peer * @param url the {@link URL} to crawl * @param referrer the hash of the referrer URL * @param name the name of the document to crawl * @param appdate the time when the url was first time appeared * @param profileHandle the name of the prefetch profile. This must not be null! - * @param depth the crawling depth of the entry + * @param depth the crawling depth of the entry * @param anchors number of anchors of the parent * @param forkfactor sum of anchors of all ancestors */ public Request( - final byte[] initiator, - final DigestURI url, - final byte[] referrerhash, - final String name, + final byte[] initiator, + final DigestURI url, + final byte[] referrerhash, + final String name, final Date appdate, final String profileHandle, - final int depth, - final int anchors, + final int depth, + final int anchors, final int forkfactor, final long size ) { @@ -129,22 +129,22 @@ public class Request extends WorkflowJob { this.status = WorkflowJob.STATUS_INITIATED; this.size = size; } - + public Request(final Row.Entry entry) throws IOException { assert (entry != null); insertEntry(entry); } private void insertEntry(final Row.Entry entry) throws IOException { - final String urlstring = entry.getColString(2); + final String urlstring = entry.getColUTF8(2); if (urlstring == null) throw new IOException ("url string is null"); this.initiator = entry.getColBytes(1, true); - this.initiator = (initiator == null) ? null : ((initiator.length == 0) ? null : initiator); + this.initiator = (this.initiator == null) ? null : ((this.initiator.length == 0) ? null : this.initiator); this.url = new DigestURI(urlstring, entry.getPrimaryKeyBytes()); this.refhash = (entry.empty(3)) ? null : entry.getColBytes(3, true); - this.name = (entry.empty(4)) ? "" : entry.getColString(4).trim(); + this.name = (entry.empty(4)) ? "" : entry.getColUTF8(4).trim(); this.appdate = entry.getColLong(5); - this.profileHandle = (entry.empty(6)) ? null : entry.getColString(6).trim(); + this.profileHandle = (entry.empty(6)) ? null : entry.getColASCII(6).trim(); this.depth = (int) entry.getColLong(7); this.anchors = (int) entry.getColLong(8); this.forkfactor = (int) entry.getColLong(9); @@ -153,35 +153,35 @@ public class Request extends WorkflowJob { //this.lastmodified = entry.getColLong(13); this.size = entry.getColLong(14); this.statusMessage = "loaded(kelondroRow.Entry)"; - this.initialHash = url.hashCode(); + this.initialHash = this.url.hashCode(); return; } - + public int hashCode() { // overloads Object.hashCode() return this.initialHash; } - - public void setStatus(final String s, int code) { + + public void setStatus(final String s, final int code) { //System.out.println("***DEBUG*** crawler status " + s + ", " + code + " for " + this.url.toNormalform(true, false)); this.statusMessage = s; this.status = code; } - + public String getStatus() { return this.statusMessage; } - + public Row.Entry toRow() { - final byte[] appdatestr = NaturalOrder.encodeLong(appdate, rowdef.width(5)); + final byte[] appdatestr = NaturalOrder.encodeLong(this.appdate, rowdef.width(5)); final byte[] loaddatestr = NaturalOrder.encodeLong(0 /*loaddate*/, rowdef.width(12)); final byte[] serverdatestr = NaturalOrder.encodeLong(0 /*lastmodified*/, rowdef.width(13)); final byte[] sizestr = NaturalOrder.encodeLong(this.size, rowdef.width(14)); // store the hash in the hash cache - byte[] namebytes = UTF8.getBytes(this.name); + final byte[] namebytes = UTF8.getBytes(this.name); final byte[][] entry = new byte[][] { this.url.hash(), - initiator, + this.initiator, this.url.toString().getBytes(), this.refhash, namebytes, @@ -197,12 +197,12 @@ public class Request extends WorkflowJob { sizestr}; return rowdef.newEntry(entry); } - + public DigestURI url() { // the url - return url; + return this.url; } - + public void redirectURL(final DigestURI redirectedURL) { // replace old URL by new one. This should only be used in case of url redirection this.url = redirectedURL; @@ -215,7 +215,7 @@ public class Request extends WorkflowJob { public byte[] initiator() { // returns the hash of the initiating peer - return initiator; + return this.initiator; } public boolean proxy() { @@ -232,7 +232,7 @@ public class Request extends WorkflowJob { // the date when the url was loaded return new Date(this.loaddate); } - + public Date lastmodified() { // the date that the server returned as document date return new Date(this.lastmodified); @@ -255,8 +255,8 @@ public class Request extends WorkflowJob { public String profileHandle() { // the handle of the crawl profile - assert profileHandle.length() == Word.commonHashLength : profileHandle + " != " + Word.commonHashLength; + assert this.profileHandle.length() == Word.commonHashLength : this.profileHandle + " != " + Word.commonHashLength; return this.profileHandle; } - + } \ No newline at end of file diff --git a/source/net/yacy/dbtest.java b/source/net/yacy/dbtest.java index b93affde6..ed3fc981f 100644 --- a/source/net/yacy/dbtest.java +++ b/source/net/yacy/dbtest.java @@ -402,7 +402,7 @@ public class dbtest { Row.Entry row; while (i.hasNext()) { row = i.next(); - for (int j = 0; j < row.columns(); j++) System.out.print(row.getColString(j) + ","); + for (int j = 0; j < row.columns(); j++) System.out.print(row.getColUTF8(j) + ","); System.out.println(); } } diff --git a/source/net/yacy/kelondro/data/navigation/NavigationReferenceRow.java b/source/net/yacy/kelondro/data/navigation/NavigationReferenceRow.java index 1e2abe404..00af8f5f0 100644 --- a/source/net/yacy/kelondro/data/navigation/NavigationReferenceRow.java +++ b/source/net/yacy/kelondro/data/navigation/NavigationReferenceRow.java @@ -9,7 +9,7 @@ // $LastChangedBy$ // // LICENSE -// +// // This program is free software; you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by // the Free Software Foundation; either version 2 of the License, or @@ -61,7 +61,7 @@ public final class NavigationReferenceRow extends AbstractReference implements N }, Base64Order.enhancedCoder ); - + // static properties private static final int col_navhash = 0; // n 24 the navigation hash and reference hash b64-encoded private static final int col_count = 1; // c the number of occurences @@ -78,7 +78,7 @@ public final class NavigationReferenceRow extends AbstractReference implements N public static final int flag_app_emphasized = 6; // word is emphasized in text (i.e. bold, italics, special size) private final Row.Entry entry; - + public NavigationReferenceRow( final byte[] termhash, final byte[] refhash, @@ -94,32 +94,32 @@ public final class NavigationReferenceRow extends AbstractReference implements N this.entry.setCol(col_pos, pos); this.entry.setCol(col_flags, flags); } - + public NavigationReferenceRow(final byte[] row) { this.entry = navEntryRow.newEntry(row); } - + public NavigationReferenceRow(final Row.Entry entry) { this.entry = entry; } - + @Override public NavigationReferenceRow clone() { final byte[] b = new byte[navEntryRow.objectsize]; - System.arraycopy(entry.bytes(), 0, b, 0, navEntryRow.objectsize); + System.arraycopy(this.entry.bytes(), 0, b, 0, navEntryRow.objectsize); return new NavigationReferenceRow(b); } public String toPropertyForm() { - return entry.toPropertyForm('=', true, true, false, false); + return this.entry.toPropertyForm('=', true, true, false, false); } - + public Entry toKelondroEntry() { return this.entry; } public String navigationHash() { - return this.entry.getColString(col_navhash); + return this.entry.getColASCII(col_navhash); } public byte[] urlhash() { @@ -138,35 +138,35 @@ public final class NavigationReferenceRow extends AbstractReference implements N assert p == 0 : "p = " + p; return (int) this.entry.getColLong(col_pos); } - + public byte flags() { return (byte) this.entry.getColLong(col_flags); } - + @Override public String toString() { return toPropertyForm(); } - + @Override public int hashCode() { - return this.navigationHash().hashCode(); + return navigationHash().hashCode(); } - + @Override public boolean equals(final Object obj) { if (this == obj) return true; if (obj == null) return false; if (!(obj instanceof NavigationReferenceRow)) return false; - NavigationReferenceRow other = (NavigationReferenceRow) obj; - return this.navigationHash().equals(other.navigationHash()); + final NavigationReferenceRow other = (NavigationReferenceRow) obj; + return navigationHash().equals(other.navigationHash()); } - + public boolean isOlder(final Reference other) { return false; } - + // unsupported operations: public void join(final Reference oe) { @@ -180,5 +180,5 @@ public final class NavigationReferenceRow extends AbstractReference implements N public Collection positions() { throw new UnsupportedOperationException(); } - + } diff --git a/source/net/yacy/kelondro/index/Row.java b/source/net/yacy/kelondro/index/Row.java index 2f2c3f960..133f6d0d9 100644 --- a/source/net/yacy/kelondro/index/Row.java +++ b/source/net/yacy/kelondro/index/Row.java @@ -36,6 +36,7 @@ import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.ConcurrentHashMap; import java.util.regex.Pattern; +import net.yacy.cora.document.ASCII; import net.yacy.cora.document.UTF8; import net.yacy.cora.ranking.AbstractOrder; import net.yacy.cora.ranking.Order; @@ -475,15 +476,57 @@ public final class Row { throw new kelondroException("ROW", "addCol did not find appropriate encoding"); } - public final String getColString(final int column) { + public final byte[] getPrimaryKeyBytes() { + if (this.rowinstance[this.offset] == 0) return null; + if (Row.this.row.length == 1 && this.offset == 0 && this.rowinstance.length == Row.this.primaryKeyLength) { + // avoid memory allocation in case that the row consists in only the primary key + return this.rowinstance; + } + final byte[] c = new byte[Row.this.primaryKeyLength]; + System.arraycopy(this.rowinstance, this.offset, c, 0, Row.this.primaryKeyLength); + return c; + } + + public final String getPrimaryKeyUTF8() { + if (this.rowinstance[this.offset] == 0) return null; + if (Row.this.row.length == 1 && this.offset == 0 && this.rowinstance.length == Row.this.primaryKeyLength) { + // avoid memory allocation in case that the row consists in only the primary key + return UTF8.String(this.rowinstance); + } + return UTF8.String(this.rowinstance, this.offset, Row.this.primaryKeyLength); + } + + public final String getPrimaryKeyASCII() { + if (this.rowinstance[this.offset] == 0) return null; + if (Row.this.row.length == 1 && this.offset == 0 && this.rowinstance.length == Row.this.primaryKeyLength) { + // avoid memory allocation in case that the row consists in only the primary key + return ASCII.String(this.rowinstance); + } + return ASCII.String(this.rowinstance, this.offset, Row.this.primaryKeyLength); + } + + public final String getColUTF8(final int column) { final int clstrt = Row.this.colstart[column]; - int length = Row.this.row[column].cellwidth; if (this.rowinstance[this.offset + clstrt] == 0) return null; + final int length = getColLength(column, clstrt); + if (length == 0) return null; + return UTF8.String(this.rowinstance, this.offset + clstrt, length); + } + + public final String getColASCII(final int column) { + final int clstrt = Row.this.colstart[column]; + if (this.rowinstance[this.offset + clstrt] == 0) return null; + final int length = getColLength(column, clstrt); + if (length == 0) return null; + return ASCII.String(this.rowinstance, this.offset + clstrt, length); + } + + private final int getColLength(final int column, final int clstrt) { + int length = Row.this.row[column].cellwidth; assert length <= this.rowinstance.length - this.offset - clstrt; if (length > this.rowinstance.length - this.offset - clstrt) length = this.rowinstance.length - this.offset - clstrt; while ((length > 0) && (this.rowinstance[this.offset + clstrt + length - 1] == 0)) length--; - if (length == 0) return null; - return UTF8.String(this.rowinstance, this.offset + clstrt, length); + return length; } public final long getColLong(final int column) { @@ -517,16 +560,6 @@ public final class Row { return this.rowinstance[this.offset + Row.this.colstart[column]]; } - public final byte[] getPrimaryKeyBytes() { - if (Row.this.columns() == 1 && this.offset == 0 && this.rowinstance.length == Row.this.primaryKeyLength) { - // avoid memory allocation in case that the row consists in only the primary key - return this.rowinstance; - } - final byte[] c = new byte[Row.this.primaryKeyLength]; - System.arraycopy(this.rowinstance, this.offset, c, 0, Row.this.primaryKeyLength); - return c; - } - public final int getPrimaryKeyLength() { return Row.this.primaryKeyLength; } diff --git a/source/net/yacy/kelondro/index/RowCollection.java b/source/net/yacy/kelondro/index/RowCollection.java index 4372cffcc..61103ab01 100644 --- a/source/net/yacy/kelondro/index/RowCollection.java +++ b/source/net/yacy/kelondro/index/RowCollection.java @@ -105,7 +105,7 @@ public class RowCollection implements Sortable, Iterable, this.chunkcount = chunkcachelength / rowdef.objectsize; // patch problem } this.lastTimeWrote = (exportedCollection.getColLong(exp_last_wrote) + 10957) * day; - final String sortOrderKey = exportedCollection.getColString(exp_order_type); + final String sortOrderKey = exportedCollection.getColASCII(exp_order_type); ByteOrder oldOrder = null; if ((sortOrderKey == null) || (sortOrderKey.equals("__"))) { oldOrder = null; diff --git a/source/net/yacy/kelondro/index/RowSet.java b/source/net/yacy/kelondro/index/RowSet.java index 47be42021..e69f8410a 100644 --- a/source/net/yacy/kelondro/index/RowSet.java +++ b/source/net/yacy/kelondro/index/RowSet.java @@ -583,7 +583,7 @@ public class RowSet extends RowCollection implements Index, Iterable Row.Entry entry; while (ii.hasNext()) { entry = ii.next(); - s = ASCII.String(entry.getPrimaryKeyBytes()).trim(); + s = entry.getPrimaryKeyASCII().trim(); System.out.print(s + ", "); if (s.equals("drei")) ii.remove(); } diff --git a/source/net/yacy/kelondro/table/SQLTable.java b/source/net/yacy/kelondro/table/SQLTable.java index 4e16ea938..a6044392c 100644 --- a/source/net/yacy/kelondro/table/SQLTable.java +++ b/source/net/yacy/kelondro/table/SQLTable.java @@ -219,7 +219,7 @@ public class SQLTable implements Index, Iterable { final PreparedStatement sqlStatement = this.theDBConnection.prepareStatement(sqlQuery); - sqlStatement.setString(1, row.getColString(0)); + sqlStatement.setString(1, row.getPrimaryKeyASCII()); sqlStatement.setBytes(2, row.bytes()); sqlStatement.execute(); @@ -240,7 +240,7 @@ public class SQLTable implements Index, Iterable { final PreparedStatement sqlStatement = this.theDBConnection.prepareStatement(sqlQuery); - sqlStatement.setString(1, row.getColString(0)); + sqlStatement.setString(1, row.getPrimaryKeyASCII()); sqlStatement.setBytes(2, row.bytes()); sqlStatement.execute(); diff --git a/source/net/yacy/kelondro/util/FileUtils.java b/source/net/yacy/kelondro/util/FileUtils.java index 6a6e9fde0..71c2896cb 100644 --- a/source/net/yacy/kelondro/util/FileUtils.java +++ b/source/net/yacy/kelondro/util/FileUtils.java @@ -1,4 +1,4 @@ -// serverFileUtils.java +// serverFileUtils.java // ------------------------------------------- // (C) by Michael Peter Christen; mc@yacy.net // first published on http://www.anomic.de @@ -69,20 +69,20 @@ import net.yacy.kelondro.logging.Log; public final class FileUtils { private static final int DEFAULT_BUFFER_SIZE = 1024; // this is also the maximum chunk size - + public static long copy(final InputStream source, final OutputStream dest) throws IOException { return copy(source, dest, -1); } - + /** * Copies an InputStream to an OutputStream. - * + * * @param source InputStream * @param dest OutputStream * @param count the total amount of bytes to copy (-1 for all, else must be greater than zero) * @return Total number of bytes copied. - * @throws IOException - * + * @throws IOException + * * @see #copy(InputStream source, File dest) * @see #copyRange(File source, OutputStream dest, int start) * @see #copy(File source, OutputStream dest) @@ -94,27 +94,27 @@ public final class FileUtils { // no bytes to copy return 0; } - - final byte[] buffer = new byte[DEFAULT_BUFFER_SIZE]; + + final byte[] buffer = new byte[DEFAULT_BUFFER_SIZE]; int chunkSize = (int) ((count > 0) ? Math.min(count, DEFAULT_BUFFER_SIZE) : DEFAULT_BUFFER_SIZE); - + int c; long total = 0; while ((c = source.read(buffer, 0, chunkSize)) > 0) { dest.write(buffer, 0, c); dest.flush(); total += c; - + if (count > 0) { chunkSize = (int) Math.min(count - total, DEFAULT_BUFFER_SIZE); if (chunkSize == 0) break; } - + } dest.flush(); - + return total; } - + public static int copy(final File source, final Charset inputCharset, final Writer dest) throws IOException { InputStream fis = null; try { @@ -123,24 +123,24 @@ public final class FileUtils { } finally { if (fis != null) try { fis.close(); } catch (final Exception e) {} } - } - + } + public static int copy(final InputStream source, final Writer dest) throws IOException { final InputStreamReader reader = new InputStreamReader(source); return copy(reader,dest); } - + public static int copy(final InputStream source, final Writer dest, final Charset inputCharset) throws IOException { final InputStreamReader reader = new InputStreamReader(source,inputCharset); return copy(reader,dest); } - + public static int copy(final String source, final Writer dest) throws IOException { dest.write(source); dest.flush(); return source.length(); } - + public static int copy(final Reader source, final Writer dest) throws IOException { assert source != null; assert dest != null; @@ -163,25 +163,25 @@ public final class FileUtils { } return count; } - + public static void copy(final InputStream source, final File dest) throws IOException { copy(source,dest,-1); } - + /** * Copies an InputStream to a File. - * + * * @param source InputStream * @param dest File * @param count the amount of bytes to copy - * @throws IOException + * @throws IOException * @see #copy(InputStream source, OutputStream dest) * @see #copyRange(File source, OutputStream dest, int start) * @see #copy(File source, OutputStream dest) * @see #copy(File source, File dest) */ public static void copy(final InputStream source, final File dest, final long count) throws IOException { - String path = dest.getParent(); + final String path = dest.getParent(); if (path != null && path.length() > 0) new File(path).mkdirs(); FileOutputStream fos = null; try { @@ -197,7 +197,7 @@ public final class FileUtils { * @param source File * @param dest OutputStream * @param start Number of bytes to skip from the beginning of the File - * @throws IOException + * @throws IOException * @see #copy(InputStream source, OutputStream dest) * @see #copy(InputStream source, File dest) * @see #copy(File source, OutputStream dest) @@ -219,7 +219,7 @@ public final class FileUtils { * Copies a File to an OutputStream. * @param source File * @param dest OutputStream - * @throws IOException + * @throws IOException * @see #copy(InputStream source, OutputStream dest) * @see #copy(InputStream source, File dest) * @see #copyRange(File source, OutputStream dest, int start) @@ -240,7 +240,7 @@ public final class FileUtils { * @param source File * @param dest File * @param count the amount of bytes to copy - * @throws IOException + * @throws IOException * @see #copy(InputStream source, OutputStream dest) * @see #copy(InputStream source, File dest) * @see #copyRange(File source, OutputStream dest, int start) @@ -263,7 +263,7 @@ public final class FileUtils { dest.write(source, 0, source.length); dest.flush(); } - + public static void copy(final byte[] source, final File dest) throws IOException { copy(new ByteArrayInputStream(source), dest); } @@ -271,14 +271,14 @@ public final class FileUtils { public static byte[] read(final InputStream source) throws IOException { return read(source,-1); } - + public static byte[] read(final InputStream source, final int count) throws IOException { if (count > 0) { - byte[] b = new byte[count]; - int c = source.read(b, 0, count); + final byte[] b = new byte[count]; + final int c = source.read(b, 0, count); assert c == count: "count = " + count + ", c = " + c; if (c != count) { - byte[] bb = new byte[c]; + final byte[] bb = new byte[c]; System.arraycopy(b, 0, bb, 0, c); return bb; } @@ -339,7 +339,7 @@ public final class FileUtils { if (zipOut != null) try { zipOut.close(); } catch (final Exception e) {} } } - + /** * This function determines if a byte array is gzip compressed and uncompress it * @param source properly gzip compressed byte array @@ -348,10 +348,10 @@ public final class FileUtils { */ public static byte[] uncompressGZipArray(byte[] source) throws IOException { if (source == null) return null; - + // support of gzipped data (requested by roland) /* "Bitwise OR of signed byte value - * + * * [...] Values loaded from a byte array are sign extended to 32 bits before * any any bitwise operations are performed on the value. Thus, if b[0] * contains the value 0xff, and x is initially 0, then the code ((x << @@ -366,22 +366,22 @@ public final class FileUtils { final GZIPInputStream zippedContent = new GZIPInputStream(byteInput); final byte[] data = new byte[1024]; int read = 0; - + // reading gzip file and store it uncompressed while((read = zippedContent.read(data, 0, 1024)) != -1) { byteOutput.write(data, 0, read); } zippedContent.close(); - byteOutput.close(); - + byteOutput.close(); + source = byteOutput.toByteArray(); } catch (final Exception e) { if (!e.getMessage().equals("Not in GZIP format")) { throw new IOException(e.getMessage()); } } - } - + } + return source; } @@ -413,7 +413,7 @@ public final class FileUtils { return null; } } - + public static void saveMap(final File file, final Map props, final String comment) throws IOException { PrintWriter pw = null; final File tf = new File(file.toString() + "." + (System.currentTimeMillis() % 1000)); @@ -470,8 +470,8 @@ public final class FileUtils { os = zos; } if(os != null) { - for (final Iterator i = set.iterator(); i.hasNext(); ) { - os.write(i.next()); + for (final byte[] b : set) { + os.write(b); if (sep != null) os.write(UTF8.getBytes(sep)); } os.close(); @@ -495,31 +495,28 @@ public final class FileUtils { } if (os != null) { final Iterator i = set.iterator(); - String key; if (i.hasNext()) { - key = UTF8.String(i.next().getPrimaryKeyBytes()); - os.write(UTF8.getBytes(key)); + os.write(i.next().getPrimaryKeyBytes()); } while (i.hasNext()) { - key = UTF8.String(i.next().getPrimaryKeyBytes()); if (sep != null) os.write(UTF8.getBytes(sep)); - os.write(UTF8.getBytes(key)); + os.write(i.next().getPrimaryKeyBytes()); } os.close(); } forceMove(tf, file); } - public static ConcurrentHashMap table(Reader r) { - BufferedReader br = new BufferedReader(r); + public static ConcurrentHashMap table(final Reader r) { + final BufferedReader br = new BufferedReader(r); return table(new StringsIterator(br)); } - + //private final static Pattern escaped_equal = Pattern.compile("\\="); //private final static Pattern escaped_newline = Pattern.compile("\\n"); //private final static Pattern escaped_backslash = Pattern.compile("\\"); - public static ConcurrentHashMap table(Iterator li) { + public static ConcurrentHashMap table(final Iterator li) { String line; final ConcurrentHashMap props = new ConcurrentHashMap(); while (li.hasNext()) { @@ -532,31 +529,31 @@ public final class FileUtils { } while ( pos > 0 && line.charAt(pos-1) == '\\'); if (pos > 0) { //String key = escaped_equal.matcher(line.substring(0, pos).trim()).replaceAll("="); - String key = line.substring(0, pos).trim().replace("\\=", "=").replace("\\n", "\n").replace("\\", "\\"); - String value = line.substring(pos + 1).trim().replace("\\n", "\n").replace("\\\\", "\\"); + final String key = line.substring(0, pos).trim().replace("\\=", "=").replace("\\n", "\n").replace("\\", "\\"); + final String value = line.substring(pos + 1).trim().replace("\\n", "\n").replace("\\\\", "\\"); props.put(key, value); } } return props; } - + public static Map table(final byte[] a) { return table(strings(a)); } - public static Iterator strings(byte[] a) { + public static Iterator strings(final byte[] a) { if (a == null) return new ArrayList().iterator(); try { return new StringsIterator(new BufferedReader(new InputStreamReader(new ByteArrayInputStream(a), "UTF-8"))); - } catch (UnsupportedEncodingException e) { + } catch (final UnsupportedEncodingException e) { return null; } } - + /** * Read lines of a file into an ArrayList. - * + * * @param listFile the file * @return the resulting array as an ArrayList */ @@ -578,12 +575,12 @@ public final class FileUtils { } return list; } - + /** * Write a String to a file (used for string representation of lists). - * + * * @param listFile the file to write to * @param out the String to write * @return returns true if successful, false otherwise @@ -607,19 +604,19 @@ public final class FileUtils { /** * Read lines of a text file into a String, optionally ignoring comments. - * + * * @param listFile the File to read from. * @param withcomments If false ignore lines starting with '#'. * @return String representation of the file content. */ public static String getListString(final File listFile, final boolean withcomments){ final StringBuilder temp = new StringBuilder(300); - - BufferedReader br = null; + + BufferedReader br = null; try{ br = new BufferedReader(new InputStreamReader(new FileInputStream(listFile))); temp.ensureCapacity((int) listFile.length()); - + // Read the List String line = ""; while ((line = br.readLine()) != null) { @@ -647,7 +644,7 @@ public final class FileUtils { public static List getDirListing(final String dirname){ return getDirListing(dirname, null); } - + /** * Read content of a directory into a String array of file names. * @param dirname The directory to get the file listing from. If it doesn't exist yet, @@ -663,7 +660,7 @@ public final class FileUtils { /** * Read content of a directory into a String array of file names. - * + * * @param dir The directory to get the file listing from. If it doesn't exist yet, * it will be created. * @return array of file names @@ -671,7 +668,7 @@ public final class FileUtils { public static List getDirListing(final File dir){ return getDirListing(dir, null); } - + /** * Read content of a directory into a String array of file names. * @param dir The directory to get the file listing from. If it doesn't exist yet, @@ -682,7 +679,7 @@ public final class FileUtils { * @return array of file names */ public static List getDirListing(final File dir, final String filter){ - List ret = new LinkedList(); + final List ret = new LinkedList(); File[] fileList; if (dir != null ) { if (!dir.exists()) { @@ -697,13 +694,13 @@ public final class FileUtils { return ret; } return null; - } + } // same as below public static ArrayList getDirsRecursive(final File dir, final String notdir){ return getDirsRecursive(dir, notdir, true); } - + /** * Returns a List of all dirs and subdirs as File Objects * @@ -727,23 +724,23 @@ public final class FileUtils { return resultList; } - + /** * Write elements of an Array of Strings to a file (one element per line). - * + * * @param listFile the file to write to * @param list the Array to write * @return returns true if successful, false otherwise */ public static boolean writeList(final File listFile, final String[] list){ final StringBuilder out = new StringBuilder(list.length * 40 + 1); - for(int i=0;i < list.length; i++){ - out.append(list[i]).append(CR).append(LF); + for (final String element : list) { + out.append(element).append(CR).append(LF); } return FileUtils.writeList(listFile, new String(out)); //(File, String) } - + public static class StringsIterator implements Iterator { private final BufferedReader reader; private String nextLine; @@ -753,21 +750,21 @@ public final class FileUtils { next(); } public boolean hasNext() { - return nextLine != null; + return this.nextLine != null; } public String next() { - String line = nextLine; + final String line = this.nextLine; try { - while ((nextLine = reader.readLine()) != null) { - nextLine = nextLine.trim(); - if (nextLine.length() > 0) break; + while ((this.nextLine = this.reader.readLine()) != null) { + this.nextLine = this.nextLine.trim(); + if (this.nextLine.length() > 0) break; } - } catch (IOException e) { - nextLine = null; - } catch (OutOfMemoryError e) { + } catch (final IOException e) { + this.nextLine = null; + } catch (final OutOfMemoryError e) { Log.logException(e); - nextLine = null; + this.nextLine = null; } return line; } @@ -775,9 +772,9 @@ public final class FileUtils { public void remove() { throw new UnsupportedOperationException(); } - + } - + /** * @param from * @param to @@ -790,7 +787,7 @@ public final class FileUtils { FileUtils.deletedelete(from); } } - + /** * Moves all files from a directory to another. * @param from_dir Directory which contents will be moved. @@ -805,10 +802,10 @@ public final class FileUtils { Log.logWarning("serverFileUtils", "moveAll(): could not move from "+ from_dir + list[i] +" to "+ to_dir + list[i]); } } - + public static class dirlistComparator implements Comparator, Serializable { - + /** * generated serial */ @@ -823,27 +820,27 @@ public final class FileUtils { return file1.getName().compareToIgnoreCase(file2.getName()); } } - } + } - public static final File createTempFile(Class classObj, final String name) throws IOException { + public static final File createTempFile(final Class classObj, final String name) throws IOException { String parserClassName = classObj.getName(); int idx = parserClassName.lastIndexOf('.'); if (idx != -1) { parserClassName = parserClassName.substring(idx+1); - } - + } + // get the file extension idx = name.lastIndexOf('/'); - final String fileName = (idx != -1) ? name.substring(idx+1) : name; - + final String fileName = (idx != -1) ? name.substring(idx+1) : name; + idx = fileName.lastIndexOf('.'); final String fileExt = (idx > -1) ? fileName.substring(idx+1) : ""; - + // create the temp file final File tempFile = File.createTempFile(parserClassName + "_" + ((idx>-1)?fileName.substring(0,idx):fileName), (fileExt.length()>0)?"."+fileExt:fileExt); return tempFile; } - + /** * copies the input stream to one output stream (byte per byte) * @param in @@ -862,7 +859,7 @@ public final class FileUtils { out.flush(); return count; } - + /** * copies the input stream to both output streams (byte per byte) * @param in @@ -874,7 +871,7 @@ public final class FileUtils { public static int copyToStreams(final BufferedInputStream in, final BufferedOutputStream out0, final BufferedOutputStream out1) throws IOException { assert out0 != null; assert out1 != null; - + int count = 0; // copy bytes int b; @@ -899,7 +896,7 @@ public final class FileUtils { public static int copyToWriter(final BufferedInputStream data, final BufferedWriter writer, final Charset charSet) throws IOException { // the docs say: "For top efficiency, consider wrapping an InputStreamReader within a BufferedReader." final Reader sourceReader = new InputStreamReader(data, charSet); - + int count = 0; // copy bytes int b; @@ -910,13 +907,13 @@ public final class FileUtils { writer.flush(); return count; } - + public static int copyToWriters(final BufferedInputStream data, final BufferedWriter writer0, final BufferedWriter writer1, final Charset charSet) throws IOException { // the docs say: "For top efficiency, consider wrapping an InputStreamReader within a BufferedReader." assert writer0 != null; assert writer1 != null; final Reader sourceReader = new InputStreamReader(data, charSet); - + int count = 0; // copy bytes int b; @@ -929,7 +926,7 @@ public final class FileUtils { writer1.flush(); return count; } - + /** * delete files and directories * if a directory is not empty, delete also everything inside @@ -943,10 +940,10 @@ public final class FileUtils { if (path.isDirectory()) { final String[] list = path.list(); if (list != null) { - for (String s: list) deletedelete(new File(path, s)); + for (final String s: list) deletedelete(new File(path, s)); } } - + int c = 0; while (c++ < 20) { if (!path.exists()) break; @@ -954,35 +951,35 @@ public final class FileUtils { // some OS may be slow when giving up file pointer //System.runFinalization(); //System.gc(); - try { Thread.sleep(200); } catch (InterruptedException e) { break; } + try { Thread.sleep(200); } catch (final InterruptedException e) { break; } } if (path.exists()) { path.deleteOnExit(); String p = ""; try { p = path.getCanonicalPath(); - } catch (IOException e1) { + } catch (final IOException e1) { Log.logException(e1); } if (System.getProperties().getProperty("os.name","").toLowerCase().startsWith("windows")) { // deleting files on windows sometimes does not work with java try { - String command = "cmd /C del /F /Q \"" + p + "\""; - Process r = Runtime.getRuntime().exec(command); + final String command = "cmd /C del /F /Q \"" + p + "\""; + final Process r = Runtime.getRuntime().exec(command); if (r == null) { Log.logSevere("FileUtils", "cannot execute command: " + command); } else { - byte[] response = read(r.getInputStream()); + final byte[] response = read(r.getInputStream()); Log.logInfo("FileUtils", "deletedelete: " + UTF8.String(response)); } - } catch (IOException e) { + } catch (final IOException e) { Log.logException(e); - } + } } if (path.exists()) Log.logSevere("FileUtils", "cannot delete file " + p); } } - + public static void main(final String[] args) { try { writeAndGZip("ein zwei drei, Zauberei".getBytes(), new File("zauberei.txt.gz")); diff --git a/source/net/yacy/peers/yacyNewsDB.java b/source/net/yacy/peers/yacyNewsDB.java index 87baf39d1..ae16d6d01 100644 --- a/source/net/yacy/peers/yacyNewsDB.java +++ b/source/net/yacy/peers/yacyNewsDB.java @@ -164,11 +164,11 @@ public class yacyNewsDB { private Record b2r(final Row.Entry b) { if (b == null) return null; return new yacyNewsDB.Record( - b.getColString(0), - b.getColString(1), - (b.empty(2)) ? null : my_SHORT_SECOND_FORMATTER.parse(b.getColString(2), GenericFormatter.UTCDiffString()), + b.getPrimaryKeyASCII(), + b.getColUTF8(1), + (b.empty(2)) ? null : my_SHORT_SECOND_FORMATTER.parse(b.getColASCII(2), GenericFormatter.UTCDiffString()), (int) b.getColLong(3), - MapTools.string2map(b.getColString(4), ",") + MapTools.string2map(b.getColUTF8(4), ",") ); } diff --git a/source/net/yacy/peers/yacyNewsQueue.java b/source/net/yacy/peers/yacyNewsQueue.java index 205967859..2de984749 100644 --- a/source/net/yacy/peers/yacyNewsQueue.java +++ b/source/net/yacy/peers/yacyNewsQueue.java @@ -159,7 +159,7 @@ public class yacyNewsQueue { yacyNewsDB.Record b2r(final Row.Entry b) throws IOException { if (b == null) return null; - final String id = b.getColString(0); + final String id = b.getPrimaryKeyASCII(); //Date touched = yacyCore.parseUniversalDate(UTF8.String(b[1])); return this.newsDB.get(id); } diff --git a/source/net/yacy/search/index/MetadataRepository.java b/source/net/yacy/search/index/MetadataRepository.java index 2f9c024de..d46aa888a 100644 --- a/source/net/yacy/search/index/MetadataRepository.java +++ b/source/net/yacy/search/index/MetadataRepository.java @@ -302,7 +302,7 @@ public final class MetadataRepository implements Iterable { final Row.Entry entry = this.urlIndexFile.get(urlHashBytes, true); // getting the wrong url string - oldUrlStr = entry.getColString(1).trim(); + oldUrlStr = entry.getColUTF8(1).trim(); int pos = -1; if ((pos = oldUrlStr.indexOf("://")) != -1) {