boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

Merge branch 'master' of gitorious.org:yacy/rc1

Browse Source
pull/1/head
admin 13 years ago
parent 23afee58fe 86b3385847
commit 391fc9bd57
9 changed files with 69 additions and 40 deletions
Show Stats Download Patch File Download Diff File

2
defaults/yacy.init
Unescape View File

@ -125,7 +125,7 @@ update.cycle = 168
# It is recommended to set this list to low developer version numbers
update.blacklist = ...[123]
# a update can also restricted with a concept property, which can decide if an
# update is only valid if it either is a main release or any svn release including new development releases
# update is only valid if it either is a main release or any release including new development releases
# Valid keywords are 'main' and 'any'
update.concept = any
# the following values are set automatically:

4
htroot/ConfigUpdate_p.java
Unescape View File

@ -127,7 +127,7 @@ public class ConfigUpdate_p {
sb.getLog().logInfo("AUTO-UPDATE: downloading more recent release " + updateVersion.getUrl());
final File downloaded = updateVersion.downloadRelease();
prop.putHTML("candeploy_autoUpdate_downloadedRelease", updateVersion.getName());
final boolean devenvironment = new File(sb.getAppPath(), ".svn").exists();
final boolean devenvironment = new File(sb.getAppPath(), ".git").exists();
if (devenvironment) {
sb.getLog().logInfo("AUTO-UPDATE: omitting update because this is a development environment");
prop.put("candeploy_autoUpdate", "3");
@ -156,7 +156,7 @@ public class ConfigUpdate_p {
// version information
final String versionstring = yacyBuildProperties.getVersion() + "/" + yacyBuildProperties.getSVNRevision();
prop.putHTML("candeploy_versionpp", versionstring);
final boolean devenvironment = new File(sb.getAppPath(), ".svn").exists();
final boolean devenvironment = new File(sb.getAppPath(), ".git").exists();
float thisVersion = Float.parseFloat(yacyBuildProperties.getVersion());
// cut off the SVN Rev in the Version
try {

2
htroot/Steering.java
Unescape View File

@ -73,7 +73,7 @@ public class Steering {
if (post.containsKey("update")) {
Log.logInfo("STEERING", "update request from " + requestIP);
final boolean devenvironment = new File(sb.getAppPath(), ".svn").exists();
final boolean devenvironment = new File(sb.getAppPath(), ".git").exists();
final String releaseFileName = post.get("releaseinstall", "");
final File releaseFile = new File(sb.getDataPath(), "DATA/RELEASE/".replace("/", File.separator) + releaseFileName);
if ((!devenvironment) && (releaseFileName.length() > 0) && (releaseFile.exists())) {

9
source/net/yacy/kelondro/data/word/WordReferenceFactory.java
Unescape View File

@ -26,8 +26,10 @@
package net.yacy.kelondro.data.word;
import java.util.Collections;
import java.util.Iterator;
import java.util.Map;
import java.util.SortedMap;
import java.util.TreeMap;
import net.yacy.cora.document.ASCII;
@ -39,15 +41,18 @@ import net.yacy.kelondro.util.ByteBuffer;
public class WordReferenceFactory implements ReferenceFactory<WordReference> {
@Override
public WordReference produceSlow(final Entry e) {
return new WordReferenceRow(e);
}
@Override
public WordReference produceFast(final WordReference r) {
if (r instanceof WordReferenceVars) return r;
return new WordReferenceVars(r);
}
@Override
public Row getRow() {
return WordReferenceRow.urlEntryRow;
}
@ -110,8 +115,8 @@ public class WordReferenceFactory implements ReferenceFactory<WordReference> {
* @param peerhash
* @return
*/
public static final TreeMap<String, StringBuilder> decompressIndex(ByteBuffer ci, final String peerhash) {
TreeMap<String, StringBuilder> target = new TreeMap<String, StringBuilder>();
public static final SortedMap<String, StringBuilder> decompressIndex(ByteBuffer ci, final String peerhash) {
SortedMap<String, StringBuilder> target = Collections.synchronizedSortedMap(new TreeMap<String, StringBuilder>());
// target is a mapping from url-hashes to a string of peer-hashes
if (ci.byteAt(0) != '{' || ci.byteAt(ci.length() - 1) != '}') return target;
//System.out.println("DEBUG-DECOMPRESS: input is " + ci.toString());

30
source/net/yacy/kelondro/table/SplitTable.java
Unescape View File

@ -106,12 +106,14 @@ public class SplitTable implements Index, Iterable<Row.Entry> {
init();
}
@Override
public long mem() {
long m = 0;
for (final Index i: this.tables.values()) m += i.mem();
return m;
}
@Override
public final byte[] smallestKey() {
final HandleSet keysort = new HandleSet(this.rowdef.primaryKeyLength, this.rowdef.objectOrder, this.tables.size());
for (final Index oi: this.tables.values()) try {
@ -122,6 +124,7 @@ public class SplitTable implements Index, Iterable<Row.Entry> {
return keysort.smallestKey();
}
@Override
public final byte[] largestKey() {
final HandleSet keysort = new HandleSet(this.rowdef.primaryKeyLength, this.rowdef.objectOrder, this.tables.size());
for (final Index oi: this.tables.values()) try {
@ -222,6 +225,7 @@ public class SplitTable implements Index, Iterable<Row.Entry> {
}
final Table a = table;
final Thread p = new Thread() {
@Override
public void run() {
a.warmUp();
}
@ -245,6 +249,7 @@ public class SplitTable implements Index, Iterable<Row.Entry> {
*/
}
@Override
public void clear() throws IOException {
close();
final String[] l = this.path.list();
@ -273,10 +278,12 @@ public class SplitTable implements Index, Iterable<Row.Entry> {
FileUtils.deletedelete(tabledir);
}
@Override
public String filename() {
return new File(this.path, this.prefix).toString();
}
@Override
public int size() {
final Iterator<Index> i = this.tables.values().iterator();
int s = 0;
@ -284,6 +291,7 @@ public class SplitTable implements Index, Iterable<Row.Entry> {
return s;
}
@Override
public boolean isEmpty() {
final Iterator<Index> i = this.tables.values().iterator();
while (i.hasNext()) if (!i.next().isEmpty()) return false;
@ -298,20 +306,24 @@ public class SplitTable implements Index, Iterable<Row.Entry> {
return s;
}
@Override
public Row row() {
return this.rowdef;
}
@Override
public boolean has(final byte[] key) {
return keeperOf(key) != null;
}
@Override
public Row.Entry get(final byte[] key, final boolean forcecopy) throws IOException {
final Index keeper = keeperOf(key);
if (keeper == null) return null;
return keeper.get(key, forcecopy);
}
@Override
public Map<byte[], Row.Entry> get(final Collection<byte[]> keys, final boolean forcecopy) throws IOException, InterruptedException {
final Map<byte[], Row.Entry> map = new TreeMap<byte[], Row.Entry>(row().objectOrder);
Row.Entry entry;
@ -344,6 +356,8 @@ public class SplitTable implements Index, Iterable<Row.Entry> {
// check size and age of given table; in case it is too large or too old
// create a new table
assert table != null;
long t = System.currentTimeMillis();
if (((t / 1000) % 10) != 0) return table; // we check only every 10 seconds because all these file and parser operations are very expensive
final String name = new File(table.filename()).getName();
long d;
try {
@ -352,12 +366,13 @@ public class SplitTable implements Index, Iterable<Row.Entry> {
Log.logSevere("SplitTable", "", e);
d = 0;
}
if (d + this.fileAgeLimit < System.currentTimeMillis() || new File(this.path, name).length() >= this.fileSizeLimit) {
if (d + this.fileAgeLimit < t || new File(this.path, name).length() >= this.fileSizeLimit) {
return newTable();
}
return table;
}
@Override
public Row.Entry replace(final Row.Entry row) throws IOException, RowSpaceExceededException {
assert row.objectsize() <= this.rowdef.objectsize;
Index keeper = keeperOf(row.getPrimaryKeyBytes());
@ -377,6 +392,7 @@ public class SplitTable implements Index, Iterable<Row.Entry> {
* @throws IOException
* @throws RowSpaceExceededException
*/
@Override
public boolean put(final Row.Entry row) throws IOException, RowSpaceExceededException {
assert row.objectsize() <= this.rowdef.objectsize;
final byte[] key = row.getPrimaryKeyBytes();
@ -406,6 +422,7 @@ public class SplitTable implements Index, Iterable<Row.Entry> {
return null;
}
@Override
public void addUnique(final Row.Entry row) throws IOException, RowSpaceExceededException {
assert row.objectsize() <= this.rowdef.objectsize;
Index table = (this.current == null) ? null : this.tables.get(this.current);
@ -416,6 +433,7 @@ public class SplitTable implements Index, Iterable<Row.Entry> {
table.addUnique(row);
}
@Override
public List<RowCollection> removeDoubles() throws IOException, RowSpaceExceededException {
final Iterator<Index> i = this.tables.values().iterator();
final List<RowCollection> report = new ArrayList<RowCollection>();
@ -425,18 +443,21 @@ public class SplitTable implements Index, Iterable<Row.Entry> {
return report;
}
@Override
public boolean delete(final byte[] key) throws IOException {
final Index table = keeperOf(key);
if (table == null) return false;
return table.delete(key);
}
@Override
public Row.Entry remove(final byte[] key) throws IOException {
final Index table = keeperOf(key);
if (table == null) return null;
return table.remove(key);
}
@Override
public Row.Entry removeOne() throws IOException {
final Iterator<Index> i = this.tables.values().iterator();
Index table, maxtable = null;
@ -454,6 +475,7 @@ public class SplitTable implements Index, Iterable<Row.Entry> {
return maxtable.removeOne();
}
@Override
public List<Row.Entry> top(final int count) throws IOException {
final Iterator<Index> i = this.tables.values().iterator();
Index table, maxtable = null;
@ -471,6 +493,7 @@ public class SplitTable implements Index, Iterable<Row.Entry> {
return maxtable.top(count);
}
@Override
public CloneableIterator<byte[]> keys(final boolean up, final byte[] firstKey) throws IOException {
final List<CloneableIterator<byte[]>> c = new ArrayList<CloneableIterator<byte[]>>(this.tables.size());
final Iterator<Index> i = this.tables.values().iterator();
@ -482,6 +505,7 @@ public class SplitTable implements Index, Iterable<Row.Entry> {
return MergeIterator.cascade(c, this.rowdef.objectOrder, MergeIterator.simpleMerge, up);
}
@Override
public CloneableIterator<Row.Entry> rows(final boolean up, final byte[] firstKey) throws IOException {
final List<CloneableIterator<Row.Entry>> c = new ArrayList<CloneableIterator<Row.Entry>>(this.tables.size());
final Iterator<Index> i = this.tables.values().iterator();
@ -491,6 +515,7 @@ public class SplitTable implements Index, Iterable<Row.Entry> {
return MergeIterator.cascade(c, this.entryOrder, MergeIterator.simpleMerge, up);
}
@Override
public Iterator<Entry> iterator() {
try {
return rows();
@ -499,6 +524,7 @@ public class SplitTable implements Index, Iterable<Row.Entry> {
}
}
@Override
@SuppressWarnings("unchecked")
public synchronized CloneableIterator<Row.Entry> rows() throws IOException {
final CloneableIterator<Row.Entry>[] c = new CloneableIterator[this.tables.size()];
@ -510,6 +536,7 @@ public class SplitTable implements Index, Iterable<Row.Entry> {
return StackIterator.stack(c);
}
@Override
public synchronized void close() {
if (this.tables == null) return;
/*
@ -527,6 +554,7 @@ public class SplitTable implements Index, Iterable<Row.Entry> {
this.tables = null;
}
@Override
public void deleteOnExit() {
for (final Index i: this.tables.values()) i.deleteOnExit();
}

2
source/net/yacy/search/Switchboard.java
Unescape View File

@ -1788,7 +1788,7 @@ public final class Switchboard extends serverSwitch {
// there is a version that is more recent. Load it and re-start with it
this.log.logInfo("AUTO-UPDATE: downloading more recent release " + updateVersion.getUrl());
final File downloaded = updateVersion.downloadRelease();
final boolean devenvironment = new File(this.getAppPath(), ".svn").exists();
final boolean devenvironment = new File(this.getAppPath(), ".git").exists();
if (devenvironment) {
this.log.logInfo("AUTO-UPDATE: omitting update because this is a development environment");
} else if ((downloaded == null) || (!downloaded.exists()) || (downloaded.length() == 0)) {

2
source/net/yacy/search/query/RWIProcess.java
Unescape View File

@ -435,7 +435,7 @@ public final class RWIProcess extends Thread
// we wait at most 30 milliseconds to get a maximum total waiting time of 300 milliseconds for 10 results
long wait = waitTimeRecommendation();
if ( wait > 0 ) {
System.out.println("*** RWIProcess extra wait: " + wait + "ms; expectedRemoteReferences = " + this.expectedRemoteReferences.get() + ", receivedRemoteReferences = " + this.receivedRemoteReferences.get() + ", initialExpectedRemoteReferences = " + this.maxExpectedRemoteReferences.get());
//System.out.println("*** RWIProcess extra wait: " + wait + "ms; expectedRemoteReferences = " + this.expectedRemoteReferences.get() + ", receivedRemoteReferences = " + this.receivedRemoteReferences.get() + ", initialExpectedRemoteReferences = " + this.maxExpectedRemoteReferences.get());
Thread.sleep(wait);
}
// loop as long as we can expect that we should get more results

53
source/net/yacy/search/query/SearchEvent.java
Unescape View File

@ -26,6 +26,7 @@
package net.yacy.search.query;
import java.util.Collections;
import java.util.Iterator;
import java.util.Map;
import java.util.SortedMap;
@ -516,8 +517,8 @@ public final class SearchEvent
private final Semaphore trigger;
public SecondarySearchSuperviser() {
this.abstractsCache = new TreeMap<String, SortedMap<String, StringBuilder>>();
this.checkedPeers = new TreeSet<String>();
this.abstractsCache = Collections.synchronizedSortedMap(new TreeMap<String, SortedMap<String, StringBuilder>>());
this.checkedPeers = Collections.synchronizedSortedSet(new TreeSet<String>());
this.trigger = new Semaphore(0);
}
@ -527,16 +528,14 @@ public final class SearchEvent
* @param wordhash
* @param singleAbstract // a mapping from url-hashes to a string of peer-hashes
*/
public void addAbstract(final String wordhash, final TreeMap<String, StringBuilder> singleAbstract) {
public void addAbstract(final String wordhash, final SortedMap<String, StringBuilder> singleAbstract) {
final SortedMap<String, StringBuilder> oldAbstract;
synchronized ( this.abstractsCache ) {
oldAbstract = this.abstractsCache.get(wordhash);
if ( oldAbstract == null ) {
// new abstracts in the cache
this.abstractsCache.put(wordhash, singleAbstract);
return;
}
}
// extend the abstracts in the cache: join the single abstracts
new Thread() {
@Override
@ -544,11 +543,9 @@ public final class SearchEvent
for ( final Map.Entry<String, StringBuilder> oneref : singleAbstract.entrySet() ) {
final String urlhash = oneref.getKey();
final StringBuilder peerlistNew = oneref.getValue();
synchronized ( oldAbstract ) {
final StringBuilder peerlistOld = oldAbstract.put(urlhash, peerlistNew);
if ( peerlistOld != null ) {
peerlistOld.append(peerlistNew);
}
final StringBuilder peerlistOld = oldAbstract.put(urlhash, peerlistNew);
if ( peerlistOld != null ) {
peerlistOld.append(peerlistNew);
}
}
}
@ -567,27 +564,25 @@ public final class SearchEvent
SortedMap<String, StringBuilder> urlPeerlist;
int p;
boolean hasURL;
synchronized ( this ) {
final Iterator<Map.Entry<String, SortedMap<String, StringBuilder>>> i =
this.abstractsCache.entrySet().iterator();
while ( i.hasNext() ) {
entry = i.next();
word = entry.getKey();
urlPeerlist = entry.getValue();
hasURL = true;
for ( int j = 0; j < urls.length(); j = j + 12 ) {
url = urls.substring(j, j + 12);
peerlist = urlPeerlist.get(url);
p = (peerlist == null) ? -1 : peerlist.indexOf(peerhash);
if ( (p < 0) || (p % 12 != 0) ) {
hasURL = false;
break;
}
}
if ( hasURL ) {
wordlist += word;
final Iterator<Map.Entry<String, SortedMap<String, StringBuilder>>> i =
this.abstractsCache.entrySet().iterator();
while ( i.hasNext() ) {
entry = i.next();
word = entry.getKey();
urlPeerlist = entry.getValue();
hasURL = true;
for ( int j = 0; j < urls.length(); j = j + 12 ) {
url = urls.substring(j, j + 12);
peerlist = urlPeerlist.get(url);
p = (peerlist == null) ? -1 : peerlist.indexOf(peerhash);
if ( (p < 0) || (p % 12 != 0) ) {
hasURL = false;
break;
}
}
if ( hasURL ) {
wordlist += word;
}
}
return wordlist;
}

5
yacy-svn-4.spec
Unescape View File

@ -9,8 +9,8 @@ Version: @REPL_VERSION@_@REPL_REVISION_NR@
Release: 3
License: GPL
Group: Application/Internet
Source0: svn://svn.berlios.de/yacy/trunk
URL: http://www.yacy.net/yacy/
Source0: git@gitorious.org:yacy/rc1.git
URL: http://yacy.net
Requires: bash
Requires: sudo
Requires: coreutils
@ -90,6 +90,7 @@ cp AUTHORS COPYRIGHT ChangeLog gpl.txt readme.txt ${RPM_BUILD_ROOT}$YACYDDIR/
install -m 744 *.sh ${RPM_BUILD_ROOT}$YACYCDIR/ # start/stop/kill scripts
rm -r `find ${RPM_BUILD_ROOT}/ -type d -name '.svn'` # delete unwanted .svn-folders
rm -r `find ${RPM_BUILD_ROOT}/ -type d -name '.git'` # delete unwanted .git-folders
# location for init-script
install -d ${RPM_BUILD_ROOT}/etc/init.d/

Write Preview
Loading…
Cancel
Save