boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

Merge branch 'master' of https://git.gitorious.org/yacy/rc1.git

Browse Source
pull/1/head
sixcooler 14 years ago
parent d9c56aa37a da55a359e9
commit 6714a9364a
11 changed files with 132 additions and 46 deletions
Show Stats Download Patch File Download Diff File

14
htroot/Crawler_p.java
Unescape View File

@ -174,7 +174,13 @@ public class Crawler_p {
} else if (crawlingStartURL.isFTP()) {
newcrawlingMustMatch = "ftp://" + crawlingStartURL.getHost();
} else {
newcrawlingMustMatch = "https?://" + crawlingStartURL.getHost();
final String host = crawlingStartURL.getHost();
if (host.startsWith("www.")) {
newcrawlingMustMatch = "https?://" + crawlingStartURL.getHost();
} else {
// if the www is not given we accept that also
newcrawlingMustMatch = "https?://(www.)?" + crawlingStartURL.getHost();
}
}
if (subPath) newcrawlingMustMatch += crawlingStartURL.getPath();
newcrawlingMustMatch += ".*";
@ -374,7 +380,7 @@ public class Crawler_p {
String tagStr = tags.toString();
if (tagStr.length() > 2 && tagStr.startsWith("[") && tagStr.endsWith("]")) tagStr = tagStr.substring(1, tagStr.length() - 2);
// we will create always a bookmark to use this to track crawled hosts
// we will create always a bookmark to use this to track crawled hosts
final BookmarksDB.Bookmark bookmark = sb.bookmarksDB.createBookmark(crawlingStart, "admin");
if (bookmark != null) {
bookmark.setProperty(BookmarksDB.Bookmark.BOOKMARK_TITLE, title);
@ -384,11 +390,11 @@ public class Crawler_p {
bookmark.setTags(tags, true);
sb.bookmarksDB.saveBookmark(bookmark);
}
// do the same for ymarks
// TODO: could a non admin user add crawls?
sb.tables.bookmarks.createBookmark(sb.loader, url, YMarkTables.USER_ADMIN, true, "crawlStart", "/Crawl Start");
// liftoff!
prop.put("info", "8");//start msg
prop.putHTML("info_crawlingURL", post.get("crawlingURL"));

10
htroot/YMarks.java
Unescape View File

@ -10,18 +10,18 @@ public class YMarks {
final Switchboard sb = (Switchboard) env;
final serverObjects prop = new serverObjects();
final UserDB.Entry user = sb.userDB.getUser(header);
final boolean isAdmin = (sb.verifyAuthentication(header, true));
final boolean isAdmin = (sb.verifyAuthentication(header, false));
final boolean isAuthUser = user!= null && user.hasRight(UserDB.AccessRight.BOOKMARK_RIGHT);
if(isAdmin || isAuthUser) {
prop.put("login", 1);
String bmk_user = (isAuthUser ? user.getUserName() : YMarkTables.USER_ADMIN);
final String bmk_user = (isAuthUser ? user.getUserName() : YMarkTables.USER_ADMIN);
prop.putHTML("user", bmk_user.substring(0,1).toUpperCase() + bmk_user.substring(1));
} else {
prop.put("login", 0);
}
return prop;
}
}

2
htroot/index.java
Unescape View File

@ -97,8 +97,8 @@ public class index {
prop.put("excluded", "0");
prop.put("combine", "0");
prop.put("resultbottomline", "0");
prop.put("maximumRecords", maximumRecords);
prop.put("searchoptions", searchoptions);
prop.put("searchoptions_maximumRecords", maximumRecords);
prop.put("searchoptions_count-10", (count == 10) ? "1" : "0");
prop.put("searchoptions_count-50", (count == 50) ? "1" : "0");
prop.put("searchoptions_count-100", (count == 100) ? "1" : "0");

25
htroot/yacysearch.java
Unescape View File

@ -75,6 +75,7 @@ import net.yacy.search.ranking.RankingProfile;
import net.yacy.search.snippet.ContentDomain;
import de.anomic.data.DidYouMean;
import de.anomic.data.UserDB;
import de.anomic.data.ymark.YMarkTables;
import de.anomic.server.serverCore;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
@ -483,7 +484,7 @@ public class yacysearch {
// if a minus-button was hit, remove a special reference first
if (post != null && post.containsKey("deleteref")) {
try {
if (!sb.verifyAuthentication(header, true)) {
if (!sb.verifyAuthentication(header, false)) {
prop.put("AUTHENTICATE", "admin log-in"); // force log-in
return prop;
}
@ -500,6 +501,9 @@ public class yacysearch {
map.put("refid", "");
sb.peers.newsPool.publishMyNews(sb.peers.mySeed(), NewsPool.CATEGORY_SURFTIPP_VOTE_ADD, map);
}
// delete the search history since this still shows the entry
SearchEventCache.delete(delHash);
} catch (final IOException e) {
Log.logException(e);
}
@ -507,7 +511,7 @@ public class yacysearch {
// if a plus-button was hit, create new voting message
if (post != null && post.containsKey("recommendref")) {
if (!sb.verifyAuthentication(header, true)) {
if (!sb.verifyAuthentication(header, false)) {
prop.put("AUTHENTICATE", "admin log-in"); // force log-in
return prop;
}
@ -535,6 +539,23 @@ public class yacysearch {
}
}
// if a bookmarks-button was hit, create new bookmark entry
if (post != null && post.containsKey("bookmarkref")) {
if (!sb.verifyAuthentication(header, false)) {
prop.put("AUTHENTICATE", "admin log-in"); // force log-in
return prop;
}
final String bookmarkHash = post.get("bookmarkref", ""); // urlhash
final URIMetadataRow urlentry = indexSegment.urlMetadata().load(UTF8.getBytes(bookmarkHash));
if (urlentry != null) {
final URIMetadataRow.Components metadata = urlentry.metadata();
try {
sb.tables.bookmarks.createBookmark(sb.loader, metadata.url(), YMarkTables.USER_ADMIN, true, "searchresult", "/search");
} catch (final Throwable e) {
}
}
}
// do the search
final HandleSet queryHashes = Word.words2hashesHandles(query[0]);
final Pattern snippetPattern = QueryParams.stringSearchPattern(originalquerystring);

4
htroot/yacysearchitem.html
Unescape View File

@ -9,7 +9,9 @@
<img width="16" height="9" src="/env/grafics/heuristic_new.gif" title="heuristic:#[name]# (new link)" style="width:16px; height:9px;" alt="heuristic:#[name]# (new link)"/>
#(/heuristic)#
#(authorized)#::
<a href="/api/ymarks/add_ymark.html?urlHash=#[urlhash]#&folders=/Search+Result&redirect=/Bookmarks.html?edit=#[urlhash]#" class="bookmarklink" title="bookmark"><img width="11" height="11" src="/env/grafics/empty.gif" title="bookmark" alt="bookmark" class="recommendIcon" /></a>
#(bookmark)#::
<a href="#[bookmarklink]#" class="bookmarklink" title="recommend"><img width="11" height="11" src="/env/grafics/empty.gif" title="bookmark" alt="bookmark" class="recommendIcon" /></a>
#(/bookmark)#
#(recommend)#
<img width="11" height="11" src="/env/grafics/empty.gif" title="" alt="recommend" class="recommendIcon" />
<img width="11" height="11" src="/env/grafics/empty.gif" title="" alt="delete" class="deleteIcon" />

13
htroot/yacysearchitem.java
Unescape View File

@ -119,7 +119,7 @@ public class yacysearchitem {
Log.logException(e1);
faviconURL = null;
}
final String resource = theQuery.domType.toString();
prop.put("content", 1); // switch on specific content
prop.put("content_showDate", sb.getConfigBool("search.result.show.date", true) ? 1 : 0);
prop.put("content_showSize", sb.getConfigBool("search.result.show.size", true) ? 1 : 0);
@ -127,11 +127,14 @@ public class yacysearchitem {
prop.put("content_showParser", sb.getConfigBool("search.result.show.parser", true) ? 1 : 0);
prop.put("content_showPictures", sb.getConfigBool("search.result.show.pictures", true) ? 1 : 0);
prop.put("content_authorized", authenticated ? "1" : "0");
final String urlhash = ASCII.String(result.hash());
prop.put("content_authorized_bookmark", sb.tables.bookmarks.hasBookmark("admin", urlhash) ? "0" : "1");
prop.putHTML("content_authorized_bookmark_bookmarklink", "/yacysearch.html?query=" + theQuery.queryString.replace(' ', '+') + "&Enter=Search&count=" + theQuery.displayResults() + "&offset=" + (theQuery.neededResults() - theQuery.displayResults()) + "&order=" + crypt.simpleEncode(theQuery.ranking.toExternalString()) + "&resource=" + resource + "&time=3&bookmarkref=" + urlhash + "&urlmaskfilter=.*");
prop.put("content_authorized_recommend", (sb.peers.newsPool.getSpecific(NewsPool.OUTGOING_DB, NewsPool.CATEGORY_SURFTIPP_ADD, "url", result.urlstring()) == null) ? "1" : "0");
prop.putHTML("content_authorized_recommend_deletelink", "/yacysearch.html?query=" + theQuery.queryString.replace(' ', '+') + "&Enter=Search&count=" + theQuery.displayResults() + "&offset=" + (theQuery.neededResults() - theQuery.displayResults()) + "&order=" + crypt.simpleEncode(theQuery.ranking.toExternalString()) + "&resource=local&time=3&deleteref=" + ASCII.String(result.hash()) + "&urlmaskfilter=.*");
prop.putHTML("content_authorized_recommend_recommendlink", "/yacysearch.html?query=" + theQuery.queryString.replace(' ', '+') + "&Enter=Search&count=" + theQuery.displayResults() + "&offset=" + (theQuery.neededResults() - theQuery.displayResults()) + "&order=" + crypt.simpleEncode(theQuery.ranking.toExternalString()) + "&resource=local&time=3&recommendref=" + ASCII.String(result.hash()) + "&urlmaskfilter=.*");
prop.put("content_authorized_urlhash", ASCII.String(result.hash()));
final String resulthashString = ASCII.String(result.hash());
prop.putHTML("content_authorized_recommend_deletelink", "/yacysearch.html?query=" + theQuery.queryString.replace(' ', '+') + "&Enter=Search&count=" + theQuery.displayResults() + "&offset=" + (theQuery.neededResults() - theQuery.displayResults()) + "&order=" + crypt.simpleEncode(theQuery.ranking.toExternalString()) + "&resource=" + resource + "&time=3&deleteref=" + urlhash + "&urlmaskfilter=.*");
prop.putHTML("content_authorized_recommend_recommendlink", "/yacysearch.html?query=" + theQuery.queryString.replace(' ', '+') + "&Enter=Search&count=" + theQuery.displayResults() + "&offset=" + (theQuery.neededResults() - theQuery.displayResults()) + "&order=" + crypt.simpleEncode(theQuery.ranking.toExternalString()) + "&resource=" + resource + "&time=3&recommendref=" + urlhash + "&urlmaskfilter=.*");
prop.put("content_authorized_urlhash", urlhash);
final String resulthashString = urlhash;
prop.putHTML("content_title", result.title());
prop.putXML("content_title-xml", result.title());
prop.putJSON("content_title-json", result.title());

67
source/de/anomic/data/ymark/YMarkTables.java
Unescape View File

@ -37,6 +37,7 @@ import java.util.TreeMap;
import java.util.TreeSet;
import java.util.regex.Pattern;
import net.yacy.cora.document.ASCII;
import net.yacy.document.Document;
import net.yacy.document.Parser.Failure;
import net.yacy.kelondro.blob.Tables;
@ -200,7 +201,7 @@ public class YMarkTables {
final Pattern p = Pattern.compile(patternBuilder.toString());
return this.worktables.iterator(bmk_table, YMarkEntry.BOOKMARK.FOLDERS.key(), p);
}
public Iterator<Tables.Row> getBookmarksByTag(final String bmk_user, final String[] tagArray) throws IOException {
final String bmk_table = TABLES.BOOKMARKS.tablename(bmk_user);
final StringBuilder patternBuilder = new StringBuilder(BUFFER_LENGTH);
@ -213,15 +214,15 @@ public class YMarkTables {
patternBuilder.append(p3);
patternBuilder.append('|');
}
patternBuilder.deleteCharAt(patternBuilder.length()-1);
patternBuilder.deleteCharAt(patternBuilder.length()-1);
patternBuilder.append(p6);
patternBuilder.append(tagArray.length);
patternBuilder.append('}');
final Pattern p = Pattern.compile(patternBuilder.toString(), Pattern.CASE_INSENSITIVE);
return this.worktables.iterator(bmk_table, YMarkEntry.BOOKMARK.TAGS.key(), p);
}
public List<Row> orderBookmarksBy(final Iterator<Row> rowIterator, final String sortname, final String sortorder) {
final List<Row> sortList = new ArrayList<Row>();
Row row;
@ -230,7 +231,7 @@ public class YMarkTables {
if(row != null)
sortList.add(row);
}
Collections.sort(sortList, new TablesRowComparator(sortname, sortorder));
Collections.sort(sortList, new TablesRowComparator(sortname, sortorder));
return sortList;
}
@ -243,7 +244,7 @@ public class YMarkTables {
addBookmark(bmk_user, bmk, merge, true);
}
}
public void replaceTags(final Iterator<Row> rowIterator, final String bmk_user, final String tagString, final String replaceString) throws IOException, RowSpaceExceededException {
final String[] tagArray = YMarkUtil.cleanTagsString(tagString).split(YMarkUtil.TAGS_SEPARATOR);
final StringBuilder tagStringBuilder = new StringBuilder(BUFFER_LENGTH);
@ -251,41 +252,41 @@ public class YMarkTables {
while (rowIterator.hasNext()) {
row = rowIterator.next();
if(row != null) {
for(int i=0; i<tagArray.length; i++) {
for (final String element : tagArray) {
tagStringBuilder.setLength(0);
tagStringBuilder.append(row.get(YMarkEntry.BOOKMARK.TAGS.key(), YMarkEntry.BOOKMARK.TAGS.deflt()));
int start = tagStringBuilder.indexOf(tagArray[i]);
int start = tagStringBuilder.indexOf(element);
int end = start;
while (end<=tagStringBuilder.length() && end != -1) {
if (end == (tagStringBuilder.length())) {
if (end-start == tagArray[i].length()) {
if (end-start == element.length()) {
if (start > 0)
start--; // also replace the tag separator
start--; // also replace the tag separator
tagStringBuilder.replace(start, end, YMarkUtil.EMPTY_STRING);
}
break;
} else if (tagStringBuilder.charAt(end) == ',') {
if (end-start == tagArray[i].length()) {
} else if (tagStringBuilder.charAt(end) == ',') {
if (end-start == element.length()) {
if (start > 0)
start--; // also replace the tag separator
start--; // also replace the tag separator
tagStringBuilder.replace(start, end, YMarkUtil.EMPTY_STRING);
} else {
start = tagStringBuilder.indexOf(tagArray[i], end+1);
start = tagStringBuilder.indexOf(element, end+1);
end = start;
}
} else if (tagStringBuilder.charAt(end) == ' ') {
start = tagStringBuilder.indexOf(tagArray[i], end);
start = tagStringBuilder.indexOf(element, end);
end = start;
} else {
end++;
}
}
}
tagStringBuilder.append(YMarkUtil.TAGS_SEPARATOR);
tagStringBuilder.append(replaceString);
row.put(YMarkEntry.BOOKMARK.TAGS.key(), YMarkUtil.cleanTagsString(tagStringBuilder.toString()));
this.worktables.update(TABLES.BOOKMARKS.tablename(bmk_user), row);
}
}
}
}
}
@ -306,26 +307,27 @@ public class YMarkTables {
bmk.put(YMarkEntry.BOOKMARK.DATE_VISITED.key(), (new YMarkDate()).toString());
addBookmark(bmk_user, bmk, true, true);
}
public void createBookmark(final LoaderDispatcher loader, final String url, final String bmk_user, final boolean autotag, final String tagsString, final String foldersString) throws IOException, Failure, RowSpaceExceededException {
createBookmark(loader, new DigestURI(url), bmk_user, autotag, tagsString, foldersString);
}
public void createBookmark(final LoaderDispatcher loader, final DigestURI url, final String bmk_user, final boolean autotag, final String tagsString, final String foldersString) throws IOException, Failure, RowSpaceExceededException {
final YMarkEntry bmk_entry = new YMarkEntry(false);
final YMarkEntry bmk_entry = new YMarkEntry(false);
final YMarkMetadata meta = new YMarkMetadata(url);
final Document document = meta.loadDocument(loader);
final EnumMap<YMarkMetadata.METADATA, String> metadata = meta.loadMetadata();
bmk_entry.put(YMarkEntry.BOOKMARK.URL.key(), url.toNormalform(true, false));
if(!this.worktables.has(YMarkTables.TABLES.BOOKMARKS.tablename(bmk_user), YMarkUtil.getBookmarkId(url.toNormalform(true, false)))) {
final EnumMap<YMarkMetadata.METADATA, String> metadata = meta.loadMetadata();
final String urls = url.toNormalform(true, false);
bmk_entry.put(YMarkEntry.BOOKMARK.URL.key(), urls);
if(!this.worktables.has(YMarkTables.TABLES.BOOKMARKS.tablename(bmk_user), YMarkUtil.getBookmarkId(urls))) {
bmk_entry.put(YMarkEntry.BOOKMARK.PUBLIC.key(), "false");
bmk_entry.put(YMarkEntry.BOOKMARK.TITLE.key(), metadata.get(YMarkMetadata.METADATA.TITLE));
bmk_entry.put(YMarkEntry.BOOKMARK.DESC.key(), metadata.get(YMarkMetadata.METADATA.DESCRIPTION));
bmk_entry.put(YMarkEntry.BOOKMARK.DESC.key(), metadata.get(YMarkMetadata.METADATA.DESCRIPTION));
}
bmk_entry.put(YMarkEntry.BOOKMARK.FOLDERS.key(), YMarkUtil.cleanFoldersString(foldersString));
final StringBuilder strb = new StringBuilder();
if(autotag) {
if(autotag) {
final String autotags = YMarkAutoTagger.autoTag(document, 3, this.worktables.bookmarks.getTags(bmk_user));
strb.append(autotags);
}
@ -333,10 +335,19 @@ public class YMarkTables {
strb.append(YMarkUtil.TAGS_SEPARATOR);
strb.append(tagsString);
}
bmk_entry.put(YMarkEntry.BOOKMARK.TAGS.key(),YMarkUtil.cleanTagsString(strb.toString()));
bmk_entry.put(YMarkEntry.BOOKMARK.TAGS.key(),YMarkUtil.cleanTagsString(strb.toString()));
this.worktables.bookmarks.addBookmark(bmk_user, bmk_entry, true, true);
}
public boolean hasBookmark(final String bmk_user, final String urlhash) {
final String bmk_table = TABLES.BOOKMARKS.tablename(bmk_user);
try {
return this.worktables.has(bmk_table, ASCII.getBytes(urlhash));
} catch (final IOException e) {
return false;
}
}
public void addBookmark(final String bmk_user, final YMarkEntry bmk, final boolean mergeTags, final boolean mergeFolders) throws IOException, RowSpaceExceededException {
final String bmk_table = TABLES.BOOKMARKS.tablename(bmk_user);
final String date = String.valueOf(System.currentTimeMillis());

1
source/net/yacy/peers/Seed.java
Unescape View File

@ -434,6 +434,7 @@ public class Seed implements Cloneable, Comparable<Seed>, Comparator<Seed> {
* @return the hexadecimal representation of the given base64 hash
*/
public static String b64Hash2hexHash(final String b64Hash) {
if (b64Hash.length() > 12) return "";
// the hash string represents 12 * 6 bit = 72 bits. This is too much for a long integer.
return Digest.encodeHex(Base64Order.enhancedCoder.decode(b64Hash));
}

15
source/net/yacy/search/query/QueryParams.java
Unescape View File

@ -63,6 +63,21 @@ public final class QueryParams {
public enum Searchdom {
LOCAL, CLUSTER, GLOBAL;
public static Searchdom contentdomParser(final String dom) {
if ("local".equals(dom)) return LOCAL;
else if ("global".equals(dom)) return GLOBAL;
else if ("cluster".equals(dom)) return CLUSTER;
return LOCAL;
}
@Override
public String toString() {
if (this == LOCAL) return "local";
else if (this == CLUSTER) return "global"; // yes thats right: global, not cluster because a cluster search is a global search
else if (this == GLOBAL) return "global";
return "local";
}
}
private static final String ampersand = "&amp;";

7
source/net/yacy/search/query/SearchEventCache.java
Unescape View File

@ -65,6 +65,13 @@ public class SearchEventCache {
if (oldEvent == null) cacheInsert++;
}
public static boolean delete(final String urlhash) {
for (final SearchEvent event: lastEvents.values()) {
if (event.result().delete(urlhash)) return true;
}
return false;
}
public static void cleanupEvents(boolean all) {
// remove old events in the event cache
if (MemoryControl.shortStatus()) all = true;

20
source/net/yacy/search/query/SnippetProcess.java
Unescape View File

@ -36,6 +36,7 @@ import net.yacy.cora.document.MultiProtocolURI;
import net.yacy.cora.protocol.ResponseHeader;
import net.yacy.cora.ranking.ScoreMap;
import net.yacy.cora.ranking.WeakPriorityBlockingQueue;
import net.yacy.cora.ranking.WeakPriorityBlockingQueue.Element;
import net.yacy.cora.ranking.WeakPriorityBlockingQueue.ReverseElement;
import net.yacy.cora.services.federated.solr.SolrConnector;
import net.yacy.cora.services.federated.yacy.CacheStrategy;
@ -532,4 +533,23 @@ public class SnippetProcess {
}
// finished, no more actions possible here
}
/**
* delete a specific entry from the search results
* this is used if the user clicks on a '-' sign beside the search result
* @param urlhash
* @return true if an entry was deleted, false otherwise
*/
public boolean delete(final String urlhash) {
final Iterator<Element<ResultEntry>> i = this.result.iterator();
Element<ResultEntry> entry;
while (i.hasNext()) {
entry = i.next();
if (urlhash.equals(ASCII.String(entry.getElement().url().hash()))) {
i.remove();
return true;
}
}
return false;
}
}

Write Preview
Loading…
Cancel
Save