diff --git a/htroot/Crawler_p.java b/htroot/Crawler_p.java index 6663fe06f..740862cf7 100644 --- a/htroot/Crawler_p.java +++ b/htroot/Crawler_p.java @@ -174,7 +174,13 @@ public class Crawler_p { } else if (crawlingStartURL.isFTP()) { newcrawlingMustMatch = "ftp://" + crawlingStartURL.getHost(); } else { - newcrawlingMustMatch = "https?://" + crawlingStartURL.getHost(); + final String host = crawlingStartURL.getHost(); + if (host.startsWith("www.")) { + newcrawlingMustMatch = "https?://" + crawlingStartURL.getHost(); + } else { + // if the www is not given we accept that also + newcrawlingMustMatch = "https?://(www.)?" + crawlingStartURL.getHost(); + } } if (subPath) newcrawlingMustMatch += crawlingStartURL.getPath(); newcrawlingMustMatch += ".*"; @@ -374,7 +380,7 @@ public class Crawler_p { String tagStr = tags.toString(); if (tagStr.length() > 2 && tagStr.startsWith("[") && tagStr.endsWith("]")) tagStr = tagStr.substring(1, tagStr.length() - 2); - // we will create always a bookmark to use this to track crawled hosts + // we will create always a bookmark to use this to track crawled hosts final BookmarksDB.Bookmark bookmark = sb.bookmarksDB.createBookmark(crawlingStart, "admin"); if (bookmark != null) { bookmark.setProperty(BookmarksDB.Bookmark.BOOKMARK_TITLE, title); @@ -384,11 +390,11 @@ public class Crawler_p { bookmark.setTags(tags, true); sb.bookmarksDB.saveBookmark(bookmark); } - + // do the same for ymarks // TODO: could a non admin user add crawls? sb.tables.bookmarks.createBookmark(sb.loader, url, YMarkTables.USER_ADMIN, true, "crawlStart", "/Crawl Start"); - + // liftoff! prop.put("info", "8");//start msg prop.putHTML("info_crawlingURL", post.get("crawlingURL")); diff --git a/htroot/YMarks.java b/htroot/YMarks.java index ecc8a3339..8fd0019b9 100644 --- a/htroot/YMarks.java +++ b/htroot/YMarks.java @@ -10,18 +10,18 @@ public class YMarks { final Switchboard sb = (Switchboard) env; final serverObjects prop = new serverObjects(); final UserDB.Entry user = sb.userDB.getUser(header); - final boolean isAdmin = (sb.verifyAuthentication(header, true)); + final boolean isAdmin = (sb.verifyAuthentication(header, false)); final boolean isAuthUser = user!= null && user.hasRight(UserDB.AccessRight.BOOKMARK_RIGHT); - + if(isAdmin || isAuthUser) { prop.put("login", 1); - String bmk_user = (isAuthUser ? user.getUserName() : YMarkTables.USER_ADMIN); + final String bmk_user = (isAuthUser ? user.getUserName() : YMarkTables.USER_ADMIN); prop.putHTML("user", bmk_user.substring(0,1).toUpperCase() + bmk_user.substring(1)); - + } else { prop.put("login", 0); } - + return prop; } } \ No newline at end of file diff --git a/htroot/index.java b/htroot/index.java index e6a97ac04..400274020 100644 --- a/htroot/index.java +++ b/htroot/index.java @@ -97,8 +97,8 @@ public class index { prop.put("excluded", "0"); prop.put("combine", "0"); prop.put("resultbottomline", "0"); - prop.put("maximumRecords", maximumRecords); prop.put("searchoptions", searchoptions); + prop.put("searchoptions_maximumRecords", maximumRecords); prop.put("searchoptions_count-10", (count == 10) ? "1" : "0"); prop.put("searchoptions_count-50", (count == 50) ? "1" : "0"); prop.put("searchoptions_count-100", (count == 100) ? "1" : "0"); diff --git a/htroot/yacysearch.java b/htroot/yacysearch.java index 0c788b31f..51898b970 100644 --- a/htroot/yacysearch.java +++ b/htroot/yacysearch.java @@ -75,6 +75,7 @@ import net.yacy.search.ranking.RankingProfile; import net.yacy.search.snippet.ContentDomain; import de.anomic.data.DidYouMean; import de.anomic.data.UserDB; +import de.anomic.data.ymark.YMarkTables; import de.anomic.server.serverCore; import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; @@ -483,7 +484,7 @@ public class yacysearch { // if a minus-button was hit, remove a special reference first if (post != null && post.containsKey("deleteref")) { try { - if (!sb.verifyAuthentication(header, true)) { + if (!sb.verifyAuthentication(header, false)) { prop.put("AUTHENTICATE", "admin log-in"); // force log-in return prop; } @@ -500,6 +501,9 @@ public class yacysearch { map.put("refid", ""); sb.peers.newsPool.publishMyNews(sb.peers.mySeed(), NewsPool.CATEGORY_SURFTIPP_VOTE_ADD, map); } + + // delete the search history since this still shows the entry + SearchEventCache.delete(delHash); } catch (final IOException e) { Log.logException(e); } @@ -507,7 +511,7 @@ public class yacysearch { // if a plus-button was hit, create new voting message if (post != null && post.containsKey("recommendref")) { - if (!sb.verifyAuthentication(header, true)) { + if (!sb.verifyAuthentication(header, false)) { prop.put("AUTHENTICATE", "admin log-in"); // force log-in return prop; } @@ -535,6 +539,23 @@ public class yacysearch { } } + // if a bookmarks-button was hit, create new bookmark entry + if (post != null && post.containsKey("bookmarkref")) { + if (!sb.verifyAuthentication(header, false)) { + prop.put("AUTHENTICATE", "admin log-in"); // force log-in + return prop; + } + final String bookmarkHash = post.get("bookmarkref", ""); // urlhash + final URIMetadataRow urlentry = indexSegment.urlMetadata().load(UTF8.getBytes(bookmarkHash)); + if (urlentry != null) { + final URIMetadataRow.Components metadata = urlentry.metadata(); + try { + sb.tables.bookmarks.createBookmark(sb.loader, metadata.url(), YMarkTables.USER_ADMIN, true, "searchresult", "/search"); + } catch (final Throwable e) { + } + } + } + // do the search final HandleSet queryHashes = Word.words2hashesHandles(query[0]); final Pattern snippetPattern = QueryParams.stringSearchPattern(originalquerystring); diff --git a/htroot/yacysearchitem.html b/htroot/yacysearchitem.html index 80bc5451f..87ed0b642 100644 --- a/htroot/yacysearchitem.html +++ b/htroot/yacysearchitem.html @@ -9,7 +9,9 @@ heuristic:#[name]# (new link) #(/heuristic)# #(authorized)#:: - bookmark + #(bookmark)#:: + bookmark + #(/bookmark)# #(recommend)# recommend delete diff --git a/htroot/yacysearchitem.java b/htroot/yacysearchitem.java index e4b0d1706..491840b2f 100644 --- a/htroot/yacysearchitem.java +++ b/htroot/yacysearchitem.java @@ -119,7 +119,7 @@ public class yacysearchitem { Log.logException(e1); faviconURL = null; } - + final String resource = theQuery.domType.toString(); prop.put("content", 1); // switch on specific content prop.put("content_showDate", sb.getConfigBool("search.result.show.date", true) ? 1 : 0); prop.put("content_showSize", sb.getConfigBool("search.result.show.size", true) ? 1 : 0); @@ -127,11 +127,14 @@ public class yacysearchitem { prop.put("content_showParser", sb.getConfigBool("search.result.show.parser", true) ? 1 : 0); prop.put("content_showPictures", sb.getConfigBool("search.result.show.pictures", true) ? 1 : 0); prop.put("content_authorized", authenticated ? "1" : "0"); + final String urlhash = ASCII.String(result.hash()); + prop.put("content_authorized_bookmark", sb.tables.bookmarks.hasBookmark("admin", urlhash) ? "0" : "1"); + prop.putHTML("content_authorized_bookmark_bookmarklink", "/yacysearch.html?query=" + theQuery.queryString.replace(' ', '+') + "&Enter=Search&count=" + theQuery.displayResults() + "&offset=" + (theQuery.neededResults() - theQuery.displayResults()) + "&order=" + crypt.simpleEncode(theQuery.ranking.toExternalString()) + "&resource=" + resource + "&time=3&bookmarkref=" + urlhash + "&urlmaskfilter=.*"); prop.put("content_authorized_recommend", (sb.peers.newsPool.getSpecific(NewsPool.OUTGOING_DB, NewsPool.CATEGORY_SURFTIPP_ADD, "url", result.urlstring()) == null) ? "1" : "0"); - prop.putHTML("content_authorized_recommend_deletelink", "/yacysearch.html?query=" + theQuery.queryString.replace(' ', '+') + "&Enter=Search&count=" + theQuery.displayResults() + "&offset=" + (theQuery.neededResults() - theQuery.displayResults()) + "&order=" + crypt.simpleEncode(theQuery.ranking.toExternalString()) + "&resource=local&time=3&deleteref=" + ASCII.String(result.hash()) + "&urlmaskfilter=.*"); - prop.putHTML("content_authorized_recommend_recommendlink", "/yacysearch.html?query=" + theQuery.queryString.replace(' ', '+') + "&Enter=Search&count=" + theQuery.displayResults() + "&offset=" + (theQuery.neededResults() - theQuery.displayResults()) + "&order=" + crypt.simpleEncode(theQuery.ranking.toExternalString()) + "&resource=local&time=3&recommendref=" + ASCII.String(result.hash()) + "&urlmaskfilter=.*"); - prop.put("content_authorized_urlhash", ASCII.String(result.hash())); - final String resulthashString = ASCII.String(result.hash()); + prop.putHTML("content_authorized_recommend_deletelink", "/yacysearch.html?query=" + theQuery.queryString.replace(' ', '+') + "&Enter=Search&count=" + theQuery.displayResults() + "&offset=" + (theQuery.neededResults() - theQuery.displayResults()) + "&order=" + crypt.simpleEncode(theQuery.ranking.toExternalString()) + "&resource=" + resource + "&time=3&deleteref=" + urlhash + "&urlmaskfilter=.*"); + prop.putHTML("content_authorized_recommend_recommendlink", "/yacysearch.html?query=" + theQuery.queryString.replace(' ', '+') + "&Enter=Search&count=" + theQuery.displayResults() + "&offset=" + (theQuery.neededResults() - theQuery.displayResults()) + "&order=" + crypt.simpleEncode(theQuery.ranking.toExternalString()) + "&resource=" + resource + "&time=3&recommendref=" + urlhash + "&urlmaskfilter=.*"); + prop.put("content_authorized_urlhash", urlhash); + final String resulthashString = urlhash; prop.putHTML("content_title", result.title()); prop.putXML("content_title-xml", result.title()); prop.putJSON("content_title-json", result.title()); diff --git a/source/de/anomic/data/ymark/YMarkTables.java b/source/de/anomic/data/ymark/YMarkTables.java index f40c4a29f..84f439c9e 100644 --- a/source/de/anomic/data/ymark/YMarkTables.java +++ b/source/de/anomic/data/ymark/YMarkTables.java @@ -37,6 +37,7 @@ import java.util.TreeMap; import java.util.TreeSet; import java.util.regex.Pattern; +import net.yacy.cora.document.ASCII; import net.yacy.document.Document; import net.yacy.document.Parser.Failure; import net.yacy.kelondro.blob.Tables; @@ -200,7 +201,7 @@ public class YMarkTables { final Pattern p = Pattern.compile(patternBuilder.toString()); return this.worktables.iterator(bmk_table, YMarkEntry.BOOKMARK.FOLDERS.key(), p); } - + public Iterator getBookmarksByTag(final String bmk_user, final String[] tagArray) throws IOException { final String bmk_table = TABLES.BOOKMARKS.tablename(bmk_user); final StringBuilder patternBuilder = new StringBuilder(BUFFER_LENGTH); @@ -213,15 +214,15 @@ public class YMarkTables { patternBuilder.append(p3); patternBuilder.append('|'); } - patternBuilder.deleteCharAt(patternBuilder.length()-1); + patternBuilder.deleteCharAt(patternBuilder.length()-1); patternBuilder.append(p6); - + patternBuilder.append(tagArray.length); patternBuilder.append('}'); final Pattern p = Pattern.compile(patternBuilder.toString(), Pattern.CASE_INSENSITIVE); return this.worktables.iterator(bmk_table, YMarkEntry.BOOKMARK.TAGS.key(), p); } - + public List orderBookmarksBy(final Iterator rowIterator, final String sortname, final String sortorder) { final List sortList = new ArrayList(); Row row; @@ -230,7 +231,7 @@ public class YMarkTables { if(row != null) sortList.add(row); } - Collections.sort(sortList, new TablesRowComparator(sortname, sortorder)); + Collections.sort(sortList, new TablesRowComparator(sortname, sortorder)); return sortList; } @@ -243,7 +244,7 @@ public class YMarkTables { addBookmark(bmk_user, bmk, merge, true); } } - + public void replaceTags(final Iterator rowIterator, final String bmk_user, final String tagString, final String replaceString) throws IOException, RowSpaceExceededException { final String[] tagArray = YMarkUtil.cleanTagsString(tagString).split(YMarkUtil.TAGS_SEPARATOR); final StringBuilder tagStringBuilder = new StringBuilder(BUFFER_LENGTH); @@ -251,41 +252,41 @@ public class YMarkTables { while (rowIterator.hasNext()) { row = rowIterator.next(); if(row != null) { - for(int i=0; i 0) - start--; // also replace the tag separator + start--; // also replace the tag separator tagStringBuilder.replace(start, end, YMarkUtil.EMPTY_STRING); } break; - } else if (tagStringBuilder.charAt(end) == ',') { - if (end-start == tagArray[i].length()) { + } else if (tagStringBuilder.charAt(end) == ',') { + if (end-start == element.length()) { if (start > 0) - start--; // also replace the tag separator + start--; // also replace the tag separator tagStringBuilder.replace(start, end, YMarkUtil.EMPTY_STRING); } else { - start = tagStringBuilder.indexOf(tagArray[i], end+1); + start = tagStringBuilder.indexOf(element, end+1); end = start; } } else if (tagStringBuilder.charAt(end) == ' ') { - start = tagStringBuilder.indexOf(tagArray[i], end); + start = tagStringBuilder.indexOf(element, end); end = start; } else { end++; } - } + } tagStringBuilder.append(YMarkUtil.TAGS_SEPARATOR); tagStringBuilder.append(replaceString); row.put(YMarkEntry.BOOKMARK.TAGS.key(), YMarkUtil.cleanTagsString(tagStringBuilder.toString())); this.worktables.update(TABLES.BOOKMARKS.tablename(bmk_user), row); } - } + } } } @@ -306,26 +307,27 @@ public class YMarkTables { bmk.put(YMarkEntry.BOOKMARK.DATE_VISITED.key(), (new YMarkDate()).toString()); addBookmark(bmk_user, bmk, true, true); } - + public void createBookmark(final LoaderDispatcher loader, final String url, final String bmk_user, final boolean autotag, final String tagsString, final String foldersString) throws IOException, Failure, RowSpaceExceededException { createBookmark(loader, new DigestURI(url), bmk_user, autotag, tagsString, foldersString); } - + public void createBookmark(final LoaderDispatcher loader, final DigestURI url, final String bmk_user, final boolean autotag, final String tagsString, final String foldersString) throws IOException, Failure, RowSpaceExceededException { - - final YMarkEntry bmk_entry = new YMarkEntry(false); + + final YMarkEntry bmk_entry = new YMarkEntry(false); final YMarkMetadata meta = new YMarkMetadata(url); final Document document = meta.loadDocument(loader); - final EnumMap metadata = meta.loadMetadata(); - bmk_entry.put(YMarkEntry.BOOKMARK.URL.key(), url.toNormalform(true, false)); - if(!this.worktables.has(YMarkTables.TABLES.BOOKMARKS.tablename(bmk_user), YMarkUtil.getBookmarkId(url.toNormalform(true, false)))) { + final EnumMap metadata = meta.loadMetadata(); + final String urls = url.toNormalform(true, false); + bmk_entry.put(YMarkEntry.BOOKMARK.URL.key(), urls); + if(!this.worktables.has(YMarkTables.TABLES.BOOKMARKS.tablename(bmk_user), YMarkUtil.getBookmarkId(urls))) { bmk_entry.put(YMarkEntry.BOOKMARK.PUBLIC.key(), "false"); bmk_entry.put(YMarkEntry.BOOKMARK.TITLE.key(), metadata.get(YMarkMetadata.METADATA.TITLE)); - bmk_entry.put(YMarkEntry.BOOKMARK.DESC.key(), metadata.get(YMarkMetadata.METADATA.DESCRIPTION)); + bmk_entry.put(YMarkEntry.BOOKMARK.DESC.key(), metadata.get(YMarkMetadata.METADATA.DESCRIPTION)); } bmk_entry.put(YMarkEntry.BOOKMARK.FOLDERS.key(), YMarkUtil.cleanFoldersString(foldersString)); final StringBuilder strb = new StringBuilder(); - if(autotag) { + if(autotag) { final String autotags = YMarkAutoTagger.autoTag(document, 3, this.worktables.bookmarks.getTags(bmk_user)); strb.append(autotags); } @@ -333,10 +335,19 @@ public class YMarkTables { strb.append(YMarkUtil.TAGS_SEPARATOR); strb.append(tagsString); } - bmk_entry.put(YMarkEntry.BOOKMARK.TAGS.key(),YMarkUtil.cleanTagsString(strb.toString())); + bmk_entry.put(YMarkEntry.BOOKMARK.TAGS.key(),YMarkUtil.cleanTagsString(strb.toString())); this.worktables.bookmarks.addBookmark(bmk_user, bmk_entry, true, true); } - + + public boolean hasBookmark(final String bmk_user, final String urlhash) { + final String bmk_table = TABLES.BOOKMARKS.tablename(bmk_user); + try { + return this.worktables.has(bmk_table, ASCII.getBytes(urlhash)); + } catch (final IOException e) { + return false; + } + } + public void addBookmark(final String bmk_user, final YMarkEntry bmk, final boolean mergeTags, final boolean mergeFolders) throws IOException, RowSpaceExceededException { final String bmk_table = TABLES.BOOKMARKS.tablename(bmk_user); final String date = String.valueOf(System.currentTimeMillis()); diff --git a/source/net/yacy/peers/Seed.java b/source/net/yacy/peers/Seed.java index a5426f9e4..c06d4bf78 100644 --- a/source/net/yacy/peers/Seed.java +++ b/source/net/yacy/peers/Seed.java @@ -434,6 +434,7 @@ public class Seed implements Cloneable, Comparable, Comparator { * @return the hexadecimal representation of the given base64 hash */ public static String b64Hash2hexHash(final String b64Hash) { + if (b64Hash.length() > 12) return ""; // the hash string represents 12 * 6 bit = 72 bits. This is too much for a long integer. return Digest.encodeHex(Base64Order.enhancedCoder.decode(b64Hash)); } diff --git a/source/net/yacy/search/query/QueryParams.java b/source/net/yacy/search/query/QueryParams.java index c75f4d3a3..cb0c9c688 100644 --- a/source/net/yacy/search/query/QueryParams.java +++ b/source/net/yacy/search/query/QueryParams.java @@ -63,6 +63,21 @@ public final class QueryParams { public enum Searchdom { LOCAL, CLUSTER, GLOBAL; + + public static Searchdom contentdomParser(final String dom) { + if ("local".equals(dom)) return LOCAL; + else if ("global".equals(dom)) return GLOBAL; + else if ("cluster".equals(dom)) return CLUSTER; + return LOCAL; + } + + @Override + public String toString() { + if (this == LOCAL) return "local"; + else if (this == CLUSTER) return "global"; // yes thats right: global, not cluster because a cluster search is a global search + else if (this == GLOBAL) return "global"; + return "local"; + } } private static final String ampersand = "&"; diff --git a/source/net/yacy/search/query/SearchEventCache.java b/source/net/yacy/search/query/SearchEventCache.java index 24e706c7b..92d1a4445 100644 --- a/source/net/yacy/search/query/SearchEventCache.java +++ b/source/net/yacy/search/query/SearchEventCache.java @@ -65,6 +65,13 @@ public class SearchEventCache { if (oldEvent == null) cacheInsert++; } + public static boolean delete(final String urlhash) { + for (final SearchEvent event: lastEvents.values()) { + if (event.result().delete(urlhash)) return true; + } + return false; + } + public static void cleanupEvents(boolean all) { // remove old events in the event cache if (MemoryControl.shortStatus()) all = true; diff --git a/source/net/yacy/search/query/SnippetProcess.java b/source/net/yacy/search/query/SnippetProcess.java index f130dfc26..a120403b5 100644 --- a/source/net/yacy/search/query/SnippetProcess.java +++ b/source/net/yacy/search/query/SnippetProcess.java @@ -36,6 +36,7 @@ import net.yacy.cora.document.MultiProtocolURI; import net.yacy.cora.protocol.ResponseHeader; import net.yacy.cora.ranking.ScoreMap; import net.yacy.cora.ranking.WeakPriorityBlockingQueue; +import net.yacy.cora.ranking.WeakPriorityBlockingQueue.Element; import net.yacy.cora.ranking.WeakPriorityBlockingQueue.ReverseElement; import net.yacy.cora.services.federated.solr.SolrConnector; import net.yacy.cora.services.federated.yacy.CacheStrategy; @@ -532,4 +533,23 @@ public class SnippetProcess { } // finished, no more actions possible here } + + /** + * delete a specific entry from the search results + * this is used if the user clicks on a '-' sign beside the search result + * @param urlhash + * @return true if an entry was deleted, false otherwise + */ + public boolean delete(final String urlhash) { + final Iterator> i = this.result.iterator(); + Element entry; + while (i.hasNext()) { + entry = i.next(); + if (urlhash.equals(ASCII.String(entry.getElement().url().hash()))) { + i.remove(); + return true; + } + } + return false; + } }