You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
519 lines
21 KiB
519 lines
21 KiB
/**
|
|
* QueryGoal
|
|
* Copyright 2012 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
|
|
* First published 16.11.2005 on http://yacy.net
|
|
*
|
|
* This library is free software; you can redistribute it and/or
|
|
* modify it under the terms of the GNU Lesser General Public
|
|
* License as published by the Free Software Foundation; either
|
|
* version 2.1 of the License, or (at your option) any later version.
|
|
*
|
|
* This library is distributed in the hope that it will be useful,
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
* Lesser General Public License for more details.
|
|
*
|
|
* You should have received a copy of the GNU Lesser General Public License
|
|
* along with this program in the file lgpl21.txt
|
|
* If not, see <http://www.gnu.org/licenses/>.
|
|
*/
|
|
|
|
|
|
package net.yacy.search.query;
|
|
|
|
import java.io.UnsupportedEncodingException;
|
|
import java.net.URLEncoder;
|
|
import java.nio.charset.StandardCharsets;
|
|
import java.util.ArrayList;
|
|
import java.util.Collection;
|
|
import java.util.Iterator;
|
|
import java.util.List;
|
|
import java.util.Locale;
|
|
import java.util.Set;
|
|
import java.util.SortedSet;
|
|
import java.util.TreeSet;
|
|
|
|
import org.apache.http.HttpStatus;
|
|
|
|
import net.yacy.cora.document.WordCache;
|
|
import net.yacy.cora.federate.solr.connector.AbstractSolrConnector;
|
|
import net.yacy.cora.order.NaturalOrder;
|
|
import net.yacy.cora.protocol.Domains;
|
|
import net.yacy.cora.storage.HandleSet;
|
|
import net.yacy.document.parser.html.AbstractScraper;
|
|
import net.yacy.document.parser.html.CharacterCoding;
|
|
import net.yacy.kelondro.data.word.Word;
|
|
import net.yacy.kelondro.util.SetTools;
|
|
import net.yacy.search.index.Segment;
|
|
import net.yacy.search.schema.CollectionSchema;
|
|
|
|
public class QueryGoal {
|
|
|
|
private static char space = ' ';
|
|
private static char sq = '\'';
|
|
private static char dq = '"';
|
|
private static String seps = ".:;#*`,!$%()=?^<>/&_";
|
|
|
|
public String query_original;
|
|
private HandleSet include_hashes, exclude_hashes;
|
|
private final NormalizedWords include_words, exclude_words;
|
|
private final ArrayList<String> include_strings, exclude_strings;
|
|
|
|
public static class NormalizedWords extends TreeSet<String> {
|
|
|
|
private static final long serialVersionUID = -3050851079671868007L;
|
|
|
|
public NormalizedWords() {
|
|
super(NaturalOrder.naturalComparator);
|
|
}
|
|
|
|
public NormalizedWords(String[] rawWords) {
|
|
super(NaturalOrder.naturalComparator);
|
|
for (String word: rawWords) super.add(word.toLowerCase(Locale.ENGLISH));
|
|
}
|
|
|
|
public NormalizedWords(Collection<String> rawWords) {
|
|
super(NaturalOrder.naturalComparator);
|
|
for (String word: rawWords) super.add(word.toLowerCase(Locale.ENGLISH));
|
|
}
|
|
|
|
@Override
|
|
public boolean add(String word) {
|
|
return super.add(word.toLowerCase(Locale.ENGLISH));
|
|
}
|
|
|
|
@Override
|
|
public boolean contains(Object word) {
|
|
if (!(word instanceof String)) return false;
|
|
return super.contains(((String) word).toLowerCase(Locale.ENGLISH));
|
|
}
|
|
}
|
|
|
|
public QueryGoal(HandleSet include_hashes, HandleSet exclude_hashes) {
|
|
this.query_original = null;
|
|
this.include_words = new NormalizedWords();
|
|
this.exclude_words = new NormalizedWords();
|
|
this.include_strings = new ArrayList<String>();
|
|
this.exclude_strings = new ArrayList<String>();
|
|
this.include_hashes = include_hashes;
|
|
this.exclude_hashes = exclude_hashes;
|
|
}
|
|
|
|
/**
|
|
* Creates a QueryGoal from a search query string
|
|
* @param query_words search string (the actual search terms, excluding application specific modifier)
|
|
*/
|
|
public QueryGoal(String query_words) {
|
|
assert query_words != null;
|
|
this.query_original = query_words;
|
|
this.include_words = new NormalizedWords();
|
|
this.exclude_words = new NormalizedWords();
|
|
this.include_strings = new ArrayList<String>();
|
|
this.exclude_strings = new ArrayList<String>();
|
|
|
|
// remove funny symbols
|
|
query_words = CharacterCoding.html2unicode(AbstractScraper.stripAllTags(query_words.toCharArray())).toLowerCase().trim();
|
|
int c;
|
|
for (int i = 0; i < seps.length(); i++) {
|
|
while ((c = query_words.indexOf(seps.charAt(i))) >= 0) {
|
|
query_words = query_words.substring(0, c) + (((c + 1) < query_words.length()) ? (' ' + query_words.substring(c + 1)) : "");
|
|
}
|
|
}
|
|
|
|
// parse first quoted strings
|
|
parseQuery(query_words, this.include_strings, this.exclude_strings);
|
|
|
|
// .. end then take these strings apart to generate word lists
|
|
for (String s: this.include_strings) parseQuery(s, this.include_words, this.include_words);
|
|
for (String s: this.exclude_strings) parseQuery(s, this.exclude_words, this.exclude_words);
|
|
|
|
WordCache.learn(this.include_words);
|
|
WordCache.learn(this.exclude_words);
|
|
|
|
this.include_hashes = null;
|
|
this.exclude_hashes = null;
|
|
}
|
|
|
|
|
|
/*
|
|
* EBNF of a query
|
|
*
|
|
* query = {whitespace, phrase}, [whitespace]
|
|
* whitespace = space, {space}
|
|
* space = ' '
|
|
* phrase = ['-'], string
|
|
* string = {any character without sq, dq and whitespace} | sq, {any character without sq}, sq | dq, {any character without dq}, dq
|
|
* sq = '\''
|
|
* dq = '"'
|
|
*/
|
|
private static void parseQuery(String s, Collection<String> include_string, Collection<String> exclude_string) {
|
|
while (s.length() > 0) {
|
|
// parse query
|
|
int p = 0;
|
|
while (p < s.length() && s.charAt(p) == space) p++;
|
|
s = s.substring(p);
|
|
if (s.length() == 0) return;
|
|
|
|
// parse phrase
|
|
boolean inc = true;
|
|
if (s.charAt(0) == '-') {
|
|
inc = false;
|
|
s = s.substring(1);
|
|
} else if (s.charAt(0) == '+') {
|
|
inc = true;
|
|
s = s.substring(1);
|
|
}
|
|
if (s.length() == 0) return;
|
|
|
|
// parse string
|
|
char stop = space;
|
|
if (s.charAt(0) == dq) {
|
|
stop = s.charAt(0);
|
|
s = s.substring(1);
|
|
} else if (s.charAt(0) == sq) {
|
|
stop = s.charAt(0);
|
|
s = s.substring(1);
|
|
}
|
|
p = 0;
|
|
while (p < s.length() && s.charAt(p) != stop) p++;
|
|
String string = s.substring(0, p);
|
|
p++; // go behind the stop character (eats up space, sq and dq)
|
|
s = p < s.length() ? s.substring(p) : "";
|
|
if (string.length() > 0) {
|
|
if (inc) {
|
|
if (!include_string.contains(string)) include_string.add(string);
|
|
} else {
|
|
if (!exclude_string.contains(string)) exclude_string.add(string);
|
|
}
|
|
}
|
|
}
|
|
// in case that the include_string contains several entries including 1-char tokens and also more-than-1-char tokens,
|
|
// then remove the 1-char tokens to prevent that we are to strict. This will make it possible to be a bit more fuzzy
|
|
// in the search where it is appropriate
|
|
boolean contains_single = false, contains_multiple = false;
|
|
for (String token: include_string) {
|
|
if (token.length() == 1) contains_single = true; else contains_multiple = true;
|
|
}
|
|
if (contains_single && contains_multiple) {
|
|
Iterator<String> i = include_string.iterator();
|
|
while (i.hasNext()) if (i.next().length() == 1) i.remove();
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Search query string (without YaCy specific modifier like site:xxx or /smb)
|
|
* the modifier are held separately in a search paramter modifier
|
|
*
|
|
* @param encodeHTML
|
|
* @return the search query string
|
|
*/
|
|
public String getQueryString(final boolean encodeHTML) {
|
|
if (this.query_original == null) return null;
|
|
String ret;
|
|
if (encodeHTML){
|
|
try {
|
|
ret = URLEncoder.encode(this.query_original, StandardCharsets.UTF_8.name());
|
|
} catch (final UnsupportedEncodingException e) {
|
|
ret = this.query_original;
|
|
}
|
|
} else {
|
|
ret = this.query_original;
|
|
}
|
|
return ret;
|
|
}
|
|
|
|
/**
|
|
* @return a set of hashes of words to be included in the search result.
|
|
* if possible, use getIncludeWords instead
|
|
*/
|
|
public HandleSet getIncludeHashes() {
|
|
if (this.include_hashes == null) this.include_hashes = Word.words2hashesHandles(include_words);
|
|
return this.include_hashes;
|
|
}
|
|
|
|
/**
|
|
* @return a set of hashes of words to be excluded in the search result
|
|
* if possible, use getExcludeWords instead
|
|
*/
|
|
public HandleSet getExcludeHashes() {
|
|
if (this.exclude_hashes == null) this.exclude_hashes = Word.words2hashesHandles(exclude_words);
|
|
return this.exclude_hashes;
|
|
}
|
|
|
|
public int getIncludeSize() {
|
|
assert this.include_hashes == null || this.include_words.size() == 0 || this.include_hashes.size() == this.include_words.size();
|
|
return this.include_hashes == null ? this.include_words.size() : this.include_hashes.size();
|
|
}
|
|
|
|
public int getExcludeSize() {
|
|
assert this.exclude_hashes == null || this.exclude_words.size() == 0 || this.exclude_hashes.size() == this.exclude_words.size();
|
|
return this.exclude_hashes == null ? this.exclude_words.size() : this.exclude_hashes.size();
|
|
}
|
|
|
|
/**
|
|
* @return an iterator on the set of words to be included in the search result
|
|
*/
|
|
public Iterator<String> getIncludeWords() {
|
|
return this.include_words.iterator();
|
|
}
|
|
|
|
/**
|
|
* @return a copy of the set of words to be included in the search result
|
|
*/
|
|
public Set<String> getIncludeWordsSet() {
|
|
return new NormalizedWords(this.include_words);
|
|
}
|
|
|
|
/**
|
|
* @return an iterator on the set of words to be excluded from the search result
|
|
*/
|
|
public Iterator<String> getExcludeWords() {
|
|
return this.exclude_words.iterator();
|
|
}
|
|
|
|
/**
|
|
* @return a copy of the set of words to be excluded from the search result
|
|
*/
|
|
public Set<String> getExcludeWordsSet() {
|
|
return new NormalizedWords(this.exclude_words);
|
|
}
|
|
|
|
/**
|
|
* @return a list of include strings which reproduces the original order of the search words and quotation
|
|
*/
|
|
public Iterator<String> getIncludeStrings() {
|
|
return this.include_strings.iterator();
|
|
}
|
|
|
|
/**
|
|
* @return a list of exclude strings which reproduces the original order of the search words and quotation
|
|
*/
|
|
public Iterator<String> getExcludeStrings() {
|
|
return this.exclude_strings.iterator();
|
|
}
|
|
|
|
public void removeIncludeWords(Set<String> words) {
|
|
if (!words.isEmpty()) {
|
|
SetTools.excludeDestructiveByTestSmallInLarge(this.exclude_words, words); //remove stopwords
|
|
SetTools.excludeDestructiveByTestSmallInLarge(this.exclude_strings, words); //remove stopwords
|
|
if (include_hashes != null) for (String word: words) this.include_hashes.remove(Word.word2hash(word));
|
|
}
|
|
}
|
|
|
|
/**
|
|
* the include string may be useful (and better) for highlight/snippet computation
|
|
* @return the query string containing only the positive literals (includes) and without whitespace characters
|
|
*/
|
|
public String getIncludeString() {
|
|
if (this.include_strings.size() == 0) return "";
|
|
StringBuilder sb = new StringBuilder(10 * include_strings.size());
|
|
for (String s: this.include_strings) sb.append(s).append(' ');
|
|
return sb.toString().substring(0, sb.length() - 1);
|
|
}
|
|
|
|
public boolean isCatchall() {
|
|
if (this.include_hashes != null && this.include_hashes.has(Segment.catchallHash)) return true;
|
|
if (this.include_strings == null || this.include_strings.size() != 1) return false;
|
|
return (this.include_strings.contains(Segment.catchallString));
|
|
}
|
|
|
|
public boolean containsInclude(String word) {
|
|
if (word == null || word.length() == 0) return false;
|
|
|
|
String t = word.toLowerCase(Locale.ENGLISH);
|
|
return this.include_strings.contains(t) || this.include_words.contains(t);
|
|
}
|
|
|
|
public boolean matches(String text) {
|
|
if (text == null || text.length() == 0) return false;
|
|
|
|
// parse special requests
|
|
if (isCatchall()) return true;
|
|
|
|
String t = text.toLowerCase(Locale.ENGLISH);
|
|
for (String i: this.include_strings) if (t.indexOf(i.toLowerCase()) < 0) return false;
|
|
for (String e: this.exclude_strings) if (t.indexOf(e.toLowerCase()) >= 0) return false;
|
|
return true;
|
|
}
|
|
|
|
public void filterOut(final SortedSet<String> blueList) {
|
|
// filter out words that appear in this set
|
|
// this is applied to the queryHashes
|
|
for (String word: blueList) {
|
|
this.include_words.remove(word);
|
|
this.include_strings.remove(word);
|
|
}
|
|
final HandleSet blues = Word.words2hashesHandles(blueList);
|
|
for (final byte[] b: blues) this.include_hashes.remove(b);
|
|
}
|
|
|
|
/**
|
|
* Generate a Solr filter query to receive valid urls
|
|
*
|
|
* This filters out error-urls.
|
|
* On noimages=true a filter is added to exclude links to images
|
|
* using the content_type (as well as urls with common image file extension)
|
|
*
|
|
* @param noimages true if filter for images should be included
|
|
* @return Solr filter query
|
|
*/
|
|
public List<String> collectionTextFilterQuery(boolean noimages) {
|
|
final ArrayList<String> fqs = new ArrayList<>();
|
|
|
|
// add filter to prevent that results come from failed urls
|
|
fqs.add(CollectionSchema.httpstatus_i.getSolrFieldName() + ":" + HttpStatus.SC_OK);
|
|
if (noimages) {
|
|
fqs.add("-" + CollectionSchema.content_type.getSolrFieldName() + ":(image/*)");
|
|
fqs.add("-" + CollectionSchema.url_file_ext_s.getSolrFieldName() + ":(jpg OR png OR gif)");
|
|
}
|
|
|
|
return fqs;
|
|
}
|
|
|
|
public StringBuilder collectionTextQuery() {
|
|
|
|
// parse special requests
|
|
if (isCatchall()) return new StringBuilder(AbstractSolrConnector.CATCHALL_QUERY);
|
|
|
|
// add goal query
|
|
return getGoalQuery();
|
|
}
|
|
|
|
/**
|
|
* Generate a Solr filter query to receive valid image results.
|
|
*
|
|
* This filters error-urls out and includes urls with mime image/*, as well
|
|
* as urls with links to images when strict is false.
|
|
* We use the mime (image/*) only to find images as the parser assigned the
|
|
* best mime to index documents. This applies also to parsed file systems.
|
|
* This ensures that no text urls with image-fileextension is returned
|
|
* (as some large internet sites like to use such urls)
|
|
*
|
|
* @param strict when true, do not include non-image urls with links to images
|
|
* @return Solr filter query for image urls
|
|
*/
|
|
public List<String> collectionImageFilterQuery(final boolean strict) {
|
|
final ArrayList<String> fqs = new ArrayList<>();
|
|
|
|
// add filter to prevent that results come from failed urls
|
|
fqs.add(CollectionSchema.httpstatus_i.getSolrFieldName() + ":" + HttpStatus.SC_OK);
|
|
StringBuilder filter = new StringBuilder(CollectionSchema.content_type.getSolrFieldName()).append(":(image/*)");
|
|
if (!strict) {
|
|
filter.append(" OR ").append(CollectionSchema.images_urlstub_sxt.getSolrFieldName())
|
|
.append(AbstractSolrConnector.CATCHALL_DTERM);
|
|
}
|
|
fqs.add(filter.toString());
|
|
return fqs;
|
|
}
|
|
|
|
/**
|
|
* Generate Solr filter queries to receive valid audio content results.
|
|
*
|
|
* This filters out documents with bad HTTP status and includes documents with MIME type matching the prefix audio/* as well
|
|
* documents with links to audio content when strict is false.
|
|
*
|
|
* @param strict when true, do not include non-audio urls with links to audio
|
|
* @return Solr filter queries for audio content URLs
|
|
*/
|
|
public List<String> collectionAudioFilterQuery(final boolean strict) {
|
|
final ArrayList<String> fqs = new ArrayList<>();
|
|
|
|
// add filter to prevent that results come from failed urls
|
|
fqs.add(CollectionSchema.httpstatus_i.getSolrFieldName() + ":" + HttpStatus.SC_OK);
|
|
StringBuilder filter = new StringBuilder(CollectionSchema.content_type.getSolrFieldName()).append(":(audio/*)");
|
|
if (!strict) {
|
|
filter.append(" OR ").append(CollectionSchema.audiolinkscount_i.getSolrFieldName()).append(":[1 TO *]");
|
|
}
|
|
fqs.add(filter.toString());
|
|
return fqs;
|
|
}
|
|
|
|
/**
|
|
* Generate Solr filter queries to receive valid video content results.
|
|
*
|
|
* This filters out documents with bad HTTP status and includes documents with MIME type matching the prefix video/* as well
|
|
* documents with links to video content when strict is false.
|
|
*
|
|
* @param strict when true, do not include non-video urls with links to video
|
|
* @return Solr filter queries for video content URLs
|
|
*/
|
|
public List<String> collectionVideoFilterQuery(final boolean strict) {
|
|
final ArrayList<String> fqs = new ArrayList<>();
|
|
|
|
// add filter to prevent that results come from failed urls
|
|
fqs.add(CollectionSchema.httpstatus_i.getSolrFieldName() + ":" + HttpStatus.SC_OK);
|
|
StringBuilder filter = new StringBuilder(CollectionSchema.content_type.getSolrFieldName()).append(":(video/*)");
|
|
if (!strict) {
|
|
filter.append(" OR ").append(CollectionSchema.videolinkscount_i.getSolrFieldName()).append(":[1 TO *]");
|
|
}
|
|
fqs.add(filter.toString());
|
|
return fqs;
|
|
}
|
|
|
|
/**
|
|
* Generate Solr filter queries to receive valid application specific content results.
|
|
*
|
|
* This filters out documents with bad HTTP status and includes documents with MIME type matching the prefix application/* as well
|
|
* docuemnts with links to application specific content when strict is false.
|
|
*
|
|
* @param strict when true, do not include non-video urls with links to video
|
|
* @return Solr filter queries for application specific content URLs
|
|
*/
|
|
public List<String> collectionApplicationFilterQuery(final boolean strict) {
|
|
final ArrayList<String> fqs = new ArrayList<>();
|
|
|
|
// add filter to prevent that results come from failed urls
|
|
fqs.add(CollectionSchema.httpstatus_i.getSolrFieldName() + ":" + HttpStatus.SC_OK);
|
|
StringBuilder filter = new StringBuilder(CollectionSchema.content_type.getSolrFieldName())
|
|
.append(":(application/*)");
|
|
if (!strict) {
|
|
filter.append(" OR ").append(CollectionSchema.applinkscount_i.getSolrFieldName()).append(":[1 TO *]");
|
|
}
|
|
fqs.add(filter.toString());
|
|
return fqs;
|
|
}
|
|
|
|
public StringBuilder collectionImageQuery(final QueryModifier modifier) {
|
|
final StringBuilder q = new StringBuilder(80);
|
|
|
|
// parse special requests
|
|
if (isCatchall()) return new StringBuilder(AbstractSolrConnector.CATCHALL_QUERY);
|
|
|
|
// add goal query
|
|
StringBuilder w = getGoalQuery();
|
|
q.append(w);
|
|
|
|
// combine these queries for all relevant fields
|
|
if (w.length() > 0) {
|
|
String hostname = modifier == null || modifier.sitehost == null || modifier.sitehost.length() == 0 ? null : Domains.getSmartSLD(modifier.sitehost);
|
|
q.append(" AND (");
|
|
q.append('(').append(CollectionSchema.images_text_t.getSolrFieldName()).append(':').append(hostname == null ? w : "(" + w + " " /*NOT an OR!, the hostname shall only boost*/ + hostname + ")").append("^100.0) OR ");
|
|
q.append('(').append(CollectionSchema.title.getSolrFieldName()).append(':').append(w).append("^50.0) OR ");
|
|
q.append('(').append(CollectionSchema.keywords.getSolrFieldName()).append(':').append(w).append("^10.0) OR ");
|
|
q.append('(').append(CollectionSchema.text_t.getSolrFieldName()).append(':').append(w).append(')');
|
|
q.append(')');
|
|
}
|
|
|
|
return q;
|
|
}
|
|
|
|
private StringBuilder getGoalQuery() {
|
|
int wc = 0;
|
|
StringBuilder w = new StringBuilder(80);
|
|
for (String s: include_strings) {
|
|
if (Segment.catchallString.equals(s)) continue;
|
|
if (wc > 0) w.append(" AND ");
|
|
if (s.indexOf('~') >= 0 || s.indexOf('*') >= 0 || s.indexOf('?') >= 0) w.append(s); else w.append(dq).append(s).append(dq);
|
|
wc++;
|
|
}
|
|
for (String s: exclude_strings){
|
|
if (wc > 0) w.append(" AND -");
|
|
if (s.indexOf('~') >= 0 || s.indexOf('*') >= 0 || s.indexOf('?') >= 0) w.append(s); else w.append(dq).append(s).append(dq);
|
|
wc++;
|
|
}
|
|
if (wc > 1) {w.insert(0, '('); w.append(')');}
|
|
return w;
|
|
}
|
|
|
|
}
|