boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

log queries anonymous from gsa+solr requests

Browse Source
pull/1/head
Michael Peter Christen 12 years ago
parent ab6ec4ec52
commit 48a82bc705
4 changed files with 76 additions and 25 deletions
Show Stats Download Patch File Download Diff File

8
htroot/gsa/searchresult.java
Unescape View File

@ -30,6 +30,7 @@ import net.yacy.cora.protocol.RequestHeader;
import net.yacy.cora.services.federated.solr.GSAResponseWriter;
import net.yacy.kelondro.logging.Log;
import net.yacy.search.Switchboard;
import net.yacy.search.query.AccessTracker;
import net.yacy.search.query.SnippetProcess;
import net.yacy.search.solr.EmbeddedSolrConnector;
@ -38,6 +39,7 @@ import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.util.FastWriter;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.search.DocSlice;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
@ -98,6 +100,7 @@ public class searchresult {
//post.put(, post.remove("site"));//required, example: col1|col2
//post.put(, post.remove("client"));//required, example: myfrontend
//post.put(, post.remove("output"));//required, example: xml,xml_no_dtd
String q = post.get(CommonParams.Q, "");
post.put(CommonParams.ROWS, post.remove("num"));
post.put(CommonParams.ROWS, Math.min(post.getInt("num", 10), (authenticated) ? 5000 : 100));
post.remove("num");
@ -152,6 +155,11 @@ public class searchresult {
try {ow.close();} catch (IOException e1) {}
}
// log result
Object rv = response.getValues().get("response");
if (rv != null && rv instanceof DocSlice) {
AccessTracker.addToDump(q, Integer.toString(((DocSlice) rv).matches()));
}
return null;
}
}

8
htroot/solr/select.java
Unescape View File

@ -35,6 +35,7 @@ import net.yacy.cora.services.federated.solr.OpensearchResponseWriter;
import net.yacy.kelondro.logging.Log;
import net.yacy.search.Switchboard;
import net.yacy.search.SwitchboardConstants;
import net.yacy.search.query.AccessTracker;
import net.yacy.search.query.SnippetProcess;
import net.yacy.search.solr.EmbeddedSolrConnector;
import net.yacy.search.solr.SolrServlet;
@ -48,6 +49,7 @@ import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.QueryResponseWriter;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.response.XSLTResponseWriter;
import org.apache.solr.search.DocSlice;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
@ -132,6 +134,7 @@ public class select {
// rename post fields according to result style
if (!post.containsKey(CommonParams.Q)) post.put(CommonParams.Q, post.remove("query")); // sru patch
String q = post.get(CommonParams.Q, "");
if (!post.containsKey(CommonParams.START)) post.put(CommonParams.START, post.remove("startRecord")); // sru patch
post.put(CommonParams.ROWS, Math.min(post.getInt(CommonParams.ROWS, post.getInt("maximumRecords", 10)), (authenticated) ? 5000 : 100));
post.remove("maximumRecords");
@ -183,6 +186,11 @@ public class select {
try {ow.close();} catch (IOException e1) {}
}
// log result
Object rv = response.getValues().get("response");
if (rv != null && rv instanceof DocSlice) {
AccessTracker.addToDump(q, Integer.toString(((DocSlice) rv).matches()));
}
return null;
}
}

3
source/net/yacy/search/Switchboard.java
Unescape View File

@ -291,6 +291,7 @@ public final class Switchboard extends serverSwitch
// set loglevel and log
setLog(new Log("SWITCHBOARD"));
AccessTracker.setDumpFile(new File("DATA/LOG/queries.log"));
// set default peer name
Seed.ANON_PREFIX = getConfig("peernameprefix", "_anon");
@ -1589,7 +1590,7 @@ public final class Switchboard extends serverSwitch
Cache.close();
this.tables.close();
Domains.close();
AccessTracker.dumpLog(new File("DATA/LOG/queries.log"));
AccessTracker.dumpLog();
Switchboard.urlBlacklist.close();
UPnP.deletePortMapping();
this.tray.remove();

82
source/net/yacy/search/query/AccessTracker.java
Unescape View File

@ -42,15 +42,23 @@ import net.yacy.kelondro.util.MemoryControl;
public class AccessTracker {
private final static long DUMP_PERIOD = 60000L;
public static final int minSize = 100;
public static final int maxSize = 1000;
public static final int maxAge = 24 * 60 * 60 * 1000;
public enum Location {local, remote}
private static LinkedList<QueryParams> localSearches = new LinkedList<QueryParams>();
private static LinkedList<QueryParams> remoteSearches = new LinkedList<QueryParams>();
private static ArrayList<String> log = new ArrayList<String>();
private static final LinkedList<QueryParams> localSearches = new LinkedList<QueryParams>();
private static final LinkedList<QueryParams> remoteSearches = new LinkedList<QueryParams>();
private static final ArrayList<String> log = new ArrayList<String>();
private static long lastLogDump = System.currentTimeMillis();
private static File dumpFile = null;
public static void setDumpFile(File f) {
dumpFile = f;
}
public static void add(final Location location, final QueryParams query) {
if (location == Location.local) synchronized (localSearches) {add(localSearches, query);}
@ -100,36 +108,62 @@ public class AccessTracker {
}
private static void addToDump(final QueryParams query) {
//if (query.resultcount == 0) return;
if (query.queryString == null || query.queryString.isEmpty()) return;
addToDump(query.queryString, Integer.toString(query.resultcount), new Date(query.starttime));
}
public static void addToDump(String querystring, String resultcount) {
addToDump(querystring, resultcount, new Date());
if (lastLogDump + DUMP_PERIOD < System.currentTimeMillis()) {
lastLogDump = System.currentTimeMillis();
dumpLog();
}
}
private static void addToDump(String querystring, String resultcount, Date d) {
//if (query.resultcount == 0) return;
if (querystring == null || querystring.isEmpty()) return;
final StringBuilder sb = new StringBuilder(40);
sb.append(GenericFormatter.SHORT_SECOND_FORMATTER.format(new Date(query.starttime)));
sb.append(GenericFormatter.SHORT_SECOND_FORMATTER.format(d));
sb.append(' ');
sb.append(Integer.toString(query.resultcount));
sb.append(resultcount);
sb.append(' ');
sb.append(query.queryString);
log.add(sb.toString());
sb.append(querystring);
synchronized (log) {
log.add(sb.toString());
}
}
public static void dumpLog(final File file) {
public static void dumpLog() {
while (!localSearches.isEmpty()) {
addToDump(localSearches.removeFirst());
}
RandomAccessFile raf = null;
try {
raf = new RandomAccessFile(file, "rw");
raf.seek(raf.length());
for (final String s: log) {
raf.write(UTF8.getBytes(s));
raf.writeByte(10);
Thread t = new Thread() {
@Override
public void run() {
ArrayList<String> logCopy = new ArrayList<String>();
synchronized (log) {
logCopy.addAll(log);
log.clear();
}
RandomAccessFile raf = null;
try {
raf = new RandomAccessFile(dumpFile, "rw");
raf.seek(raf.length());
for (final String s: logCopy) {
raf.write(UTF8.getBytes(s));
raf.writeByte(10);
}
logCopy.clear();
} catch (final FileNotFoundException e) {
Log.logException(e);
} catch (final IOException e) {
Log.logException(e);
} finally {
if (raf != null) try {raf.close();} catch (IOException e) {}
}
}
log.clear();
} catch (final FileNotFoundException e) {
Log.logException(e);
} catch (final IOException e) {
Log.logException(e);
} finally {
if (raf != null) try {raf.close();} catch (IOException e) {}
}
};
t.start();
}
}

Write Preview
Loading…
Cancel
Save