boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

back to 649 :/

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@651 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
borg-0300 20 years ago
parent 5194511e8e
commit 81cb8feb15
3 changed files with 28 additions and 70 deletions
Show Stats Download Patch File Download Diff File

16
source/de/anomic/http/httpdProxyHandler.java
Unescape View File

@ -79,7 +79,6 @@ import java.util.logging.Level;
import java.util.logging.LogManager;
import java.util.logging.Logger;
import java.util.zip.GZIPOutputStream;
import de.anomic.htmlFilter.htmlFilterContentTransformer;
import de.anomic.htmlFilter.htmlFilterOutputStream;
import de.anomic.htmlFilter.htmlFilterTransformer;
@ -94,7 +93,6 @@ import de.anomic.server.logging.serverLog;
import de.anomic.server.logging.serverMiniLogFormatter;
import de.anomic.yacy.yacyCore;
public final class httpdProxyHandler extends httpdAbstractHandler implements httpdHandler {
// static variables
@ -110,15 +108,14 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
public static int remoteProxyPort = -1;
public static String remoteProxyNoProxy = "";
public static String[] remoteProxyNoProxyPatterns = null;
private static final HashSet remoteProxyAllowProxySet = new HashSet();
private static final HashSet remoteProxyDisallowProxySet = new HashSet();
private static final HashSet remoteProxyDisallowProxySet = new HashSet();
private static htmlFilterTransformer transformer = null;
public static final String userAgent = "yacy (" + httpc.systemOST +") yacy.net";
private File htRootPath = null;
private static boolean doAccessLogging = false;
/**
@ -522,8 +519,7 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
if ((cacheFile.isFile()) && (cachedResponseHeader != null)) {
// delete the cache
sizeBeforeDelete = cacheFile.length();
cacheFile.delete();
//cacheManager.deleteFile(url); // Hermes
cacheManager.deleteFile(url);
conProp.setProperty(httpd.CONNECTION_PROP_PROXY_RESPOND_CODE,"TCP_REFRESH_MISS");
}
@ -1319,4 +1315,4 @@ public final class httpdProxyHandler extends httpdAbstractHandler implements htt
http://xlists.aza.org/moderator/cookietest/cookietest1.php
http://vancouver-webpages.com/proxy/cache-test.html
*/
*/

7
source/de/anomic/plasma/plasmaCrawlWorker.java
Unescape View File

@ -353,9 +353,9 @@ public final class plasmaCrawlWorker extends Thread {
remote.close();
log.logInfo("REJECTED WRONG MIME/EXT TYPE " + res.responseHeader.mime() + " for URL " + url.toString());
} else {
//if (cacheFile.isFile()) { // Hermes
// cacheManager.deleteFile(url);
//}
if (cacheFile.isFile()) {
cacheManager.deleteFile(url);
}
// we write the new cache entry to file system directly
cacheFile.getParentFile().mkdirs();
FileOutputStream fos = null;
@ -521,4 +521,3 @@ public final class plasmaCrawlWorker extends Thread {
}
}

75
source/de/anomic/plasma/plasmaHTCache.java
Unescape View File

@ -157,6 +157,21 @@ public final class plasmaHTCache {
responseHeaderDB.set(urlHash, responseHeader);
}
private boolean deleteFile(File file) {
if (file.exists()) {
long size = file.length();
if (file.delete()) {
currCacheSize -= size;
return true;
}
}
return false;
}
public boolean deleteFile(URL url) {
return deleteURLfromCache(url, "FROM");
}
public boolean writeFile(URL url, byte[] array) {
if (array == null) return false;
File file = getCachePath(url);
@ -188,18 +203,6 @@ public final class plasmaHTCache {
}
}
/*
private boolean deleteFile(File file) {
if (file.exists()) {
long size = file.length();
if (file.delete()) {
currCacheSize -= size;
return true;
}
}
return false;
}
private boolean deleteFileandDirs (File f, String msg) {
if (deleteFile (f)) {
log.logInfo("DELETED " + msg + " CACHE : " + f.toString());
@ -215,7 +218,7 @@ public final class plasmaHTCache {
return false;
}
}
private boolean deleteURLfromCache (URL url, String msg) {
if (deleteFileandDirs(getCachePath(url), msg)) {
try {
@ -230,9 +233,10 @@ public final class plasmaHTCache {
return false;
}
}
private void cleanupDoIt(long newCacheSize) {
File f;
while ((currCacheSize >= newCacheSize) && (cacheAge.size() > 0)) {
while (currCacheSize >= newCacheSize && cacheAge.size() > 0) {
f = (File) cacheAge.remove(cacheAge.firstKey());
if (f != null) {
log.logFinest("Trying to delete old file: " + f.toString());
@ -250,52 +254,11 @@ public final class plasmaHTCache {
}
}
}
*/
private boolean deleteFile(File file) {
long size = file.length();
if (file.exists()) {
currCacheSize -= size;
return file.delete();
} else {
return false;
}
}
public boolean deleteFile(URL url) {
return deleteFile(getCachePath(url));
}
private void cleanupCache(long newCacheSize) {
File object;
long size;
while (currCacheSize > maxCacheSize && cacheAge.size() > 0) {
object = (File) cacheAge.remove(cacheAge.firstKey());
if (object != null) {
size = object.length();
if (object.isFile() && object.delete()) {
currCacheSize -= size;
log.logInfo("DELETED OLD CACHE: " + object.toString());
object = object.getParentFile();
if (object.isDirectory() && object.list().length == 0) {
if (object.delete()) {
try {
log.logInfo("DELETED EMPTY DIRECTORY: " + object.toString());
responseHeaderDB.remove(plasmaURL.urlHash(getURL(cachePath , object)));
} catch (IOException e) {
log.logWarning("HTCACHE: IOExeption removing response header from DB: " + e.getMessage());
}
}
}
}
}
}
}
private void cleanup() {
// clean up cache to have 4% (enough) space for next entries
if ((currCacheSize >= maxCacheSize) && (cacheAge.size() > 0)) {
if (maxCacheSize > 0) cleanupCache(maxCacheSize - ((maxCacheSize / 100) * 4));
if (maxCacheSize > 0) cleanupDoIt(maxCacheSize - ((maxCacheSize / 100) * 4));
}
}

Write Preview
Loading…
Cancel
Save