//plasmaCrawlRobotsTxt.java
//-------------------------------------
//part of YACY
//(C) by Michael Peter Christen; mc@yacy.net
//first published on http://www.anomic.de
//Frankfurt, Germany, 2004
//
//This file is contributed by Martin Thelian
// [MC] moved some methods from robotsParser file that had been created by Alexander Schier to this class
//last major change: $LastChangedDate$ by $LastChangedBy$
//Revision: $LastChangedRevision$
//
//This program is free software; you can redistribute it and/or modify
//it under the terms of the GNU General public License as published by
//the Free Software Foundation; either version 2 of the License, or
//(at your option) any later version.
//
//This program is distributed in the hope that it will be useful,
//but WITHOUT ANY WARRANTY; without even the implied warranty of
//MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
//GNU General public License for more details.
//
//You should have received a copy of the GNU General public License
//along with this program; if not, write to the Free Software
//Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package net.yacy.crawler.robots;
import java.io.IOException;
import java.net.MalformedURLException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.regex.Pattern;
import net.yacy.cora.document.id.DigestURL;
import net.yacy.cora.document.id.MultiProtocolURL;
import net.yacy.cora.federate.yacy.CacheStrategy;
import net.yacy.cora.protocol.ClientIdentification;
import net.yacy.cora.protocol.HeaderFramework;
import net.yacy.cora.util.ConcurrentLog;
import net.yacy.cora.util.SpaceExceededException;
import net.yacy.crawler.retrieval.Request;
import net.yacy.crawler.retrieval.Response;
import net.yacy.data.WorkTables;
import net.yacy.kelondro.blob.BEncodedHeap;
import net.yacy.kelondro.util.NamePrefixThreadFactory;
import net.yacy.repository.Blacklist.BlacklistType;
import net.yacy.repository.LoaderDispatcher;
public class RobotsTxt {
private final static ConcurrentLog log = new ConcurrentLog(RobotsTxt.class.getName());
protected static final String ROBOTS_TXT_PATH = "/robots.txt";
protected static final String ROBOTS_DB_PATH_SEPARATOR = ";";
protected static final Pattern ROBOTS_DB_PATH_SEPARATOR_MATCHER = Pattern.compile(ROBOTS_DB_PATH_SEPARATOR);
private final ConcurrentMap<String, DomSync> syncObjects;
//private static final HashSet<String> loadedRobots = new HashSet<String>(); // only for debugging
private final WorkTables tables;
private final LoaderDispatcher loader;
/** Thread pool used to launch concurrent tasks */
private ThreadPoolExecutor threadPool;
private static class DomSync {
private DomSync() {}
}
/**
*
* @param worktables
* @param loader
* @param maxConcurrentTheads maximum active threads this instance is allowed to run for its concurrent tasks
*/
public RobotsTxt(final WorkTables worktables, LoaderDispatcher loader, final int maxActiveTheads) {
this.threadPool = new ThreadPoolExecutor(maxActiveTheads, maxActiveTheads,
0L, TimeUnit.MILLISECONDS,
new LinkedBlockingQueue<Runnable>(),
new NamePrefixThreadFactory(RobotsTxt.class.getSimpleName()));
this.syncObjects = new ConcurrentHashMap<String, DomSync>();
this.tables = worktables;
this.loader = loader;
try {
this.tables.getHeap(WorkTables.TABLE_ROBOTS_NAME);
//log.info("initiated robots table: " + this.tables.getHeap(WorkTables.TABLE_ROBOTS_NAME).getFile());
} catch (final IOException e) {
try {
this.tables.getHeap(WorkTables.TABLE_ROBOTS_NAME).clear();
} catch (final IOException e1) {
}
}
}
public void clear() throws IOException {
log.info("clearing robots table");
this.tables.getHeap(WorkTables.TABLE_ROBOTS_NAME).clear();
this.syncObjects.clear();
}
public void close() {
/* Shutdown all active robots.txt loading threads */
if(this.threadPool != null) {
this.threadPool.shutdownNow();
}
}
public int size() throws IOException {
return this.tables.getHeap(WorkTables.TABLE_ROBOTS_NAME).size();
}
public RobotsTxtEntry getEntry(final MultiProtocolURL theURL, final ClientIdentification.Agent agent) {
if (theURL == null) throw new IllegalArgumentException();
if (!theURL.getProtocol().startsWith("http")) return null;
return getEntry(getHostPort(theURL), agent, true);
}
public RobotsTxtEntry getEntry(final String urlHostPort, final ClientIdentification.Agent agent, final boolean fetchOnlineIfNotAvailableOrNotFresh) {
// this method will always return a non-null value
RobotsTxtEntry robotsTxt4Host = null;
Map<String, byte[]> record;
BEncodedHeap robotsTable = null;
try {
robotsTable = this.tables.getHeap(WorkTables.TABLE_ROBOTS_NAME);
} catch (final IOException e1) {
log.severe("tables not available", e1);
}
try {
record = robotsTable.get(robotsTable.encodedKey(urlHostPort));
} catch (final SpaceExceededException e) {
log.warn("memory exhausted", e);
record = null;
} catch (final IOException e) {
log.warn("cannot get robotstxt from table", e);
record = null;
}
if (record != null) robotsTxt4Host = new RobotsTxtEntry(urlHostPort, record);
if (fetchOnlineIfNotAvailableOrNotFresh && (
robotsTxt4Host == null ||
robotsTxt4Host.getLoadedDate() == null ||
System.currentTimeMillis() - robotsTxt4Host.getLoadedDate().getTime() > 7*24*60*60*1000
)) {
// make or get a synchronization object
DomSync syncObj = this.syncObjects.get(urlHostPort);
if (syncObj == null) {
syncObj = new DomSync();
this.syncObjects.put(urlHostPort, syncObj);
}
// we can now synchronize for each host separately
synchronized (syncObj) {
// if we have not found any data or the data is older than 7 days, we need to load it from the remote server
// check the robots table again for all threads that come here because they waited for another one
// to complete a download
try {
record = robotsTable.get(robotsTable.encodedKey(urlHostPort));
} catch (final SpaceExceededException e) {
log.warn("memory exhausted", e);
record = null;
} catch (final IOException e) {
log.warn("cannot get robotstxt from table", e);
record = null;
}
if (record != null) robotsTxt4Host = new RobotsTxtEntry(urlHostPort, record);
if (robotsTxt4Host != null &&
robotsTxt4Host.getLoadedDate() != null &&
System.currentTimeMillis() - robotsTxt4Host.getLoadedDate().getTime() <= 1*24*60*60*1000) {
return robotsTxt4Host;
}
// generating the proper url to download the robots txt
DigestURL robotsURL = robotsURL(urlHostPort);
Response response = null;
if (robotsURL != null) {
if (log.isFine()) log.fine("Trying to download the robots.txt file from URL '" + robotsURL + "'.");
Request request = new Request(robotsURL, null);
try {
response = RobotsTxt.this.loader.load(request, CacheStrategy.NOCACHE, null, agent);
} catch (final Throwable e) {
log.info("Trying to download the robots.txt file from URL '" + robotsURL.toNormalform(false) + "' failed - " + e.getMessage());
response = null;
}
}
if (response == null) {
processOldEntry(robotsTxt4Host, robotsURL, robotsTable);
} else {
processNewEntry(robotsURL, response, agent.robotIDs);
}
}
}
return robotsTxt4Host;
}
public void delete(final MultiProtocolURL theURL) {
final String urlHostPort = getHostPort(theURL);
if (urlHostPort == null) return;
final BEncodedHeap robotsTable;
try {
robotsTable = this.tables.getHeap(WorkTables.TABLE_ROBOTS_NAME);
} catch (final IOException e1) {
log.severe("tables not available", e1);
return;
}
if (robotsTable == null) return;
try {
robotsTable.delete(robotsTable.encodedKey(urlHostPort));
} catch (IOException e) {
}
}
public void ensureExist(final MultiProtocolURL theURL, final ClientIdentification.Agent agent, boolean concurrent) {
if (theURL.isLocal()) return;
final String urlHostPort = getHostPort(theURL);
if (urlHostPort == null) return;
final BEncodedHeap robotsTable;
try {
robotsTable = this.tables.getHeap(WorkTables.TABLE_ROBOTS_NAME);
} catch (final IOException e1) {
log.severe("tables not available", e1);
return;
}
if (robotsTable != null && robotsTable.containsKey(robotsTable.encodedKey(urlHostPort))) return;
Thread t = new Thread() {
@Override
public void run(){
this.setName("Robots.txt:ensureExist(" + theURL.toNormalform(true) + ")");
// make or get a synchronization object
DomSync syncObj = RobotsTxt.this.syncObjects.get(urlHostPort);
if (syncObj == null) {
syncObj = new DomSync();
RobotsTxt.this.syncObjects.put(urlHostPort, syncObj);
}
// we can now synchronize for each host separately
synchronized (syncObj) {
if (robotsTable.containsKey(robotsTable.encodedKey(urlHostPort))) return;
// generating the proper url to download the robots txt
DigestURL robotsURL = robotsURL(urlHostPort);
Response response = null;
if (robotsURL != null) {
if (log.isFine()) log.fine("Trying to download the robots.txt file from URL '" + robotsURL + "'.");
Request request = new Request(robotsURL, null);
try {
response = RobotsTxt.this.loader.load(request, CacheStrategy.NOCACHE, null, agent);
} catch (final IOException e) {
response = null;
}
}
if (response == null) {
processOldEntry(null, robotsURL, robotsTable);
} else {
processNewEntry(robotsURL, response, agent.robotIDs);
}
}
}
};
if (concurrent) {
this.threadPool.execute(t);
} else {
t.run();
}
}
/**
* @return the approximate number of threads that are actively
* executing robots.txt loading tasks
*/
public int getActiveThreads() {
return this.threadPool != null ? this.threadPool.getActiveCount() : 0;
}
private void processOldEntry(RobotsTxtEntry robotsTxt4Host, DigestURL robotsURL, BEncodedHeap robotsTable) {
// no robots.txt available, make an entry to prevent that the robots loading is done twice
if (robotsTxt4Host == null) {
// generate artificial entry
robotsTxt4Host = new RobotsTxtEntry(
robotsURL,
new ArrayList<String>(),
new ArrayList<String>(),
new Date(),
new Date(),
null,
null,
Integer.valueOf(0),
null);
} else {
robotsTxt4Host.setLoadedDate(new Date());
}
// store the data into the robots DB
final int sz = robotsTable.size();
addEntry(robotsTxt4Host);
if (robotsTable.size() <= sz) {
log.severe("new entry in robots.txt table failed, resetting database");
try {clear();} catch (final IOException e) {}
addEntry(robotsTxt4Host);
}
}
private void processNewEntry(DigestURL robotsURL, Response response, final String[] thisAgents) {
final byte[] robotsTxt = response.getContent();
//Log.logInfo("RobotsTxt", "robots of " + robotsURL.toNormalform(true, true) + ":\n" + ((robotsTxt == null) ? "null" : UTF8.String(robotsTxt))); // debug TODO remove
RobotsTxtParser parserResult;
ArrayList<String> denyPath;
if (response.getResponseHeader().getStatusCode() == 401 || response.getResponseHeader().getStatusCode() == 403) {
parserResult = new RobotsTxtParser(thisAgents);
// create virtual deny path
denyPath = new ArrayList<String>();
denyPath.add("/");
} else {
parserResult = new RobotsTxtParser(thisAgents, robotsTxt);
denyPath = parserResult.denyList();
}
// store the data into the robots DB
String etag = response.getResponseHeader().containsKey(HeaderFramework.ETAG) ? (response.getResponseHeader().get(HeaderFramework.ETAG)).trim() : null;
boolean isBrowserAgent = thisAgents.length == 1 && thisAgents[0].equals("Mozilla");
if (isBrowserAgent) denyPath.clear();
final RobotsTxtEntry robotsTxt4Host = new RobotsTxtEntry(
robotsURL,
parserResult.allowList(),
denyPath,
new Date(),
response.getResponseHeader().lastModified(),
etag,
parserResult.sitemap(),
parserResult.crawlDelayMillis(),
parserResult.agentName());
addEntry(robotsTxt4Host);
}
private String addEntry(final RobotsTxtEntry entry) {
// writes a new page and returns key
try {
final BEncodedHeap robotsTable = this.tables.getHeap(WorkTables.TABLE_ROBOTS_NAME);
robotsTable.insert(robotsTable.encodedKey(entry.getHostName()), entry.getMem());
return entry.getHostName();
} catch (final Exception e) {
log.warn("cannot write robots.txt entry", e);
return null;
}
}
public static final String getHostPort(final MultiProtocolURL theURL) {
int port = theURL.getPort();
if (port == -1) {
if (theURL.getProtocol().equalsIgnoreCase("http")) {
port = 80;
} else if (theURL.getProtocol().equalsIgnoreCase("https")) {
port = 443;
} else {
port = 80;
}
}
String host = theURL.getHost();
if (host == null) return null;
StringBuilder sb = new StringBuilder(host.length() + 6);
if (host.indexOf(':') >= 0) {sb.append('[').append(host).append(']');} else sb.append(host);
sb.append(':').append(Integer.toString(port));
return sb.toString();
}
public static boolean isRobotsURL(MultiProtocolURL url) {
return url.getPath().equals(ROBOTS_TXT_PATH);
}
/**
* generate a robots.txt url.
* @param urlHostPort a string of the form <host>':'<port> or just <host>
* @return the full robots.txt url
*/
public static DigestURL robotsURL(String urlHostPort) {
if (urlHostPort.endsWith(":80")) urlHostPort = urlHostPort.substring(0, urlHostPort.length() - 3);
DigestURL robotsURL = null;
try {
robotsURL = new DigestURL((urlHostPort.endsWith(":443") ? "https://" : "http://") + urlHostPort + ROBOTS_TXT_PATH);
} catch (final MalformedURLException e) {
log.severe("Unable to generate robots.txt URL for host:port '" + urlHostPort + "'.", e);
robotsURL = null;
}
return robotsURL;
}
public static class CheckEntry {
public final DigestURL digestURL;
public final RobotsTxtEntry robotsTxtEntry;
public final Response response;
public final String error;
public CheckEntry(DigestURL digestURL, RobotsTxtEntry robotsTxtEntry, Response response, String error) {
this.digestURL = digestURL;
this.robotsTxtEntry = robotsTxtEntry;
this.response = response;
this.error = error;
}
}
/**
* A unit task to load a robots.txt entry
*/
private class CrawlCheckTask implements Callable<CheckEntry> {
private final DigestURL url;
private final ClientIdentification.Agent userAgent;
public CrawlCheckTask(final DigestURL url, final ClientIdentification.Agent userAgent) {
this.url = url;
this.userAgent = userAgent;
}
@Override
public CheckEntry call() throws Exception {
// try to load the robots
RobotsTxtEntry robotsEntry = getEntry(this.url, this.userAgent);
boolean robotsAllowed = robotsEntry == null ? true : !robotsEntry.isDisallowed(this.url);
if (robotsAllowed) {
try {
Request request = loader.request(this.url, true, false);
Response response = loader.load(request, CacheStrategy.NOCACHE,
BlacklistType.CRAWLER, userAgent);
return new CheckEntry(this.url, robotsEntry, response, null);
} catch (final IOException e) {
return new CheckEntry(this.url, robotsEntry, null, "error response: " + e.getMessage());
}
}
return new CheckEntry(this.url, robotsEntry, null, null);
}
}
public Collection<CheckEntry> massCrawlCheck(final Collection<DigestURL> rootURLs, final ClientIdentification.Agent userAgent) {
final List<Future<CheckEntry>> futures = new ArrayList<>();
for (DigestURL u: rootURLs) {
futures.add(this.threadPool.submit(new CrawlCheckTask(u, userAgent)));
}
final Collection<CheckEntry> results = new ArrayList<>();
/* Now collect the results concurrently loaded */
for(Future<CheckEntry> future: futures) {
try {
results.add(future.get());
} catch (InterruptedException e) {
log.warn("massCrawlCheck was interrupted before retrieving all results.");
break;
} catch (ExecutionException e) {
/* A robots.txt loading failed : let's continue and try to get the next result
* (most of time this should not happen, as Exceptions are caught inside the concurrent task) */
continue;
}
}
return results;
}
}