// YMarkMetadata.java
// (C) 2011 by Stefan Förster, sof@gmx.de, Norderstedt, Germany
// first published 2010 on http://yacy.net
//
// This is a part of YaCy, a peer-to-peer based web search engine
//
// $LastChangedDate: 2011-03-09 13:50:39 +0100 (Mi, 09 Mrz 2011) $
// $LastChangedRevision: 7574 $
// $LastChangedBy: apfelmaennchen $
//
// LICENSE
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package net.yacy.data.ymark;
import java.io.IOException;
import java.net.MalformedURLException;
import java.util.EnumMap;
import net.yacy.cora.date.ISO8601Formatter;
import net.yacy.cora.document.encoding.ASCII;
import net.yacy.cora.document.id.DigestURL;
import net.yacy.cora.federate.yacy.CacheStrategy;
import net.yacy.cora.protocol.ClientIdentification;
import net.yacy.cora.util.ConcurrentLog;
import net.yacy.crawler.retrieval.Response;
import net.yacy.document.Document;
import net.yacy.document.Parser.Failure;
import net.yacy.kelondro.data.meta.URIMetadataNode;
import net.yacy.repository.LoaderDispatcher;
import net.yacy.search.index.Segment;
public class YMarkMetadata {
private DigestURL uri;
Document document;
Segment indexSegment;
public enum METADATA {
TITLE,
DESCRIPTION,
FAVICON,
KEYWORDS,
LANGUAGE,
CREATOR,
PUBLISHER,
CHARSET,
MIMETYPE,
SIZE,
WORDCOUNT,
IN_URLDB,
FRESHDATE,
LOADDATE,
MODDATE,
SNIPPET,
AUTOTAG
}
public YMarkMetadata(final DigestURL uri) {
this.uri = uri;
this.document = null;
this.indexSegment = null;
}
public YMarkMetadata(final DigestURL uri, final Segment indexSegment) {
this.uri = uri;
this.document = null;
this.indexSegment = indexSegment;
}
public YMarkMetadata(final byte[] urlHash, final Segment indexSegment) {
this.document = null;
this.indexSegment = indexSegment;
try {
this.uri = this.indexSegment.fulltext().getURL(ASCII.String(urlHash));
} catch (IOException e) {
this.uri = null;
ConcurrentLog.logException(e);
}
}
public YMarkMetadata(final Document document) {
this.document = document;
try {
this.uri = new DigestURL(this.document.dc_identifier());
} catch (final MalformedURLException e) {
this.uri = null;
}
this.indexSegment = null;
}
public Document loadDocument(final LoaderDispatcher loader, ClientIdentification.Agent agent) throws IOException, Failure {
if(this.document == null) {
Response response = null;
response = loader.load(loader.request(this.uri, true, false), CacheStrategy.IFEXIST, Integer.MAX_VALUE, null, agent);
Document[] docs = response.parse();
this.document = Document.mergeDocuments(response.url(), response.getMimeType(), docs);
}
return this.document;
}
public EnumMap<METADATA, String> getMetadata() {
final EnumMap<METADATA, String> metadata = new EnumMap<METADATA, String>(METADATA.class);
final URIMetadataNode urlEntry = this.indexSegment.fulltext().getMetadata(this.uri.hash());
if (urlEntry != null) {
metadata.put(METADATA.SIZE, String.valueOf(urlEntry.filesize()));
metadata.put(METADATA.FRESHDATE, ISO8601Formatter.FORMATTER.format(urlEntry.freshdate()));
metadata.put(METADATA.LOADDATE, ISO8601Formatter.FORMATTER.format(urlEntry.loaddate()));
metadata.put(METADATA.MODDATE, ISO8601Formatter.FORMATTER.format(urlEntry.moddate()));
metadata.put(METADATA.SNIPPET, String.valueOf(urlEntry.snippet()));
metadata.put(METADATA.WORDCOUNT, String.valueOf(urlEntry.wordCount()));
metadata.put(METADATA.MIMETYPE, String.valueOf(urlEntry.mime()));
metadata.put(METADATA.LANGUAGE, urlEntry.language());
metadata.put(METADATA.TITLE, urlEntry.dc_title());
metadata.put(METADATA.CREATOR, urlEntry.dc_creator());
metadata.put(METADATA.KEYWORDS, urlEntry.dc_subject());
metadata.put(METADATA.PUBLISHER, urlEntry.dc_publisher());
}
return metadata;
}
public EnumMap<METADATA, String> loadMetadata() {
final EnumMap<METADATA, String> metadata = new EnumMap<METADATA, String>(METADATA.class);
if(this.document != null) {
metadata.put(METADATA.TITLE, this.document.dc_title());
metadata.put(METADATA.CREATOR, this.document.dc_creator());
metadata.put(METADATA.KEYWORDS, this.document.dc_subject(' '));
metadata.put(METADATA.PUBLISHER, this.document.dc_publisher());
metadata.put(METADATA.DESCRIPTION, this.document.dc_description().length > 0 ? this.document.dc_description()[0] : "");
metadata.put(METADATA.MIMETYPE, this.document.dc_format());
metadata.put(METADATA.LANGUAGE, this.document.dc_language());
metadata.put(METADATA.CHARSET, this.document.getCharset());
// metadata.put(METADATA.SIZE, String.valueOf(document.getTextLength()));
}
return metadata;
}
}