package eu.europeana.cloud.migrator.provider;
import eu.europeana.cloud.common.model.DataProviderProperties;
import eu.europeana.cloud.migrator.ResourceMigrator;
import org.apache.log4j.Logger;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.file.DirectoryStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.*;
public abstract class DefaultResourceProvider
implements ResourceProvider {
// Keys for data provider properties
public static final String ORGANISATION_NAME_KEY = "organisationName";
public static final String OFFICIAL_ADDRESS_KEY = "officialAddress";
public static final String ORGANISATION_WEBSITE_KEY = "organisationWebsite";
public static final String ORGANISATION_WEBSITE_URL_KEY = "organisationWebsiteURL";
public static final String DIGITAL_LIBRARY_WEBSITE_KEY = "digitalLibraryWebsite";
public static final String DIGITAL_LIBRARY_WEBSITE_URL_KEY = "digitalLibraryURL";
public static final String CONTACT_PERSON_KEY = "contactPerson";
public static final String REMARKS_KEY = "remarks";
// Default data provider properties
public static final String DEFAULT_ORGANISATION_NAME = "Example Organisation";
public static final String DEFAULT_OFFICIAL_ADDRESS = "Example Address";
public static final String DEFAULT_ORGANISATION_WEBSITE = "Example Website";
public static final String DEFAULT_ORGANISATION_WEBSITE_URL = "http://www.example.com";
public static final String DEFAULT_DIGITAL_LIBRARY_WEBSITE = "Example DL Website";
public static final String DEFAULT_DIGITAL_LIBRARY_WEBSITE_URL = "http://www.example.com/digital";
public static final String DEFAULT_CONTACT_PERSON = "John Example";
public static final String DEFAULT_REMARKS = "Example remarks";
public static final String PROPERTIES_EXTENSION = ".properties";
public static final int DEFAULT_LIST_SIZE = 500;
// Logger
private static final Logger logger = Logger.getLogger(DefaultResourceProvider.class);
// Representation name from configuration
protected String representationName;
// Mapping file storing the association between local identifiers and files/directories in the filesystem
protected String mappingFile;
// list of location retrieved from configuration file
protected List<URI> locations;
// local indicator = when false the resource location is remote
protected boolean local;
// map associating resource provider with its location
private Map<String, URI> providersLocation;
// data provider identifier, may be null, if null the identifier is derived from the file path, otherwise it is used for all files
protected String dataProviderId;
protected DefaultResourceProvider(String representationName, String mappingFile, String locations, String dataProviderId) {
if (representationName == null)
throw new IllegalArgumentException("Representation name cannot be null!");
this.representationName = representationName;
this.mappingFile = mappingFile;
this.locations = new ArrayList<URI>();
this.providersLocation = new HashMap<String, URI>();
this.local = detectLocations(locations);
this.dataProviderId = dataProviderId != null ? (dataProviderId.isEmpty() ? null : dataProviderId) : null;
}
private boolean detectLocations(String locations) {
boolean allLocal = true;
boolean firstChange = true;
String[] locs = locations.split(";");
for (String loc : locs) {
URI location;
try {
location = new URI(loc);
this.locations.add(location);
} catch (URISyntaxException e) {
logger.error("URI " + loc + " is not valid.", e);
continue;
}
String scheme = location.getScheme();
boolean locationLocal = (scheme == null || scheme.isEmpty() || scheme.toLowerCase().equals("file"));
if (locationLocal != allLocal) {
if (firstChange) {
firstChange = false;
allLocal = locationLocal;
} else {
logger.error("All locations must be either local or remote");
throw new IllegalArgumentException("All locations must be either local or remote. locations: " + locations);
}
}
}
return allLocal;
}
protected DataProviderProperties getDataProviderPropertiesFromFile(File dpFile) {
Properties props = loadPropertiesFile(dpFile);
return new DataProviderProperties(props.getProperty(ORGANISATION_NAME_KEY, DEFAULT_ORGANISATION_NAME),
props.getProperty(OFFICIAL_ADDRESS_KEY, DEFAULT_OFFICIAL_ADDRESS),
props.getProperty(ORGANISATION_WEBSITE_KEY, DEFAULT_ORGANISATION_WEBSITE),
props.getProperty(ORGANISATION_WEBSITE_URL_KEY, DEFAULT_ORGANISATION_WEBSITE_URL),
props.getProperty(DIGITAL_LIBRARY_WEBSITE_KEY, DEFAULT_DIGITAL_LIBRARY_WEBSITE),
props.getProperty(DIGITAL_LIBRARY_WEBSITE_URL_KEY, DEFAULT_DIGITAL_LIBRARY_WEBSITE_URL),
props.getProperty(CONTACT_PERSON_KEY, DEFAULT_CONTACT_PERSON),
props.getProperty(REMARKS_KEY, DEFAULT_REMARKS));
}
private Properties loadPropertiesFile(File dpFile) {
Properties props = new Properties();
InputStream is = null;
try {
is = new FileInputStream(dpFile);
props.load(is);
} catch (IOException e) {
logger.error("Problem with file " + dpFile.getAbsolutePath(), e);
} finally {
try {
is.close();
} catch (IOException e) {
logger.error("Could not close input stream.", e);
}
}
return props;
}
protected DataProviderProperties getDefaultDataProviderProperties() {
return new DataProviderProperties(DEFAULT_ORGANISATION_NAME,
DEFAULT_OFFICIAL_ADDRESS,
DEFAULT_ORGANISATION_WEBSITE,
DEFAULT_ORGANISATION_WEBSITE_URL,
DEFAULT_DIGITAL_LIBRARY_WEBSITE,
DEFAULT_DIGITAL_LIBRARY_WEBSITE_URL,
DEFAULT_CONTACT_PERSON,
DEFAULT_REMARKS);
}
@Override
public String getRepresentationName() {
return representationName;
}
@Override
public List<URI> getLocations() {
return locations;
}
@Override
public boolean isLocal() {
return local;
}
@Override
public Map<String, List<FilePaths>> scan() {
Map<String, List<FilePaths>> paths = new HashMap<String, List<FilePaths>>();
if (!local) {
logger.warn("Location is not local. Scanning is not possible.");
return paths;
}
for (URI location : locations) {
collectPaths(Paths.get(location), paths, location);
sortPaths(paths);
}
return paths;
}
private void sortPaths(Map<String, List<FilePaths>> paths) {
for (List<FilePaths> list : paths.values()) {
for (FilePaths fp : list)
fp.sort();
}
}
private void collectPaths(Path rootPath, Map<String, List<FilePaths>> paths, URI location) {
if (rootPath == null)
return;
DirectoryStream<Path> dirStream = null;
try {
dirStream = Files.newDirectoryStream(rootPath);
for (Iterator<Path> i = dirStream.iterator(); i.hasNext(); ) {
Path path = i.next();
if (Files.isDirectory(path))
collectPaths(path, paths, location);
else {
String absolute = path.toAbsolutePath().toString().replace(ResourceMigrator.WINDOWS_SEPARATOR, ResourceMigrator.LINUX_SEPARATOR);
String providerId = getResourceProviderId(absolute);
if (absolute.endsWith(providerId + PROPERTIES_EXTENSION))
continue;
FilePaths providerPaths = getProviderPaths(Paths.get(location).toAbsolutePath().toString().replace(ResourceMigrator.WINDOWS_SEPARATOR, ResourceMigrator.LINUX_SEPARATOR), providerId, paths);
//providerPaths.getFullPaths().add(absolute);
providerPaths.addPath(absolute);
if (providersLocation.get(providerId) == null)
providersLocation.put(providerId, location);
}
}
} catch (IOException e) {
logger.error("There was a problem with opening direcory " + rootPath.toString(), e);
} finally {
if (dirStream != null)
try {
dirStream.close();
} catch (IOException e) {
logger.error("There was a problem with closing direcory stream for " + rootPath.toString(), e);
}
}
}
protected FilePaths getProviderPaths(String location, String providerId, Map<String, List<FilePaths>> paths) {
if (paths.get(providerId) == null)
paths.put(providerId, new ArrayList<FilePaths>());
for (FilePaths p : paths.get(providerId)) {
if (p.getLocation().equals(location))
return p;
}
FilePaths fp = new FilePaths(location, providerId);
if (usePathsFile())
fp.useFile(location.replace(ResourceMigrator.LINUX_SEPARATOR, "_").replace(":", "_") + "_" + providerId);
paths.get(providerId).add(fp);
return fp;
}
public URI getProvidersLocation(String providerId) {
return providersLocation.get(providerId);
}
/**
* Default implementation of determining filename from path.
* The result is either path without location, part of the specified path after "/" or "\" character or path itself.
*
* @param location location where path is placed, usually path starts with location
* @param path path to file either local or remote in URI syntax
* @return filename to be stored in ECloud
*/
@Override
public String getFilename(String location, String path) {
if (logger.isDebugEnabled()) {
logger.debug("Get filename for location: " + location + ", path: " + path);
}
int pos = -1;
if (path.startsWith(location)) {
path = path.substring(location.length());
if (logger.isDebugEnabled()) {
logger.debug("Path without location: " + path);
}
pos = path.indexOf(ResourceMigrator.LINUX_SEPARATOR);
if (pos == -1)
pos = path.indexOf(ResourceMigrator.WINDOWS_SEPARATOR);
} else {
pos = path.lastIndexOf(ResourceMigrator.LINUX_SEPARATOR);
if (pos == -1)
pos = path.lastIndexOf(ResourceMigrator.WINDOWS_SEPARATOR);
}
if (logger.isDebugEnabled()) {
logger.debug("Returning: " + path.substring(pos + 1));
}
// when pos == -1 whole path is returned, otherwise only part after pos
return path.substring(pos + 1);
}
@Override
public int getFileCount(String localId) {
// impossible to determine in anstract class
return -1;
}
/**
* Default resource provider splits the paths list to several lists of more or less equal size around 500.
*
* @param paths
* @return
*/
@Override
public List<FilePaths> split(List<FilePaths> paths) {
List<FilePaths> result = new ArrayList<FilePaths>();
for (FilePaths fp : paths) {
result.addAll(split(fp));
}
return result;
}
private List<FilePaths> split(FilePaths fp) {
// size of the list
int size = fp.size();
// number of full DEFAULT_LIST_SIZE parts
int count = size / DEFAULT_LIST_SIZE;
// size of the last part
int rest = size % DEFAULT_LIST_SIZE;
List<FilePaths> result = new ArrayList<FilePaths>();
// when no need to split return the same file paths object
if (size <= DEFAULT_LIST_SIZE)
result.add(fp);
else {
for (int i = 0; i < count; i++) {
FilePaths filePath = new FilePaths(fp.getLocation(), fp.getDataProvider());
filePath.getFullPaths().addAll(fp.getFullPaths().subList(i * DEFAULT_LIST_SIZE, (i + 1) * DEFAULT_LIST_SIZE < size ? (i + 1) * DEFAULT_LIST_SIZE : size));
filePath.setIdentifier(String.valueOf(i * DEFAULT_LIST_SIZE) + "_" + String.valueOf((i + 1) * DEFAULT_LIST_SIZE < size ? (i + 1) * DEFAULT_LIST_SIZE : size));
result.add(filePath);
}
if (rest > 0) {
FilePaths filePath = new FilePaths(fp.getLocation(), fp.getDataProvider());
filePath.getFullPaths().addAll(fp.getFullPaths().subList(count * DEFAULT_LIST_SIZE, size));
filePath.setIdentifier(String.valueOf(count * DEFAULT_LIST_SIZE) + "_" + String.valueOf(size));
result.add(filePath);
}
}
return result;
}
@Override
public boolean usePathsFile() {
return false;
}
@Override
public Map<String, String> getReversedMapping() {
return new HashMap<String, String>();
}
}