/* Copyright (c) 2011 Danish Maritime Authority
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this library. If not, see <http://www.gnu.org/licenses/>.
*/
package dk.dma.ais.downloader;
import dk.dma.ais.packet.AisPacketFilters;
import org.apache.tomcat.util.http.fileupload.IOUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Controller;
import org.springframework.util.StringUtils;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.ResponseBody;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import javax.servlet.http.HttpServletResponse;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.PrintStream;
import java.net.URL;
import java.net.URLConnection;
import java.nio.channels.Channels;
import java.nio.channels.ReadableByteChannel;
import java.nio.file.DirectoryStream;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.logging.Level;
import java.util.logging.Logger;
/**
* A Query service.<br>
* Used for issuing am AIS store query. The query result will be downloaded
* and stored in a client specific folder.
* <p>
* Also handles streaming, listing and deleting files from the client specific
* download folder.
*/
@Controller
@RequestMapping("/downloader/query")
@SuppressWarnings("unused")
public class QueryService {
/**
* Defined a worker pool size of 2 to constrain load
*/
private static final int EXECUTOR_POOL_SIZE = 10;
private static final long FILE_EXPIRY_MS = 1000L * 60L * 60L * 4L; // 4 hours
private final static Logger log = Logger.getLogger(QueryService.class.getName());
private final static String DOWNLOAD_SUFFIX = ".download";
private Path repoRoot;
private ExecutorService processPool;
@Value("${ais.view.url:https://ais2.e-navigation.net/aisview/rest/store/query?}")
String aisViewUrl;
@Value("${repo.root:}")
String repoRootPath;
@Value("${auth.header:}")
String authHeader;
/**
* Initializes the repository
*/
@PostConstruct
public void init() throws Exception {
log.info("******** Using AIS View URL: " + aisViewUrl);
// Create the repo root directory
if (StringUtils.isEmpty(repoRootPath)) {
repoRoot = Paths.get(System.getProperty("user.home")).resolve(".aisdownloader");
} else {
repoRoot = Paths.get(repoRootPath);
}
log.info("******** Using repo root " + repoRoot);
if (!Files.exists(getRepoRoot())) {
try {
Files.createDirectories(getRepoRoot());
} catch (IOException e) {
log.log(Level.SEVERE, "Error creating repository dir " + getRepoRoot(), e);
}
}
if (!StringUtils.isEmpty(authHeader)) {
log.info("******** Using auth header: " + authHeader);
}
// Initialize process pool
processPool = Executors.newFixedThreadPool(EXECUTOR_POOL_SIZE);
log.info("Initialized the QueryService");
}
@PreDestroy
public void cleanUp() throws Exception {
if (processPool != null && !processPool.isShutdown()) {
processPool.shutdown();
processPool = null;
}
log.info("Destroyed the QueryService");
}
/**
* Returns the repository root
* @return the repository root
*/
public Path getRepoRoot() {
return repoRoot;
}
/**
* Creates a URI from the repo file
* @param repoFile the repo file
* @return the URI for the file
*/
public String getRepoUri(Path repoFile) {
Path filePath = getRepoRoot().relativize(repoFile);
return "/rest/repo/file/" + filePath;
}
/**
* Creates a path from the repo file relative to the repo root
* @param repoFile the repo file
* @return the path for the file
*/
public String getRepoPath(Path repoFile) {
Path filePath = getRepoRoot().relativize(repoFile);
return filePath.toString().replace('\\', '/');
}
/**
* Asynchronously loads the given file
* @param url the URL to load
* @param path the path to save the file to
*/
private Future<Path> asyncLoadFile(final String url, final Path path) {
Callable<Path> job = () -> {
long t0 = System.currentTimeMillis();
// For the resulting file, drop the ".download" suffix
String name = path.getFileName().toString();
name = name.substring(0, name.length() - DOWNLOAD_SUFFIX.length());
try {
// Set up a few timeouts and fetch the attachment
URLConnection con = new URL(url).openConnection();
con.setConnectTimeout(60 * 1000); // 1 minute
con.setReadTimeout(60 * 60 * 1000); // 1 hour
if (!StringUtils.isEmpty(authHeader)) {
con.setRequestProperty ("Authorization", authHeader);
}
try (ReadableByteChannel rbc = Channels.newChannel(con.getInputStream());
FileOutputStream fos = new FileOutputStream(path.toFile())) {
fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE);
}
log.info(String.format("Copied %s -> %s in %d ms",
url,
path,
System.currentTimeMillis() - t0));
} catch (Exception e) {
log.log(Level.SEVERE, "Failed downloading " + url + ": " + e.getMessage());
// Delete the old file
if (Files.exists(path)) {
try {
Files.delete(path);
} catch (IOException e1) {
log.finer("Failed deleting old file " + path);
}
}
// Save an error file
Path errorFile = path.getParent().resolve(name + ".err.txt");
try (PrintStream err = new PrintStream(new FileOutputStream(errorFile.toFile()))) {
e.printStackTrace(err);
} catch (IOException ex) {
log.finer("Failed generating error file " + errorFile);
}
return errorFile;
}
Path resultPath = path.getParent().resolve(name);
try {
Files.move(path, resultPath);
} catch (IOException e) {
log.log(Level.SEVERE, "Failed renaming path " + path + ": " + e.getMessage());
}
return resultPath;
};
log.info("Submitting new job: " + url);
return processPool.submit(job);
}
/**
* Execute the given query
* @param clientId the client id
* @param async whether to execute synchronously or asynchronously
* @param params the query parameters
* @return the result file
*/
@RequestMapping( value = "/execute/{clientId}", method= RequestMethod.GET)
@ResponseBody
public RepoFile executeQuery(@PathVariable("clientId") String clientId,
@RequestParam(value = "async", defaultValue = "true") boolean async,
@RequestParam("params") String params) throws IOException {
String url = aisViewUrl + params;
// Create the client ID folder
Path dir = repoRoot.resolve(clientId);
if (!Files.exists(dir)) {
Files.createDirectories(dir);
}
// Create a new file to hold the result
// (could have used Files.createTempFile, but this should be enough to create a unique file)
Date now = new Date();
Path file = Files.createTempFile(
dir,
new SimpleDateFormat("MM-dd HHmmss ").format(now),
fileType(url) + DOWNLOAD_SUFFIX);
String fileName = file.getFileName().toString();
// Load the file
Future<Path> result = asyncLoadFile(url, file);
if (!async) {
try {
Path path = result.get();
// The resulting path may actually by an error file
fileName = path.getFileName().toString();
} catch (Exception e) {
log.severe("Error executing query: " + params + ", error: " + e);
}
}
// Return a RepoFile for the newly created file
RepoFile vo = new RepoFile();
vo.setName(fileName);
vo.setPath(clientId + "/" + fileName);
vo.setUpdated(now);
vo.setSize(0L);
return vo;
}
/**
* Returns the file type of the given URL
* @param url the url
* @return the file type
*/
private String fileType(String url) {
if (url.contains("OUTPUT_TO_KML")) {
return ".kml";
} else if (url.contains("OUTPUT_TO_HTML")) {
return ".html";
} else if (url.contains("table")) {
return ".csv";
} else if (url.contains("json")) {
return ".json";
}
return ".txt";
}
/**
* Streams the file specified by the path
*/
@RequestMapping( value = "/file/{clientId}/{file:.*}", method= RequestMethod.GET)
public void streamFile(@PathVariable("clientId") String clientId,
@PathVariable("file") String file,
HttpServletResponse response) throws IOException {
Path path = repoRoot
.resolve(clientId)
.resolve(file);
if (Files.notExists(path) || Files.isDirectory(path)) {
log.log(Level.WARNING, "Failed streaming file: " + path);
response.setStatus(404);
return;
}
response.setContentType(Files.probeContentType(path));
try (InputStream in = Files.newInputStream(path)) {
IOUtils.copy(in, response.getOutputStream());
response.flushBuffer();
}
}
/**
* Deletes the file specified by the path
*/
@RequestMapping( value = "/delete/{clientId}/{file:.*}", method= RequestMethod.GET)
@ResponseBody
public String deleteFile(@PathVariable("clientId") String clientId,
@PathVariable("file") String file,
HttpServletResponse response) throws IOException {
Path path = repoRoot
.resolve(clientId)
.resolve(file);
if (Files.notExists(path) || Files.isDirectory(path)) {
log.log(Level.WARNING, "Failed deleting file: " + path);
response.setStatus(404);
return "404";
}
Files.delete(path);
log.info("Deleted " + path);
return "Deleted " + path;
}
/**
* Deletes all the file of the client folder
*/
@RequestMapping( value = "/delete-all/{clientId}", method= RequestMethod.GET)
@ResponseBody
public String deleteFiles(@PathVariable("clientId") String clientId,
HttpServletResponse response) throws IOException {
int deletedFiles = 0;
Path path = repoRoot
.resolve(clientId);
if (Files.notExists(path) || !Files.isDirectory(path)) {
log.log(Level.WARNING, "Failed deleting files in " + path);
response.setStatus(404);
return "Failed deleting files in " + clientId;
}
try {
Files.walkFileTree(path, new SimpleFileVisitor<Path>() {
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
log.info("Deleting repo file :" + file);
Files.delete(file);
return FileVisitResult.CONTINUE;
}
});
} catch (IOException e) {
log.log(Level.SEVERE, "Failed cleaning up dir: " + path);
return "Failed deleting files in " + clientId;
}
return "Deleted files in dir " + clientId;
}
/**
* Returns a list of files in the folder specified by the clientId
* @return the list of files in the folder specified by the path
*/
@RequestMapping( value = "/list/{clientId:.*}", method= RequestMethod.GET, produces = "application/json;charset=UTF-8")
@ResponseBody
public List<RepoFile> listFiles(@PathVariable("clientId") String clientId) throws IOException {
List<RepoFile> result = new ArrayList<>();
Path folder = repoRoot.resolve(clientId);
if (Files.exists(folder) && Files.isDirectory(folder)) {
// Filter out directories and hidden files
DirectoryStream.Filter<Path> filter = file ->
Files.isRegularFile(file) &&
!file.getFileName().toString().startsWith(".");
try (DirectoryStream<Path> stream = Files.newDirectoryStream(folder, filter)) {
stream.forEach(f -> {
RepoFile vo = new RepoFile();
vo.setName(f.getFileName().toString());
vo.setPath(clientId + "/" + f.getFileName().toString());
try {
vo.setUpdated(new Date(Files.getLastModifiedTime(f).toMillis()));
vo.setSize(Files.size(f));
} catch (Exception e) {
log.finer("Error reading file attribute for " + f);
}
vo.setComplete(!f.getFileName().toString().endsWith(DOWNLOAD_SUFFIX));
result.add(vo);
});
}
}
Collections.sort(result);
return result;
}
/**
* Validates the AIS filter passed along. The filter must adhere to the
* grammar defined by the AisLib:
* https://github.com/dma-ais/AisLib
* @param filter the filter to validate
* @return the the filter is valid or not
*/
@RequestMapping( value = "/validate-filter", method= RequestMethod.GET)
@ResponseBody
public boolean validateFilter(@RequestParam("filter") String filter) {
// A blank filter is valid
if (StringUtils.isEmpty(filter)) {
return true;
}
// Check if the filter can be parsed
try {
AisPacketFilters.parseExpressionFilter(filter);
log.fine("Successfully parsed filter: " + filter);
return true;
} catch (Exception e) {
log.fine("Failed parsing filter: " + filter + ": " + e);
return false;
}
}
/***************************************/
/** Repo clean-up methods **/
/***************************************/
/**
* called every hour to clean up the repo
*/
@Scheduled(cron="12 27 */1 * * *")
public void cleanUpRepoFolder() {
long now = System.currentTimeMillis();
long expiredTime = now - FILE_EXPIRY_MS;
try {
Files.walkFileTree(getRepoRoot(), new SimpleFileVisitor<Path>() {
@Override
public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
if (!dir.equals(getRepoRoot()) && isDirEmpty(dir)) {
log.info("Deleting repo directory :" + dir);
Files.delete(dir);
}
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
if (Files.getLastModifiedTime(file).toMillis() < expiredTime) {
log.info("Deleting repo file :" + file);
Files.delete(file);
}
return FileVisitResult.CONTINUE;
}
});
} catch (IOException e) {
log.log(Level.SEVERE, "Failed cleaning up repo: " + e.getMessage());
}
log.info(String.format("Cleaned up repo in %d ms", System.currentTimeMillis() - now));
}
/**
* Returns if the directory is empty or not
* @param directory the directory to check
* @return if the directory is empty or not
*/
private static boolean isDirEmpty(final Path directory) throws IOException {
try(DirectoryStream<Path> dirStream = Files.newDirectoryStream(directory)) {
return !dirStream.iterator().hasNext();
} catch (Exception e) {
// Should never happen
return false;
}
}
}