SpatialDataDownloader.java

package org.wikibrain.spatial.loader;

import com.typesafe.config.Config;
import net.lingala.zip4j.core.ZipFile;
import net.lingala.zip4j.exception.ZipException;
import org.apache.commons.io.FileUtils;
import org.wikibrain.conf.Configuration;
import org.wikibrain.core.cmd.Env;
import org.wikibrain.download.FileDownloader;
import org.wikibrain.spatial.WikiBrainShapeFile;

import java.io.File;
import java.io.IOException;
import java.net.URL;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * @author Shilad Sen
 */
public class SpatialDataDownloader {
    private static final Logger LOG = LoggerFactory.getLogger(SpatialDataDownloader.class);

    private final SpatialDataFolder dir;
    private final Configuration config;

    public SpatialDataDownloader(SpatialDataFolder dir) {
        this.dir = dir;
        this.config = null;
    }

    public SpatialDataDownloader(Configuration config) {
        this.config = config;
        this.dir = new SpatialDataFolder(new File(config.get().getString("spatial.dir")));
    }

    public SpatialDataDownloader(Env env) {
        this.config = env.getConfiguration();
        this.dir = new SpatialDataFolder(new File(config.get().getString("spatial.dir")));
    }

    public WikiBrainShapeFile download(URL url, String zipShp, String refSystem, String layerGroup, String name, String encoding) throws InterruptedException, IOException {
        // Download the file if necessary
        String tokens[] = url.toString().split("/");
        String filename = refSystem + "_" + layerGroup + "_" + name + "_" + tokens[tokens.length - 1];
        File zipDest = FileUtils.getFile(dir.getRawFolder(), filename);
        if (!zipDest.isFile()) {
            FileDownloader downloader = new FileDownloader();
            downloader.download(url, zipDest);
        }

        // Unzip the file
        File tmpDir;
        try {
            ZipFile zipFile = new ZipFile(zipDest.getCanonicalPath());
            tmpDir = File.createTempFile("wikibrain", ".exploded");
            FileUtils.deleteQuietly(tmpDir);
            FileUtils.forceMkdir(tmpDir);
            LOG.info("Extracting to " + tmpDir);
            zipFile.extractAll(tmpDir.getAbsolutePath());
            FileUtils.forceDeleteOnExit(tmpDir);
        } catch (ZipException e) {
            throw new IOException(e);
        }

        // Move the requested shape file into place
        WikiBrainShapeFile src = new WikiBrainShapeFile(new File(tmpDir, zipShp), encoding);
        for (File file : src.getComponentFiles()) {
            if (!file.isFile()) {
                throw new IllegalArgumentException("expected file " + file.getAbsolutePath() + " not found");
            }
        }
        WikiBrainShapeFile dest = dir.getShapeFile(refSystem, layerGroup, name, encoding);
        src.move(dest.getFile());
        if (!dest.hasComponentFiles()) {
            throw new IllegalArgumentException();
        }

        return dest;
    }

    public WikiBrainShapeFile download(String refSysName, String layerGroup, String datasetName, boolean mappingRequired) throws InterruptedException, IOException {
        if (config == null) {
            throw new IllegalArgumentException("To download by name, SpatialDataDownloader must have a configuration");
        }
        Config c = config.getConfig("spatial.datasets", refSysName, layerGroup, datasetName);
        WikiBrainShapeFile shapeFile = download(
                new URL(c.getString("url")),
                c.getString("shp"),
                refSysName,
                layerGroup,
                datasetName,
                c.getString("encoding")
        );
        if (mappingRequired) {
            downloadMapping(new URL(c.getString("mappingUrl")), shapeFile);
        } else {
            try {
                downloadMapping(new URL(c.getString("mappingUrl")), shapeFile);
            } catch (Exception e) {
            }
        }
        return shapeFile;
    }

    public WikiBrainShapeFile download(String refSysName, String layerGroup, String datasetName) throws InterruptedException, IOException {
        return download(refSysName, layerGroup, datasetName, true);
    }

    public void downloadMapping(URL mappingUrl, WikiBrainShapeFile shapeFile) throws IOException, InterruptedException {
        File dest = shapeFile.getMappingFile();

        File tmp = File.createTempFile("wikibrain-mapping", "zip");
        FileUtils.deleteQuietly(tmp);

        if (!tmp.isFile()) {
            FileDownloader downloader = new FileDownloader();
            downloader.download(mappingUrl, tmp);
        }
        FileUtils.forceDeleteOnExit(tmp);

        // Unzip the file
        File tmpDir;
        try {
            ZipFile zipFile = new ZipFile(tmp.getCanonicalPath());
            tmpDir = File.createTempFile("wikibrain", ".exploded");
            FileUtils.deleteQuietly(tmpDir);
            FileUtils.forceMkdir(tmpDir);
            LOG.info("Extracting " + mappingUrl + " to " + tmpDir);
            zipFile.extractAll(tmpDir.getAbsolutePath());
            FileUtils.forceDeleteOnExit(tmpDir);
        } catch (ZipException e) {
            throw new IOException(e);
        }

        File src = FileUtils.getFile(tmpDir, dest.getName());
        if (!src.isFile()) {
            throw new IOException("Missing file " + dest.getName() + " in " + mappingUrl);
        }
        FileUtils.deleteQuietly(dest);
        FileUtils.moveFile(src, dest);
    }
}