/* * * Copyright 2016 Netflix, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. * */ package com.netflix.genie.web.services.impl; import com.google.common.collect.Lists; import com.netflix.genie.common.exceptions.GenieException; import com.netflix.genie.common.exceptions.GenieServerException; import com.netflix.genie.core.services.FileTransfer; import com.netflix.spectator.api.Registry; import com.netflix.spectator.api.Timer; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.FileUtils; import org.apache.commons.validator.routines.UrlValidator; import org.hibernate.validator.constraints.NotBlank; import org.springframework.http.HttpMethod; import org.springframework.http.MediaType; import org.springframework.http.client.ClientHttpResponse; import org.springframework.web.client.ResponseExtractor; import org.springframework.web.client.RestTemplate; import javax.validation.constraints.NotNull; import java.io.File; import java.io.IOException; import java.net.MalformedURLException; import java.net.URISyntaxException; import java.net.URL; import java.time.Instant; import java.util.concurrent.TimeUnit; /** * An implementation of the FileTransferService interface in which the remote locations are available via http[s]. * * @author tgianos * @since 3.0.0 */ @Slf4j public class HttpFileTransferImpl implements FileTransfer { private final UrlValidator validator = new UrlValidator(new String[]{"http", "https"}, UrlValidator.ALLOW_LOCAL_URLS); private final RestTemplate restTemplate; private final Timer downloadTimer; private final Timer uploadTimer; private final Timer getLastModifiedTimer; /** * Constructor. * * @param restTemplate The rest template to use * @param registry The metrics registry to use */ public HttpFileTransferImpl(@NotNull final RestTemplate restTemplate, @NotNull final Registry registry) { this.restTemplate = restTemplate; this.downloadTimer = registry.timer("genie.files.http.download.timer"); this.uploadTimer = registry.timer("genie.files.http.upload.timer"); this.getLastModifiedTimer = registry.timer("genie.files.http.getLastModified.timer"); } /** * {@inheritDoc} */ @Override public boolean isValid(final String fileName) throws GenieException { log.debug("Called with file name {}", fileName); return this.validator.isValid(fileName); } /** * {@inheritDoc} */ @Override public void getFile( @NotBlank(message = "Source file path cannot be empty.") final String srcRemotePath, @NotBlank(message = "Destination local path cannot be empty") final String dstLocalPath ) throws GenieException { final long start = System.nanoTime(); log.debug("Called with src path {} and destination path {}", srcRemotePath, dstLocalPath); try { final File outputFile = new File(dstLocalPath); if (!this.isValid(srcRemotePath)) { throw new GenieServerException("Unable to download " + srcRemotePath + " not a valid URL"); } this.restTemplate.execute( srcRemotePath, HttpMethod.GET, requestEntity -> requestEntity.getHeaders().setAccept(Lists.newArrayList(MediaType.ALL)), new ResponseExtractor<Void>() { @Override public Void extractData(final ClientHttpResponse response) throws IOException { // Documentation I could find pointed to the HttpEntity reading the bytes off // the stream so this should resolve memory problems if the file returned is large FileUtils.copyInputStreamToFile(response.getBody(), outputFile); return null; } } ); } finally { this.downloadTimer.record(System.nanoTime() - start, TimeUnit.NANOSECONDS); } } /** * {@inheritDoc} */ @Override public void putFile( @NotBlank(message = "Source local path cannot be empty.") final String srcLocalPath, @NotBlank(message = "Destination remote path cannot be empty") final String dstRemotePath ) throws GenieException { final long start = System.nanoTime(); try { throw new UnsupportedOperationException( "Saving a file to an HttpEndpoint isn't implemented in this version" ); } finally { this.uploadTimer.record(System.nanoTime() - start, TimeUnit.NANOSECONDS); } } /** * {@inheritDoc} */ @Override public long getLastModifiedTime(final String path) throws GenieException { final long start = System.nanoTime(); try { final URL url = new URL(path); final long time = this.restTemplate.headForHeaders(url.toURI()).getLastModified(); // Returns now if there was no last modified header as best we can do is assume file is brand new return time != -1 ? time : Instant.now().toEpochMilli(); } catch (final MalformedURLException | URISyntaxException e) { log.error(e.getLocalizedMessage(), e); throw new GenieServerException(e); } finally { this.getLastModifiedTimer.record(System.nanoTime() - start, TimeUnit.NANOSECONDS); } } }