ProxiedFileSystemUtils.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package gobblin.util;

import java.io.IOException;
import java.net.URI;
import java.security.PrivilegedExceptionAction;
import java.util.Properties;

import com.google.common.base.Optional;
import com.google.common.base.Preconditions;
import com.google.common.io.Closer;

import lombok.AllArgsConstructor;
import lombok.NonNull;
import lombok.extern.slf4j.Slf4j;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;

import gobblin.configuration.ConfigurationKeys;
import gobblin.configuration.State;


/**
 * Utility class for creating {@link FileSystem} objects while proxied as another user. This class requires access to a
 * user with secure impersonation priveleges. The {@link FileSystem} objects returned will have full permissions to
 * access any operations on behalf of the specified user.
 *
 * <p>
 *   As a user, use methods in {@link gobblin.util.ProxiedFileSystemCache} to generate the proxied file systems.
 * </p>
 *
 * @see <a href="http://hadoop.apache.org/docs/r1.2.1/Secure_Impersonation.html">Secure Impersonation</a>,
 * <a href="https://hadoop.apache.org/docs/r1.2.1/api/org/apache/hadoop/security/UserGroupInformation.html">UserGroupInformation</a>
 *
 * TODO figure out the proper generic type for the {@link Token} objects.
 */
@Slf4j
public class ProxiedFileSystemUtils {

  public static final String AUTH_TYPE_KEY = "gobblin.utility.user.proxy.auth.type";
  public static final String AUTH_TOKEN_PATH = "gobblin.utility.proxy.auth.token.path";

  // Two authentication types for Hadoop Security, through TOKEN or KEYTAB.
  public enum AuthType {
    TOKEN,
    KEYTAB;
  }

  /**
   * Creates a {@link FileSystem} that can perform any operations allowed by the specified userNameToProxyAs.
   *
   * @param userNameToProxyAs The name of the user the super user should proxy as
   * @param properties {@link java.util.Properties} containing initialization properties.
   * @param fsURI The {@link URI} for the {@link FileSystem} that should be created.
   * @param conf The {@link Configuration} for the {@link FileSystem} that should be created.
   * @return a {@link FileSystem} that can execute commands on behalf of the specified userNameToProxyAs
   * @throws IOException
   */
  static FileSystem createProxiedFileSystem(@NonNull final String userNameToProxyAs, Properties properties, URI fsURI,
      Configuration conf) throws IOException {
    Preconditions.checkArgument(properties.containsKey(AUTH_TYPE_KEY));

    switch (AuthType.valueOf(properties.getProperty(AUTH_TYPE_KEY))) {
      case TOKEN:
        Preconditions.checkArgument(properties.containsKey(AUTH_TOKEN_PATH));
        Path tokenPath = new Path(properties.getProperty(AUTH_TOKEN_PATH));
        Optional<Token<?>> proxyToken = getTokenFromSeqFile(userNameToProxyAs, tokenPath);
        if (proxyToken.isPresent()) {
          try {
            return createProxiedFileSystemUsingToken(userNameToProxyAs, proxyToken.get(), fsURI, conf);
          } catch (InterruptedException e) {
            throw new IOException("Failed to proxy as user " + userNameToProxyAs, e);
          }
        }
        throw new IOException("No delegation token found for proxy user " + userNameToProxyAs);
      case KEYTAB:
        Preconditions.checkArgument(properties.containsKey(ConfigurationKeys.SUPER_USER_NAME_TO_PROXY_AS_OTHERS)
            && properties.containsKey(ConfigurationKeys.SUPER_USER_KEY_TAB_LOCATION));
        String superUserName = properties.getProperty(ConfigurationKeys.SUPER_USER_NAME_TO_PROXY_AS_OTHERS);
        Path keytabPath = new Path(properties.getProperty(ConfigurationKeys.SUPER_USER_KEY_TAB_LOCATION));
        try {
          return createProxiedFileSystemUsingKeytab(userNameToProxyAs, superUserName, keytabPath, fsURI, conf);
        } catch (InterruptedException e) {
          throw new IOException("Failed to proxy as user " + userNameToProxyAs, e);
        }
      default:
        throw new IOException("User proxy auth type " + properties.getProperty(AUTH_TYPE_KEY) + " not recognized.");
    }
  }

  /**
   * Creates a {@link FileSystem} that can perform any operations allowed by the specified userNameToProxyAs. This
   * method first logs in as the specified super user. If Hadoop security is enabled, then logging in entails
   * authenticating via Kerberos. So logging in requires contacting the Kerberos infrastructure. A proxy user is then
   * created on behalf of the logged in user, and a {@link FileSystem} object is created using the proxy user's UGI.
   *
   * @param userNameToProxyAs The name of the user the super user should proxy as
   * @param superUserName The name of the super user with secure impersonation priveleges
   * @param superUserKeytabLocation The location of the keytab file for the super user
   * @param fsURI The {@link URI} for the {@link FileSystem} that should be created
   * @param conf The {@link Configuration} for the {@link FileSystem} that should be created
   *
   * @return a {@link FileSystem} that can execute commands on behalf of the specified userNameToProxyAs
   */
  static FileSystem createProxiedFileSystemUsingKeytab(String userNameToProxyAs, String superUserName,
      Path superUserKeytabLocation, URI fsURI, Configuration conf) throws IOException, InterruptedException {

    return loginAndProxyAsUser(userNameToProxyAs, superUserName, superUserKeytabLocation)
        .doAs(new ProxiedFileSystem(fsURI, conf));
  }

  /**
   * Create a {@link FileSystem} that can perform any operations allowed the by the specified userNameToProxyAs. This
   * method uses the {@link #createProxiedFileSystemUsingKeytab(String, String, Path, URI, Configuration)} object to perform
   * all its work. A specific set of configuration keys are required to be set in the given {@link State} object:
   *
   * <ul>
   *  <li>{@link ConfigurationKeys#FS_PROXY_AS_USER_NAME} specifies the user name to proxy as</li>
   *  <li>{@link ConfigurationKeys#SUPER_USER_NAME_TO_PROXY_AS_OTHERS} specifies the name of the user with secure
   *  impersonation priveleges</li>
   *  <li>{@link ConfigurationKeys#SUPER_USER_KEY_TAB_LOCATION} specifies the location of the super user's keytab file</li>
   * <ul>
   *
   * @param state The {@link State} object that contains all the necessary key, value pairs for
   * {@link #createProxiedFileSystemUsingKeytab(String, String, Path, URI, Configuration)}
   * @param fsURI The {@link URI} for the {@link FileSystem} that should be created
   * @param conf The {@link Configuration} for the {@link FileSystem} that should be created
   *
   * @return a {@link FileSystem} that can execute commands on behalf of the specified userNameToProxyAs
   */
  static FileSystem createProxiedFileSystemUsingKeytab(State state, URI fsURI, Configuration conf)
      throws IOException, InterruptedException {
    Preconditions.checkArgument(state.contains(ConfigurationKeys.FS_PROXY_AS_USER_NAME));
    Preconditions.checkArgument(state.contains(ConfigurationKeys.SUPER_USER_NAME_TO_PROXY_AS_OTHERS));
    Preconditions.checkArgument(state.contains(ConfigurationKeys.SUPER_USER_KEY_TAB_LOCATION));

    return createProxiedFileSystemUsingKeytab(state.getProp(ConfigurationKeys.FS_PROXY_AS_USER_NAME),
        state.getProp(ConfigurationKeys.SUPER_USER_NAME_TO_PROXY_AS_OTHERS),
        new Path(state.getProp(ConfigurationKeys.SUPER_USER_KEY_TAB_LOCATION)), fsURI, conf);
  }

  /**
   * Create a {@link FileSystem} that can perform any operations allowed the by the specified userNameToProxyAs. The
   * method first proxies as userNameToProxyAs, and then adds the specified {@link Token} to the given
   * {@link UserGroupInformation} object. It then uses the {@link UserGroupInformation#doAs(PrivilegedExceptionAction)}
   * method to create a {@link FileSystem}.
   *
   * @param userNameToProxyAs The name of the user the super user should proxy as
   * @param userNameToken The {@link Token} to add to the proxied user's {@link UserGroupInformation}.
   * @param fsURI The {@link URI} for the {@link FileSystem} that should be created
   * @param conf The {@link Configuration} for the {@link FileSystem} that should be created
   *
   * @return a {@link FileSystem} that can execute commands on behalf of the specified userNameToProxyAs
   */
  static FileSystem createProxiedFileSystemUsingToken(@NonNull String userNameToProxyAs,
      @NonNull Token<?> userNameToken, URI fsURI, Configuration conf) throws IOException, InterruptedException {
    UserGroupInformation ugi =
        UserGroupInformation.createProxyUser(userNameToProxyAs, UserGroupInformation.getLoginUser());
    ugi.addToken(userNameToken);
    return ugi.doAs(new ProxiedFileSystem(fsURI, conf));
  }

  /**
   * Returns true if superUserName can proxy as userNameToProxyAs using the specified superUserKeytabLocation, false
   * otherwise.
   */
  public static boolean canProxyAs(String userNameToProxyAs, String superUserName, Path superUserKeytabLocation) {
    try {
      loginAndProxyAsUser(userNameToProxyAs, superUserName, superUserKeytabLocation);
    } catch (IOException e) {
      return false;
    }
    return true;
  }

  /**
   * Retrives a {@link Token} from a given sequence file for a specified user. The sequence file should contain a list
   * of key, value pairs where each key corresponds to a user and each value corresponds to a {@link Token} for that
   * user.
   *
   * @param userNameKey The name of the user to retrieve a {@link Token} for
   * @param tokenFilePath The path to the sequence file containing the {@link Token}s
   *
   * @return A {@link Token} for the given user name
   */
  public static Optional<Token<?>> getTokenFromSeqFile(String userNameKey, Path tokenFilePath) throws IOException {
    log.info("Reading tokens from sequence file " + tokenFilePath);

    try (Closer closer = Closer.create()) {
      FileSystem localFs = FileSystem.getLocal(new Configuration());
      @SuppressWarnings("deprecation")
      SequenceFile.Reader tokenReader =
          closer.register(new SequenceFile.Reader(localFs, tokenFilePath, localFs.getConf()));
      Text key = new Text();
      Token<?> value = new Token<>();
      while (tokenReader.next(key, value)) {
        log.debug("Found token for user: " + key);
        if (key.toString().equals(userNameKey)) {
          return Optional.<Token<?>> of(value);
        }
      }
    }
    log.warn("Did not find any tokens for user " + userNameKey);
    return Optional.absent();
  }

  private static UserGroupInformation loginAndProxyAsUser(@NonNull String userNameToProxyAs,
      @NonNull String superUserName, Path superUserKeytabLocation) throws IOException {

    if (!UserGroupInformation.getLoginUser().getUserName().equals(superUserName)) {
      Preconditions.checkNotNull(superUserKeytabLocation);
      UserGroupInformation.loginUserFromKeytab(superUserName, superUserKeytabLocation.toString());
    }
    return UserGroupInformation.createProxyUser(userNameToProxyAs, UserGroupInformation.getLoginUser());
  }

  @AllArgsConstructor
  private static class ProxiedFileSystem implements PrivilegedExceptionAction<FileSystem> {

    @NonNull
    private URI fsURI;

    @NonNull
    private Configuration conf;

    @Override
    public FileSystem run() throws IOException {

      log.info("Creating a filesystem for user: " + UserGroupInformation.getCurrentUser());
      return FileSystem.get(this.fsURI, this.conf);
    }
  }
}