MRCompactor.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package gobblin.compaction.mapreduce;

import static gobblin.compaction.dataset.Dataset.DatasetState.COMPACTION_COMPLETE;
import static gobblin.compaction.dataset.Dataset.DatasetState.GIVEN_UP;
import static gobblin.compaction.dataset.Dataset.DatasetState.UNVERIFIED;
import static gobblin.compaction.dataset.Dataset.DatasetState.VERIFIED;
import static gobblin.compaction.mapreduce.MRCompactorJobRunner.Status.ABORTED;
import static gobblin.compaction.mapreduce.MRCompactorJobRunner.Status.COMMITTED;

import java.io.IOException;
import java.net.URI;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.PriorityBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.regex.Pattern;
import java.lang.reflect.InvocationTargetException;

import org.joda.time.DateTime;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapreduce.Job;
import org.joda.time.DateTimeZone;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.google.common.base.Functions;
import com.google.common.base.Optional;
import com.google.common.base.Preconditions;
import com.google.common.base.Stopwatch;
import com.google.common.base.Throwables;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import com.google.common.io.Closer;
import com.google.common.util.concurrent.FutureCallback;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;

import gobblin.compaction.Compactor;
import gobblin.compaction.listeners.CompactorCompletionListener;
import gobblin.compaction.listeners.CompactorCompletionListenerFactory;
import gobblin.compaction.listeners.CompactorListener;
import gobblin.compaction.dataset.Dataset;
import gobblin.compaction.dataset.DatasetsFinder;
import gobblin.compaction.dataset.TimeBasedSubDirDatasetsFinder;
import gobblin.compaction.event.CompactionSlaEventHelper;
import gobblin.compaction.verify.DataCompletenessVerifier;
import gobblin.compaction.verify.DataCompletenessVerifier.Results;
import gobblin.configuration.ConfigurationKeys;
import gobblin.configuration.State;
import gobblin.metrics.GobblinMetrics;
import gobblin.metrics.Tag;
import gobblin.metrics.event.EventSubmitter;
import gobblin.util.ClassAliasResolver;
import gobblin.util.DatasetFilterUtils;
import gobblin.util.ExecutorsUtils;
import gobblin.util.HadoopUtils;
import gobblin.util.ClusterNameTags;
import gobblin.util.FileListUtils;
import gobblin.util.recordcount.CompactionRecordCountProvider;
import gobblin.util.recordcount.IngestionRecordCountProvider;
import gobblin.util.reflection.GobblinConstructorUtils;

/**
 * MapReduce-based {@link gobblin.compaction.Compactor}. Compaction will run on each qualified {@link Dataset}
 * under {@link #COMPACTION_INPUT_DIR}.
 *
 * @author Ziyang Liu
 */

public class MRCompactor implements Compactor {

  private static final Logger LOG = LoggerFactory.getLogger(MRCompactor.class);

  public static final String COMPACTION_PREFIX = "compaction.";

  /**
   * Basic compaction properties.
   */
  public static final String COMPACTION_THREAD_POOL_SIZE = COMPACTION_PREFIX + "thread.pool.size";
  public static final int DEFAULT_COMPACTION_THREAD_POOL_SIZE = 30;
  public static final String COMPACTION_INPUT_DIR = COMPACTION_PREFIX + "input.dir";

  // The subdir name of input dataset paths, e.g., "hourly" in "/data/input/PasswordChangeEvent/hourly/2015/09/06".
  public static final String COMPACTION_INPUT_SUBDIR = COMPACTION_PREFIX + "input.subdir";
  public static final String DEFAULT_COMPACTION_INPUT_SUBDIR = "hourly";

  public static final String COMPACTION_DEST_DIR = COMPACTION_PREFIX + "dest.dir";

  // The subdir name of output dataset paths, e.g., "daily" in "/data/input/PasswordChangeEvent/daily/2015/09/06".
  public static final String COMPACTION_DEST_SUBDIR = COMPACTION_PREFIX + "dest.subdir";
  public static final String DEFAULT_COMPACTION_DEST_SUBDIR = "daily";

  // The output dir for compaction MR job, which will be moved to the final output dir for data publishing.
  public static final String COMPACTION_TMP_DEST_DIR = COMPACTION_PREFIX + "tmp.dest.dir";
  public static final String DEFAULT_COMPACTION_TMP_DEST_DIR = "/tmp/gobblin-compaction";
  public static final String COMPACTION_JOB_DIR = COMPACTION_PREFIX + "tmp.job.dir";
  public static final String COMPACTION_LATE_DIR_SUFFIX = "_late";

  public static final String COMPACTION_BLACKLIST = COMPACTION_PREFIX + "blacklist";
  public static final String COMPACTION_WHITELIST = COMPACTION_PREFIX + "whitelist";
  public static final String COMPACTION_HIGH_PRIORITY_TOPICS = COMPACTION_PREFIX + "high.priority.topics";
  public static final String COMPACTION_NORMAL_PRIORITY_TOPICS = COMPACTION_PREFIX + "normal.priority.topics";

  public static final String COMPACTION_JOB_RUNNER_CLASS = COMPACTION_PREFIX + "job.runner.class";
  public static final String DEFAULT_COMPACTION_JOB_RUNNER_CLASS =
      "gobblin.compaction.mapreduce.avro.MRCompactorAvroKeyDedupJobRunner";
  public static final String COMPACTION_TIMEZONE = COMPACTION_PREFIX + "timezone";
  public static final String DEFAULT_COMPACTION_TIMEZONE = ConfigurationKeys.PST_TIMEZONE_NAME;
  public static final String COMPACTION_FILE_SYSTEM_URI = COMPACTION_PREFIX + "file.system.uri";
  public static final String COMPACTION_MR_JOB_TIMEOUT_MINUTES = COMPACTION_PREFIX + "mr.job.timeout.minutes";
  public static final long DEFAULT_COMPACTION_MR_JOB_TIMEOUT_MINUTES = Long.MAX_VALUE;

  // Dataset finder to find datasets for compaction.
  public static final String COMPACTION_DATASETS_FINDER = COMPACTION_PREFIX + "datasets.finder";
  public static final String DEFAULT_COMPACTION_DATASETS_FINDER = TimeBasedSubDirDatasetsFinder.class.getName();


  // Rename source directories as a compaction complete indication
  // Compaction jobs using this completion mode can't share input sources
  public static final String COMPACTION_RENAME_SOURCE_DIR_ENABLED = COMPACTION_PREFIX + "rename.source.dir.enabled";
  public static final boolean DEFAULT_COMPACTION_RENAME_SOURCE_DIR_ENABLED = false;
  public static final String COMPACTION_RENAME_SOURCE_DIR_SUFFIX = "_COMPLETE";


  //The provider that provides event counts for the compaction input files.
  public static final String COMPACTION_INPUT_RECORD_COUNT_PROVIDER = COMPACTION_PREFIX + "input.record.count.provider";
  public static final String DEFAULT_COMPACTION_INPUT_RECORD_COUNT_PROVIDER =
      IngestionRecordCountProvider.class.getName();

  //The provider that provides event counts for the compaction output files.
  public static final String COMPACTION_OUTPUT_RECORD_COUNT_PROVIDER =
      COMPACTION_PREFIX + "output.record.count.provider";
  public static final String DEFAULT_COMPACTION_OUTPUT_RECORD_COUNT_PROVIDER =
      CompactionRecordCountProvider.class.getName();

  // If a dataset has already been compacted and new (late) data is found, whether recompact this dataset.
  public static final String COMPACTION_RECOMPACT_FROM_INPUT_FOR_LATE_DATA =
      COMPACTION_PREFIX + "recompact.from.input.for.late.data";
  public static final boolean DEFAULT_COMPACTION_RECOMPACT_FROM_INPUT_FOR_LATE_DATA = false;

  // The threshold of new(late) data that will trigger recompaction per dataset.
  // It follows the pattern DATASET_NAME_REGEX:THRESHOLD;DATASET_NAME_REGEX:THRESHOLD, e.g., A.*,B.*:0.2; C.*,D.*:0.3.
  // Dataset names that match A.* or B.* will have threshold 0.2. Dataset names that match C.* or D.* will have threshold 0.3.
  public static final String COMPACTION_LATEDATA_THRESHOLD_FOR_RECOMPACT_PER_DATASET =
      COMPACTION_PREFIX + "latedata.threshold.for.recompact.per.topic";
  public static final double DEFAULT_COMPACTION_LATEDATA_THRESHOLD_FOR_RECOMPACT_PER_DATASET = 1.0;

  // The threshold of new (late) files that will trigger compaction per dataset.
  // The trigger is based on the file numbers in the late output directory
  public static final String COMPACTION_LATEDATA_THRESHOLD_FILE_NUM =
      COMPACTION_PREFIX + "latedata.threshold.file.num";
  public static final int DEFAULT_COMPACTION_LATEDATA_THRESHOLD_FILE_NUM = 1000;

  // The threshold of new (late) files that will trigger compaction per dataset.
  // The trigger is based on how long the file has been in the late output directory.
  public static final String COMPACTION_LATEDATA_THRESHOLD_DURATION =
      COMPACTION_PREFIX + "latedata.threshold.duration";
  public static final String DEFAULT_COMPACTION_LATEDATA_THRESHOLD_DURATION = "24h";


  public static final String COMPACTION_RECOMPACT_CONDITION = COMPACTION_PREFIX + "recompact.condition";
  public static final String DEFAULT_COMPACTION_RECOMPACT_CONDITION = "RecompactBasedOnRatio";

  public static final String COMPACTION_RECOMPACT_COMBINE_CONDITIONS = COMPACTION_PREFIX + "recompact.combine.conditions";
  public static final String COMPACTION_RECOMPACT_COMBINE_CONDITIONS_OPERATION = COMPACTION_PREFIX + "recompact.combine.conditions.operation";
  public static final String DEFAULT_COMPACTION_RECOMPACT_COMBINE_CONDITIONS_OPERATION = "or";

  public static final String COMPACTION_COMPLETE_LISTERNER = COMPACTION_PREFIX + "complete.listener";
  public static final String DEFAULT_COMPACTION_COMPLETE_LISTERNER = "SimpleCompactorCompletionHook";

  // Whether the input data for the compaction is deduplicated.
  public static final String COMPACTION_INPUT_DEDUPLICATED = COMPACTION_PREFIX + "input.deduplicated";
  public static final boolean DEFAULT_COMPACTION_INPUT_DEDUPLICATED = false;

  // Whether the output of the compaction should be deduplicated.
  public static final String COMPACTION_OUTPUT_DEDUPLICATED = COMPACTION_PREFIX + "output.deduplicated";
  public static final boolean DEFAULT_COMPACTION_OUTPUT_DEDUPLICATED = true;

  public static final String COMPACTION_COMPLETENESS_VERIFICATION_PREFIX =
      COMPACTION_PREFIX + "completeness.verification.";

  public static final String COMPACTION_RECOMPACT_FROM_DEST_PATHS = COMPACTION_PREFIX + "recompact.from.dest.paths";
  public static final String COMPACTION_RECOMPACT_ALL_DATA = COMPACTION_PREFIX + "recompact.all.data";
  public static final boolean DEFAULT_COMPACTION_RECOMPACT_FROM_DEST_PATHS = false;
  public static final boolean DEFAULT_COMPACTION_RECOMPACT_ALL_DATA = true;

  /**
   * Configuration properties related to data completeness verification.
   */
  public static final String COMPACTION_COMPLETENESS_VERIFICATION_BLACKLIST =
      COMPACTION_COMPLETENESS_VERIFICATION_PREFIX + "blacklist";
  public static final String COMPACTION_COMPLETENESS_VERIFICATION_WHITELIST =
      COMPACTION_COMPLETENESS_VERIFICATION_PREFIX + "whitelist";
  public static final String COMPACTION_VERIFICATION_TIMEOUT_MINUTES =
      COMPACTION_COMPLETENESS_VERIFICATION_PREFIX + "timeout.minutes";
  public static final long DEFAULT_COMPACTION_VERIFICATION_TIMEOUT_MINUTES = 30;
  public static final String COMPACTION_COMPLETENESS_VERIFICATION_ENABLED =
      COMPACTION_COMPLETENESS_VERIFICATION_PREFIX + "enabled";
  public static final boolean DEFAULT_COMPACTION_COMPLETENESS_VERIFICATION_ENABLED = false;

  // Number of datasets to be passed to DataCompletenessVerifier together. By passing multiple datasets together,
  // some costs in DataCompletenessVerifier (e.g., submitting a SQL query) can be amortized.
  public static final String COMPACTION_COMPLETENESS_VERIFICATION_NUM_DATASETS_VERIFIED_TOGETHER =
      COMPACTION_COMPLETENESS_VERIFICATION_PREFIX + "num.datasets.verified.together";
  public static final int DEFAULT_COMPACTION_COMPLETENESS_VERIFICATION_NUM_DATASETS_VERIFIED_TOGETHER = 10;

  // Whether to compact and publish a datatset if its completeness cannot be verified.
  public static final String COMPACTION_COMPLETENESS_VERIFICATION_PUBLISH_DATA_IF_CANNOT_VERIFY =
      COMPACTION_COMPLETENESS_VERIFICATION_PREFIX + "publish.data.if.cannot.verify";
  public static final boolean DEFAULT_COMPACTION_COMPLETENESS_VERIFICATION_PUBLISH_DATA_IF_CANNOT_VERIFY = false;

  /**
   * Compaction configuration properties used internally.
   */
  public static final String COMPACTION_SHOULD_DEDUPLICATE = COMPACTION_PREFIX + "should.deduplicate";
  public static final String COMPACTION_JOB_DEST_PARTITION = COMPACTION_PREFIX + "job.dest.partition";
  public static final String COMPACTION_ENABLE_SUCCESS_FILE =
      COMPACTION_PREFIX + "fileoutputcommitter.marksuccessfuljobs";
  public static final String COMPACTION_JOB_LATE_DATA_MOVEMENT_TASK = COMPACTION_PREFIX + "job.late.data.movement.task";
  public static final String COMPACTION_JOB_LATE_DATA_FILES = COMPACTION_PREFIX + "job.late.data.files";
  public static final String COMPACTION_COMPLETE_FILE_NAME = "_COMPACTION_COMPLETE";
  public static final String COMPACTION_LATE_FILES_DIRECTORY = "late";
  public static final String COMPACTION_JARS = COMPACTION_PREFIX + "jars";
  public static final String COMPACTION_JAR_SUBDIR = "_gobblin_compaction_jars";
  public static final String COMPACTION_TRACKING_EVENTS_NAMESPACE = COMPACTION_PREFIX + "tracking.events";

  public static final String COMPACTION_INPUT_PATH_TIME = COMPACTION_PREFIX + "input.path.time";

  private static final long COMPACTION_JOB_WAIT_INTERVAL_SECONDS = 10;
  private static final Map<Dataset, Job> RUNNING_MR_JOBS = Maps.newConcurrentMap();

  private final State state;
  private final List<? extends Tag<?>> tags;
  private final Configuration conf;
  private final String tmpOutputDir;
  private final FileSystem fs;
  private final JobRunnerExecutor jobExecutor;
  private final Set<Dataset> datasets;
  private final Map<Dataset, MRCompactorJobRunner> jobRunnables;
  private final Closer closer;
  private final Optional<DataCompletenessVerifier> verifier;
  private final Stopwatch stopwatch;
  private final GobblinMetrics gobblinMetrics;
  private final EventSubmitter eventSubmitter;
  private final Optional<CompactorListener> compactorListener;
  private final DateTime initilizeTime;
  private final long dataVerifTimeoutMinutes;
  private final long compactionTimeoutMinutes;
  private final boolean shouldVerifDataCompl;
  private final boolean shouldPublishDataIfCannotVerifyCompl;
  private final CompactorCompletionListener compactionCompleteListener;

  public MRCompactor(Properties props, List<? extends Tag<?>> tags, Optional<CompactorListener> compactorListener)
      throws IOException {
    this.state = new State();
    this.state.addAll(props);
    this.initilizeTime = getCurrentTime();
    this.tags = tags;
    this.conf = HadoopUtils.getConfFromState(this.state);
    this.tmpOutputDir = getTmpOutputDir();
    this.fs = getFileSystem();
    this.datasets = getDatasetsFinder().findDistinctDatasets();
    this.jobExecutor = createJobExecutor();
    this.jobRunnables = Maps.newConcurrentMap();
    this.closer = Closer.create();
    this.stopwatch = Stopwatch.createStarted();
    this.gobblinMetrics = initializeMetrics();
    this.eventSubmitter = new EventSubmitter.Builder(
        GobblinMetrics.get(this.state.getProp(ConfigurationKeys.JOB_NAME_KEY)).getMetricContext(),
        MRCompactor.COMPACTION_TRACKING_EVENTS_NAMESPACE).build();
    this.compactorListener = compactorListener;
    this.dataVerifTimeoutMinutes = getDataVerifTimeoutMinutes();
    this.compactionTimeoutMinutes = getCompactionTimeoutMinutes();
    this.shouldVerifDataCompl = shouldVerifyDataCompleteness();
    this.compactionCompleteListener = getCompactionCompleteListener();
    this.verifier =
        this.shouldVerifDataCompl ? Optional.of(this.closer.register(new DataCompletenessVerifier(this.state)))
            : Optional.<DataCompletenessVerifier> absent();
    this.shouldPublishDataIfCannotVerifyCompl = shouldPublishDataIfCannotVerifyCompl();
  }

  public DateTime getInitializeTime() {
    return this.initilizeTime;
  }

  private String getTmpOutputDir() {
    return this.state.getProp(COMPACTION_TMP_DEST_DIR, DEFAULT_COMPACTION_TMP_DEST_DIR);
  }

  private FileSystem getFileSystem() throws IOException {
    if (this.state.contains(COMPACTION_FILE_SYSTEM_URI)) {
      URI uri = URI.create(this.state.getProp(COMPACTION_FILE_SYSTEM_URI));
      return FileSystem.get(uri, this.conf);
    }
    return FileSystem.get(this.conf);
  }

  private DatasetsFinder getDatasetsFinder() {
    try {
      return (DatasetsFinder) Class
          .forName(this.state.getProp(COMPACTION_DATASETS_FINDER, DEFAULT_COMPACTION_DATASETS_FINDER))
          .getConstructor(State.class).newInstance(this.state);
    } catch (Exception e) {
      throw new RuntimeException("Failed to initiailize DatasetsFinder.", e);
    }
  }

  private DateTime getCurrentTime () {
    DateTimeZone timeZone = DateTimeZone
        .forID(this.state.getProp(MRCompactor.COMPACTION_TIMEZONE, MRCompactor.DEFAULT_COMPACTION_TIMEZONE));
    return new DateTime (timeZone);
  }

  private JobRunnerExecutor createJobExecutor() {
    int threadPoolSize = getThreadPoolSize();
    BlockingQueue<Runnable> queue = new PriorityBlockingQueue<>();
    return new JobRunnerExecutor(threadPoolSize, threadPoolSize, Long.MAX_VALUE, TimeUnit.NANOSECONDS, queue);
  }

  private int getThreadPoolSize() {
    return this.state.getPropAsInt(COMPACTION_THREAD_POOL_SIZE, DEFAULT_COMPACTION_THREAD_POOL_SIZE);
  }

  private GobblinMetrics initializeMetrics() {
    ImmutableList.Builder<Tag<?>> tags = ImmutableList.builder();
    tags.addAll(this.tags);
    tags.addAll(Tag.fromMap(ClusterNameTags.getClusterNameTags()));
    GobblinMetrics gobblinMetrics =
        GobblinMetrics.get(this.state.getProp(ConfigurationKeys.JOB_NAME_KEY), null, tags.build());
    gobblinMetrics.startMetricReporting(this.state.getProperties());
    return gobblinMetrics;
  }

  @Override
  public void compact() throws IOException {
    try {
      copyDependencyJarsToHdfs();
      processDatasets();
      throwExceptionsIfAnyDatasetCompactionFailed();
      onCompactionCompletion();
    } catch (Throwable t) {

      // This throwable is logged here before propagated. Otherwise, if another throwable is thrown
      // in the finally-block, this throwable may be suppressed.
      LOG.error("Caught throwable during compaction", t);
      throw Throwables.propagate(t);
    } finally {
      try {
        shutdownExecutors();
        this.closer.close();
      } finally {
        deleteDependencyJars();
        this.gobblinMetrics.stopMetricsReporting();
      }
    }
  }

  private CompactorCompletionListener getCompactionCompleteListener () {
    ClassAliasResolver<CompactorCompletionListenerFactory> classAliasResolver = new ClassAliasResolver<>(CompactorCompletionListenerFactory.class);
    String listenerName= this.state.getProp(MRCompactor.COMPACTION_COMPLETE_LISTERNER,
        MRCompactor.DEFAULT_COMPACTION_COMPLETE_LISTERNER);
    try {
      CompactorCompletionListenerFactory factory = GobblinConstructorUtils.invokeFirstConstructor(
          classAliasResolver.resolveClass(listenerName), ImmutableList.of());
      return factory.createCompactorCompactionListener(this.state);
    } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException | InstantiationException
        | ClassNotFoundException e) {
      throw new IllegalArgumentException(e);
    }
  }

  private void onCompactionCompletion() {
    this.compactionCompleteListener.onCompactionCompletion(this);
  }

  /**
   * Copy dependency jars from local fs to HDFS.
   */
  private void copyDependencyJarsToHdfs() throws IOException {
    if (!this.state.contains(ConfigurationKeys.JOB_JAR_FILES_KEY)) {
      return;
    }
    LocalFileSystem lfs = FileSystem.getLocal(this.conf);
    Path tmpJarFileDir = new Path(this.tmpOutputDir, "_gobblin_compaction_jars");
    this.state.setProp(COMPACTION_JARS, tmpJarFileDir.toString());
    this.fs.delete(tmpJarFileDir, true);
    for (String jarFile : this.state.getPropAsList(ConfigurationKeys.JOB_JAR_FILES_KEY)) {
      for (FileStatus status : lfs.globStatus(new Path(jarFile))) {
        Path tmpJarFile = new Path(this.fs.makeQualified(tmpJarFileDir), status.getPath().getName());
        this.fs.copyFromLocalFile(status.getPath(), tmpJarFile);
        LOG.info(String.format("%s will be added to classpath", tmpJarFile));
      }
    }
  }

  /**
   * Delete dependency jars from HDFS when job is done.
   */
  private void deleteDependencyJars() throws IllegalArgumentException, IOException {
    if (this.state.contains(COMPACTION_JARS)) {
      this.fs.delete(new Path(this.state.getProp(COMPACTION_JARS)), true);
    }
  }

  private void processDatasets() {
    createJobPropsForDatasets();
    processCompactionJobs();
  }

  /**
   * Create compaction job properties for {@link Dataset}s.
   */
  private void createJobPropsForDatasets() {
    final Set<Dataset> datasetsWithProps = Sets.newHashSet();
    for (Dataset dataset : this.datasets) {
      datasetsWithProps.addAll(createJobPropsForDataset(dataset));
    }

    this.datasets.clear();
    this.datasets.addAll(datasetsWithProps);
  }

  /**
   * Existing dataset in {@link #datasets} does not have job props.
   * Create compaction job properties for each given {@link Dataset}.
   * Update datasets based on the results of creating job props for them.
   */
  private List<Dataset> createJobPropsForDataset(Dataset dataset) {
    LOG.info("Creating compaction jobs for dataset " + dataset + " with priority " + dataset.priority());
    final MRCompactorJobPropCreator jobPropCreator = getJobPropCreator(dataset);
    List<Dataset> datasetsWithProps;
    try {
      datasetsWithProps = jobPropCreator.createJobProps();
    } catch (Throwable t) {
      // If a throwable is caught when creating job properties for a dataset, skip the topic and add the throwable
      // to the dataset.
      datasetsWithProps = ImmutableList.<Dataset> of(jobPropCreator.createFailedJobProps(t));
    }
    return datasetsWithProps;
  }

  /**
   * Get an instance of {@link MRCompactorJobPropCreator}.
   */
  MRCompactorJobPropCreator getJobPropCreator(Dataset dataset) {
    try {
      return new MRCompactorJobPropCreator.Builder().withDataset(dataset).withFileSystem(this.fs).withState(this.state)
          .build();
    } catch (Exception e) {
      throw new RuntimeException(e);
    }
  }

  public Set<Dataset> getDatasets() {
    return this.datasets;
  }

  private void processCompactionJobs() {
    if (this.shouldVerifDataCompl) {
      verifyDataCompleteness();
    } else {
      setAllDatasetStatesToVerified();
    }

    this.submitCompactionJobsAndWaitForCompletion();
  }

  private boolean shouldVerifyDataCompleteness() {
    return this.state.getPropAsBoolean(COMPACTION_COMPLETENESS_VERIFICATION_ENABLED,
        DEFAULT_COMPACTION_COMPLETENESS_VERIFICATION_ENABLED);
  }

  private void verifyDataCompleteness() {
    List<Pattern> blacklist =
        DatasetFilterUtils.getPatternList(this.state, COMPACTION_COMPLETENESS_VERIFICATION_BLACKLIST);
    List<Pattern> whitelist =
        DatasetFilterUtils.getPatternList(this.state, COMPACTION_COMPLETENESS_VERIFICATION_WHITELIST);
    int numDatasetsVerifiedTogether = getNumDatasetsVerifiedTogether();
    List<Dataset> datasetsToBeVerified = Lists.newArrayList();
    for (Dataset dataset : this.datasets) {
      if (dataset.state() != UNVERIFIED) {
        continue;
      }
      if (shouldVerifyCompletenessForDataset(dataset, blacklist, whitelist)) {
        datasetsToBeVerified.add(dataset);
        if (datasetsToBeVerified.size() >= numDatasetsVerifiedTogether) {
          ListenableFuture<Results> future = this.verifier.get().verify(datasetsToBeVerified);
          addCallback(datasetsToBeVerified, future);
          datasetsToBeVerified = Lists.newArrayList();
        }
      } else {
        dataset.setState(VERIFIED);
      }
    }

    if (!datasetsToBeVerified.isEmpty()) {
      ListenableFuture<Results> future = this.verifier.get().verify(datasetsToBeVerified);
      addCallback(datasetsToBeVerified, future);
    }
  }

  /**
   * A {@link Dataset} should be verified if its not already compacted, and it satisfies the blacklist and whitelist.
   */
  private boolean shouldVerifyCompletenessForDataset(Dataset dataset, List<Pattern> blacklist,
    List<Pattern> whitelist) {
    boolean renamingRequired = this.state.getPropAsBoolean(COMPACTION_RENAME_SOURCE_DIR_ENABLED, DEFAULT_COMPACTION_RENAME_SOURCE_DIR_ENABLED);

    LOG.info ("Should verify completeness with renaming source dir : " + renamingRequired);

    return !datasetAlreadyCompacted(this.fs, dataset, renamingRequired)
        && DatasetFilterUtils.survived(dataset.getName(), blacklist, whitelist);
  }

  /**
   * Get all the renamed directories from the given paths
   * They are deepest level containing directories whose name has a suffix {@link MRCompactor#COMPACTION_RENAME_SOURCE_DIR_SUFFIX}
   * Also each directory needs to contain at least one file so empty directories will be excluded from the result
   */
  public static Set<Path> getDeepestLevelRenamedDirsWithFileExistence (FileSystem fs, Set<Path> paths) throws IOException {
    Set<Path> renamedDirs = Sets.newHashSet();
    for (FileStatus fileStatus : FileListUtils.listFilesRecursively(fs, paths)) {
      if (fileStatus.getPath().getParent().toString().endsWith(MRCompactor.COMPACTION_RENAME_SOURCE_DIR_SUFFIX)) {
        renamedDirs.add(fileStatus.getPath().getParent());
      }
    }

    return renamedDirs;
  }

    /**
     * Get all the unrenamed directories from the given paths
     * They are deepest level containing directories whose name doesn't have a suffix {@link MRCompactor#COMPACTION_RENAME_SOURCE_DIR_SUFFIX}
     * Also each directory needs to contain at least one file so empty directories will be excluded from the result
     */
  public static Set<Path> getDeepestLevelUnrenamedDirsWithFileExistence (FileSystem fs, Set<Path> paths) throws IOException {
    Set<Path> unrenamed = Sets.newHashSet();
    for (FileStatus fileStatus : FileListUtils.listFilesRecursively(fs, paths)) {
      if (!fileStatus.getPath().getParent().toString().endsWith(MRCompactor.COMPACTION_RENAME_SOURCE_DIR_SUFFIX)) {
        unrenamed.add(fileStatus.getPath().getParent());
      }
    }

    return unrenamed;
  }

  /**
   * Rename all the source directories for a specific dataset
   */
  public static void renameSourceDirAsCompactionComplete (FileSystem fs, Dataset dataset) {
    try {
      for (Path path: dataset.getRenamePaths()) {
        Path newPath = new Path (path.getParent(), path.getName() + MRCompactor.COMPACTION_RENAME_SOURCE_DIR_SUFFIX);
        LOG.info("[{}] Renaming {} to {}", dataset.getDatasetName(), path, newPath);
        fs.rename(path, newPath);
      }
    } catch (Exception e) {
      LOG.error ("Rename input path failed", e);
    }
  }

  /**
   * A {@link Dataset} is considered already compacted if either condition is true:
   * 1) When completion file strategy is used, a compaction completion means there is a file named
   *    {@link MRCompactor#COMPACTION_COMPLETE_FILE_NAME} in its {@link Dataset#outputPath()}.
   * 2) When renaming source directory strategy is used, a compaction completion means source directories
   *    {@link Dataset#inputPaths()} contains at least one directory which has been renamed to something with
   *    {@link MRCompactor#COMPACTION_RENAME_SOURCE_DIR_SUFFIX}.
   */
  public static boolean datasetAlreadyCompacted(FileSystem fs, Dataset dataset, boolean renameSourceEnable) {
    if (renameSourceEnable) {
      return checkAlreadyCompactedBasedOnSourceDirName (fs, dataset);
    } else {
      return checkAlreadyCompactedBasedOnCompletionFile(fs, dataset);
    }
  }


  /**  When renaming source directory strategy is used, a compaction completion means source directories
   *    {@link Dataset#inputPaths()} contains at least one directory which has been renamed to something with
   *    {@link MRCompactor#COMPACTION_RENAME_SOURCE_DIR_SUFFIX}.
   */
  private static boolean checkAlreadyCompactedBasedOnSourceDirName (FileSystem fs, Dataset dataset) {
    try {
      Set<Path> renamedDirs = getDeepestLevelRenamedDirsWithFileExistence(fs, dataset.inputPaths());
      return !renamedDirs.isEmpty();
    } catch (IOException e) {
      LOG.error("Failed to get deepest directories from source", e);
      return false;
    }
  }

 /**
  *  When completion file strategy is used, a compaction completion means there is a file named
  *    {@link MRCompactor#COMPACTION_COMPLETE_FILE_NAME} in its {@link Dataset#outputPath()}.
  */
  private static boolean checkAlreadyCompactedBasedOnCompletionFile(FileSystem fs, Dataset dataset) {
    Path filePath = new Path(dataset.outputPath(), MRCompactor.COMPACTION_COMPLETE_FILE_NAME);
    try {
      return fs.exists(filePath);
    } catch (IOException e) {
      LOG.error("Failed to verify the existence of file " + filePath, e);
      return false;
    }
  }

  public static long readCompactionTimestamp(FileSystem fs, Path compactionOutputPath) throws IOException {
    Path completionFilePath = new Path(compactionOutputPath, COMPACTION_COMPLETE_FILE_NAME);
    try (FSDataInputStream completionFileStream = fs.open(completionFilePath)) {
      return completionFileStream.readLong();
    }
  }

  private void addCallback(final List<Dataset> datasetsToBeVerified, ListenableFuture<Results> future) {

    Futures.addCallback(future, new FutureCallback<Results>() {

      /**
       * On success, resubmit verification for the {@link Dataset}s that should be resubmitted
       * (i.e., verification didn't pass and it didn't timeout).
       */
      @Override
      public void onSuccess(Results results) {
        List<Dataset> datasetsToBeVerifiedAgain = Lists.newArrayList();
        for (Results.Result result : results) {
          Optional<MRCompactorJobRunner> jobRunner =
              Optional.fromNullable(MRCompactor.this.jobRunnables.get(result.dataset()));

          switch (result.status()) {
            case PASSED:
              LOG.info("Completeness verification for dataset " + result.dataset() + " passed.");
              submitVerificationSuccessSlaEvent(result);
              result.dataset().setState(VERIFIED);
              if (jobRunner.isPresent()) {
                jobRunner.get().proceed();
              }
              break;
            case FAILED:
              if (shouldGiveUpVerification()) {
                LOG.info("Completeness verification for dataset " + result.dataset() + " has timed out.");
                submitVerificationSuccessSlaEvent(result);
                result.dataset().setState(GIVEN_UP);
                result.dataset().addThrowable(new RuntimeException(
                    String.format("Completeness verification for dataset %s failed or timed out.", result.dataset())));
              } else {
                LOG.info("Completeness verification for dataset " + result.dataset() + " failed. Will verify again.");
                datasetsToBeVerifiedAgain.add(result.dataset());
              }
              break;
            default:
              throw new IllegalStateException("Unrecognized result status: " + result.status());
          }
        }

        if (!datasetsToBeVerifiedAgain.isEmpty()) {
          ListenableFuture<Results> future2 = MRCompactor.this.verifier.get().verify(datasetsToBeVerifiedAgain);
          addCallback(datasetsToBeVerifiedAgain, future2);
        }
      }

      /**
       * On failure, resubmit verification for all {@link Dataset}s, unless timed out.
       */
      @Override
      public void onFailure(Throwable t) {
        LOG.error("Failed to verify completeness for the following datasets: " + datasetsToBeVerified, t);

        if (shouldGiveUpVerification()) {
          for (Dataset dataset : datasetsToBeVerified) {
            LOG.warn(String.format("Completeness verification for dataset %s has timed out.", dataset));
            submitFailureSlaEvent(dataset, CompactionSlaEventHelper.COMPLETION_VERIFICATION_FAILED_EVENT_NAME);
            dataset.setState(GIVEN_UP);
            dataset.addThrowable(new RuntimeException(
                String.format("Completeness verification for dataset %s failed or timed out.", dataset)));
          }
        } else {
          ListenableFuture<Results> future2 = MRCompactor.this.verifier.get().verify(datasetsToBeVerified);
          addCallback(datasetsToBeVerified, future2);
        }
      }
    });
  }

  /**
   * Get the number of {@link Dataset}s to be verified together. This allows multiple {@link Dataset}s
   * to share the same verification job, e.g., share the same query.
   */
  private int getNumDatasetsVerifiedTogether() {
    return this.state.getPropAsInt(COMPACTION_COMPLETENESS_VERIFICATION_NUM_DATASETS_VERIFIED_TOGETHER,
        DEFAULT_COMPACTION_COMPLETENESS_VERIFICATION_NUM_DATASETS_VERIFIED_TOGETHER);
  }

  private void setAllDatasetStatesToVerified() {
    for (Dataset dataset : this.datasets) {
      dataset.compareAndSetState(UNVERIFIED, VERIFIED);
    }
  }

  /**
   * Data completeness verification of a folder should give up if timed out.
   */
  private boolean shouldGiveUpVerification() {
    return this.stopwatch.elapsed(TimeUnit.MINUTES) >= this.dataVerifTimeoutMinutes;
  }

  private boolean shouldPublishDataIfCannotVerifyCompl() {
    return this.state.getPropAsBoolean(COMPACTION_COMPLETENESS_VERIFICATION_PUBLISH_DATA_IF_CANNOT_VERIFY,
        DEFAULT_COMPACTION_COMPLETENESS_VERIFICATION_PUBLISH_DATA_IF_CANNOT_VERIFY);
  }

  private void submitCompactionJobsAndWaitForCompletion() {
    LOG.info("Submitting compaction jobs. Number of datasets: " + this.datasets.size());

    boolean allDatasetsCompleted = false;
    while (!allDatasetsCompleted) {
      allDatasetsCompleted = true;
      for (Dataset dataset : this.datasets) {
        MRCompactorJobRunner jobRunner = MRCompactor.this.jobRunnables.get(dataset);

        if (dataset.state() == VERIFIED || dataset.state() == UNVERIFIED) {
          allDatasetsCompleted = false;
          // Run compaction for a dataset, if it is not already running or completed
          if (jobRunner == null || jobRunner.status() == ABORTED) {
            runCompactionForDataset(dataset, dataset.state() == VERIFIED);
          }
        } else if (dataset.state() == GIVEN_UP) {
          if (this.shouldPublishDataIfCannotVerifyCompl) {
            allDatasetsCompleted = false;
            if (jobRunner == null || jobRunner.status() == ABORTED) {
              runCompactionForDataset(dataset, true);
            } else {
              jobRunner.proceed();
            }
          } else {
            if (jobRunner != null) {
              jobRunner.abort();
            }
          }
        }
      }

      if (this.stopwatch.elapsed(TimeUnit.MINUTES) >= this.compactionTimeoutMinutes) {

        // Compaction timed out. Killing all compaction jobs running
        LOG.error("Compaction timed-out. Killing all running jobs");
        for (MRCompactorJobRunner jobRunner : MRCompactor.this.jobRunnables.values()) {
          jobRunner.abort();
        }
        break;
      }

      // Sleep for a few seconds before another round
      try {
        Thread.sleep(TimeUnit.SECONDS.toMillis(COMPACTION_JOB_WAIT_INTERVAL_SECONDS));
      } catch (InterruptedException e) {
        Thread.currentThread().interrupt();
        throw new RuntimeException("Interrupted while waiting", e);
      }
    }
  }

  /**
   * Run compaction job for a {@link Dataset}.
   *
   * @param dataset The input {@link Dataset} to run compaction for.
   * @param proceed Whether the compaction job is permitted to publish data. If data completeness verification
   * is enabled and the status of the inputFolder is UNVERIFIED, 'proceed' should be set to false.
   * Otherwise it should be set to true.
   */
  private void runCompactionForDataset(Dataset dataset, boolean proceed) {
    LOG.info("Running compaction for dataset " + dataset);

    try {
      MRCompactorJobRunner jobRunner = getMRCompactorJobRunner(dataset);
      this.jobRunnables.put(dataset, jobRunner);
      if (proceed) {
        jobRunner.proceed();
      }
      this.jobExecutor.execute(jobRunner);
    } catch (Throwable t) {
      dataset.skip(t);
    }
  }

  /**
   * Get an instance of {@link MRCompactorJobRunner}.
   */
  private MRCompactorJobRunner getMRCompactorJobRunner(Dataset dataset) {
    try {
      @SuppressWarnings("unchecked")
      Class<? extends MRCompactorJobRunner> cls = (Class<? extends MRCompactorJobRunner>) Class
          .forName(this.state.getProp(COMPACTION_JOB_RUNNER_CLASS, DEFAULT_COMPACTION_JOB_RUNNER_CLASS));
      return cls.getDeclaredConstructor(Dataset.class, FileSystem.class).newInstance(dataset, this.fs);
    } catch (Exception e) {
      throw new RuntimeException("Cannot instantiate MRCompactorJobRunner", e);
    }
  }

  /**
   * Keep track of running MR jobs, so if the compaction is cancelled, the MR jobs can be killed.
   */
  public static void addRunningHadoopJob(Dataset dataset, Job job) {
    MRCompactor.RUNNING_MR_JOBS.put(dataset, job);
  }

  private long getCompactionTimeoutMinutes() {
    return this.state.getPropAsLong(COMPACTION_MR_JOB_TIMEOUT_MINUTES, DEFAULT_COMPACTION_MR_JOB_TIMEOUT_MINUTES);
  }

  private long getDataVerifTimeoutMinutes() {
    return this.state.getPropAsLong(COMPACTION_VERIFICATION_TIMEOUT_MINUTES,
        DEFAULT_COMPACTION_VERIFICATION_TIMEOUT_MINUTES);
  }

  private void throwExceptionsIfAnyDatasetCompactionFailed() {
    Set<Dataset> datasetsWithThrowables = getDatasetsWithThrowables();
    int numDatasetsWithThrowables = 0;
    for (Dataset dataset : datasetsWithThrowables) {
      numDatasetsWithThrowables++;
      for (Throwable t : dataset.throwables()) {
        LOG.error("Error processing dataset " + dataset, t);
        submitFailureSlaEvent(dataset, CompactionSlaEventHelper.COMPACTION_FAILED_EVENT_NAME);
      }
    }
    if (numDatasetsWithThrowables > 0) {
      throw new RuntimeException(String.format("Failed to process %d datasets.", numDatasetsWithThrowables));
    }
  }

  /**
   * Return all {@link Dataset}s where a {@link Throwable} is thrown from the compaction job.
   */
  private Set<Dataset> getDatasetsWithThrowables() {
    Set<Dataset> datasetsWithThrowables = Sets.newHashSet();
    for (Dataset dataset : this.datasets) {
      if (!dataset.throwables().isEmpty()) {
        datasetsWithThrowables.add(dataset);
      }
    }
    return datasetsWithThrowables;
  }

  private void shutdownExecutors() {
    LOG.info("Shutting down Executors");
    ExecutorsUtils.shutdownExecutorService(this.jobExecutor, Optional.of(LOG));
  }

  @Override
  public void cancel() throws IOException {
    try {
      for (Map.Entry<Dataset, Job> entry : MRCompactor.RUNNING_MR_JOBS.entrySet()) {
        Job hadoopJob = entry.getValue();
        if (!hadoopJob.isComplete()) {
          LOG.info(String.format("Killing hadoop job %s for dataset %s", hadoopJob.getJobID(), entry.getKey()));
          hadoopJob.killJob();
        }
      }
    } finally {
      try {
        ExecutorsUtils.shutdownExecutorService(this.jobExecutor, Optional.of(LOG), 0, TimeUnit.NANOSECONDS);
      } finally {
        if (this.verifier.isPresent()) {
          this.verifier.get().closeNow();
        }
      }
    }
  }

  public static void modifyDatasetStateToRecompact (Dataset dataset) {
    // Modify the dataset for recompaction
    LOG.info ("{} changes to recompact mode", dataset.getDatasetName());
    State recompactState = new State();
    recompactState.setProp(MRCompactor.COMPACTION_RECOMPACT_FROM_DEST_PATHS, Boolean.TRUE);
    recompactState.setProp(MRCompactor.COMPACTION_JOB_LATE_DATA_MOVEMENT_TASK, Boolean.FALSE);
    dataset.modifyDatasetForRecompact(recompactState);
    dataset.setState(VERIFIED);
  }

  /**
   * A subclass of {@link ThreadPoolExecutor} for running compaction jobs, and performs necessary steps
   * after each compaction job finishes.
   */
  private class JobRunnerExecutor extends ThreadPoolExecutor {

    public JobRunnerExecutor(int corePoolSize, int maximumPoolSize, long keepAliveTime, TimeUnit unit,
        BlockingQueue<Runnable> workQueue) {
      super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue);
    }

    /**
     * When a compaction job for a {@link Dataset} finishes, if it successfully published the data (t == null
     * && jobRunner.status() == {@link MRCompactorJobRunner.Status#COMMITTED}, or if it
     * threw any {@link Throwable} (t != null), mark the {@link Dataset} as
     * {@link Dataset.DatasetState#COMPACTION_COMPLETE}.
     * If the job failed to publish the data because the input data was not complete, reduce the priority of
     * the {@link Dataset}. A new compaction job will be submitted later with a lower priority.
     */
    @Override
    protected void afterExecute(Runnable r, Throwable t) {
      Preconditions.checkArgument(r instanceof MRCompactorJobRunner,
          String.format("Runnable expected to be instance of %s, actual %s", MRCompactorJobRunner.class.getSimpleName(),
              r.getClass().getSimpleName()));

      MRCompactorJobRunner jobRunner = (MRCompactorJobRunner) r;
      MRCompactor.this.jobRunnables.remove(jobRunner.getDataset());
      if (t == null) {
        if (jobRunner.status() == COMMITTED) {
          if (jobRunner.getDataset().needToRecompact()) {
            modifyDatasetStateToRecompact (jobRunner.getDataset());
          } else {
            // Set the dataset status to COMPACTION_COMPLETE if compaction is successful.
            jobRunner.getDataset().setState(COMPACTION_COMPLETE);
          }
          if (MRCompactor.this.compactorListener.isPresent()) {
            try {
              MRCompactor.this.compactorListener.get().onDatasetCompactionCompletion(jobRunner.getDataset());
            } catch (Exception e) {
              t = e;
            }
          }
        } else if (jobRunner.getDataset().state() == GIVEN_UP
            && !MRCompactor.this.shouldPublishDataIfCannotVerifyCompl) {

          // Compaction job of a dataset has aborted, and data completeness verification has given up.
          // This dataset will not be compacted.
          LOG.info(String.format("Dataset %s will not be compacted, since data completeness cannot be verified",
              jobRunner.getDataset()));
          jobRunner.getDataset().setState(COMPACTION_COMPLETE);
        } else {
          // Compaction job of a dataset has aborted because data completeness is not verified.
          // Reduce priority and try again.
          jobRunner.getDataset().reducePriority();
        }
      }
      if (t != null) {
        // Compaction job of a dataset has failed with a throwable.
        afterExecuteWithThrowable(jobRunner, t);
      }
    }

    private void afterExecuteWithThrowable(MRCompactorJobRunner jobRunner, Throwable t) {
      jobRunner.getDataset().skip(t);
    }
  }

  /**
   * Submit an event when completeness verification is successful
   */
  private void submitVerificationSuccessSlaEvent(Results.Result result) {
    try {
      CompactionSlaEventHelper.getEventSubmitterBuilder(result.dataset(), Optional.<Job> absent(), this.fs)
      .eventSubmitter(this.eventSubmitter).eventName(CompactionSlaEventHelper.COMPLETION_VERIFICATION_SUCCESS_EVENT_NAME)
      .additionalMetadata(Maps.transformValues(result.verificationContext(), Functions.toStringFunction())).build()
      .submit();
    } catch (Throwable t) {
      LOG.warn("Failed to submit verification success event:" + t, t);
    }
  }

  /**
   * Submit a failure sla event
   */
  private void submitFailureSlaEvent(Dataset dataset, String eventName) {
    try {
      CompactionSlaEventHelper.getEventSubmitterBuilder(dataset, Optional.<Job> absent(), this.fs)
      .eventSubmitter(this.eventSubmitter).eventName(eventName).build().submit();
    } catch (Throwable t) {
      LOG.warn("Failed to submit failure sla event:" + t, t);
    }
  }
}