DefaultGobblinInstanceDriverImpl.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package gobblin.runtime.instance;

import java.net.URI;
import java.util.Collections;
import java.util.List;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Optional;
import com.google.common.base.Preconditions;
import com.google.common.util.concurrent.AbstractIdleService;

import gobblin.broker.gobblin_scopes.GobblinScopeTypes;
import gobblin.broker.iface.SharedResourcesBroker;
import gobblin.instrumented.Instrumented;
import gobblin.metrics.GobblinMetrics;
import gobblin.metrics.MetricContext;
import gobblin.metrics.Tag;
import gobblin.runtime.JobState.RunningState;
import gobblin.runtime.api.Configurable;
import gobblin.runtime.api.GobblinInstanceDriver;
import gobblin.runtime.api.GobblinInstanceLauncher.ConfigAccessor;
import gobblin.runtime.api.JobCatalog;
import gobblin.runtime.api.JobExecutionDriver;
import gobblin.runtime.api.JobExecutionLauncher;
import gobblin.runtime.api.JobExecutionState;
import gobblin.runtime.api.JobLifecycleListener;
import gobblin.runtime.api.JobSpec;
import gobblin.runtime.api.JobSpecMonitorFactory;
import gobblin.runtime.api.JobSpecScheduler;
import gobblin.runtime.api.MutableJobCatalog;
import gobblin.runtime.job_spec.ResolvedJobSpec;
import gobblin.runtime.std.DefaultJobCatalogListenerImpl;
import gobblin.runtime.std.DefaultJobExecutionStateListenerImpl;
import gobblin.runtime.std.JobLifecycleListenersList;
import gobblin.util.ExecutorsUtils;


/**
 * A default implementation of {@link GobblinInstanceDriver}. It accepts already instantiated
 * {@link JobCatalog}, {@link JobSpecMonitorFactory}, {@link JobSpecScheduler},
 * {@link JobExecutionLauncher}. It is responsibility of the caller to manage those (e.g. start,
 * shutdown, etc.)
 *
 */
public class DefaultGobblinInstanceDriverImpl extends AbstractIdleService
       implements GobblinInstanceDriver {
  protected final Logger _log;
  protected final String _instanceName;
  protected final Configurable _sysConfig;
  protected final JobCatalog _jobCatalog;
  protected final JobSpecScheduler _jobScheduler;
  protected final JobExecutionLauncher _jobLauncher;
  protected final ConfigAccessor _instanceCfg;
  protected final JobLifecycleListenersList _callbacksDispatcher;
  private final boolean _instrumentationEnabled;
  protected final MetricContext _metricCtx;
  protected JobSpecListener _jobSpecListener;
  private final StandardMetrics _metrics;
  private final SharedResourcesBroker<GobblinScopeTypes> _instanceBroker;

  public DefaultGobblinInstanceDriverImpl(String instanceName,
      Configurable sysConfig, JobCatalog jobCatalog,
      JobSpecScheduler jobScheduler,
      JobExecutionLauncher jobLauncher,
      Optional<MetricContext> baseMetricContext,
      Optional<Logger> log,
      SharedResourcesBroker<GobblinScopeTypes> instanceBroker) {
    Preconditions.checkNotNull(jobCatalog);
    Preconditions.checkNotNull(jobScheduler);
    Preconditions.checkNotNull(jobLauncher);
    Preconditions.checkNotNull(sysConfig);

    _instanceName = instanceName;
    _log = log.or(LoggerFactory.getLogger(getClass()));
    _metricCtx = baseMetricContext.or(constructMetricContext(sysConfig, _log));
    _instrumentationEnabled = null != _metricCtx && GobblinMetrics.isEnabled(sysConfig.getConfig());
    _jobCatalog = jobCatalog;
    _jobScheduler = jobScheduler;
    _jobLauncher = jobLauncher;
    _sysConfig = sysConfig;
    _instanceCfg = ConfigAccessor.createFromGlobalConfig(_sysConfig.getConfig());
    _callbacksDispatcher = new JobLifecycleListenersList(_jobCatalog, _jobScheduler, _log);
    _instanceBroker = instanceBroker;

    _metrics = new StandardMetrics(this);
  }

  private MetricContext constructMetricContext(Configurable sysConfig, Logger log) {
    gobblin.configuration.State tmpState = new gobblin.configuration.State(sysConfig.getConfigAsProperties());
    return GobblinMetrics.isEnabled(sysConfig.getConfig()) ?
          Instrumented.getMetricContext(tmpState, getClass())
          : null;
  }

  /** {@inheritDoc} */
  @Override public JobCatalog getJobCatalog() {
    return _jobCatalog;
  }

  /** {@inheritDoc} */
  @Override public MutableJobCatalog getMutableJobCatalog() {
    return (MutableJobCatalog)_jobCatalog;
  }

  /** {@inheritDoc} */
  @Override public JobSpecScheduler getJobScheduler() {
    return _jobScheduler;
  }

  /** {@inheritDoc} */
  @Override public JobExecutionLauncher getJobLauncher() {
    return _jobLauncher;
  }

  /** {@inheritDoc} */
  @Override public Configurable getSysConfig() {
    return _sysConfig;
  }

  /** {@inheritDoc} */
  @Override public SharedResourcesBroker<GobblinScopeTypes> getInstanceBroker() {
    return _instanceBroker;
  }

  /** {@inheritDoc} */
  @Override public Logger getLog() {
    return _log;
  }

  @Override protected void startUp() throws Exception {
    getLog().info("Default driver: starting ...");
    _jobSpecListener = new JobSpecListener();
    _jobCatalog.addListener(_jobSpecListener);
    getLog().info("Default driver: started.");
  }

  @Override protected void shutDown() throws Exception {
    getLog().info("Default driver: shuttind down ...");
    if (null != _jobSpecListener) {
      _jobCatalog.removeListener(_jobSpecListener);
    }
    _callbacksDispatcher.close();
    getLog().info("Default driver: shut down.");
  }

  /** Keeps track of a job execution */
  class JobStateTracker extends DefaultJobExecutionStateListenerImpl {

    public JobStateTracker() {
      super(LoggerFactory.getLogger(DefaultGobblinInstanceDriverImpl.this._log.getName() +
                                  "_jobExecutionListener"));
    }

    @Override public String toString() {
      return _log.get().getName();
    }

    @Override public void onStatusChange(JobExecutionState state, RunningState previousStatus, RunningState newStatus) {
      super.onStatusChange(state, previousStatus, newStatus);
      _callbacksDispatcher.onStatusChange(state, previousStatus, newStatus);
    }

    @Override
    public void onStageTransition(JobExecutionState state, String previousStage, String newStage) {
      super.onStageTransition(state, previousStage, newStage);
      _callbacksDispatcher.onStageTransition(state, previousStage, newStage);
    }

    @Override
    public void onMetadataChange(JobExecutionState state, String key, Object oldValue, Object newValue) {
      super.onMetadataChange(state, key, oldValue, newValue);
      _callbacksDispatcher.onMetadataChange(state, key, oldValue, newValue);
    }

  }

  /** The runnable invoked by the Job scheduler */
  class JobSpecRunnable implements Runnable {
    private final JobSpec _jobSpec;
    private final GobblinInstanceDriver _instanceDriver;

    public JobSpecRunnable(JobSpec jobSpec, GobblinInstanceDriver instanceDriver) {
      _jobSpec = jobSpec;
      _instanceDriver = instanceDriver;
    }

    @Override
    public void run() {
      try {
         JobExecutionDriver driver = _jobLauncher.launchJob(new ResolvedJobSpec(_jobSpec, _instanceDriver));
         _callbacksDispatcher.onJobLaunch(driver);
         driver.registerStateListener(new JobStateTracker());
        ExecutorsUtils.newThreadFactory(Optional.of(_log), Optional.of("gobblin-instance-driver")).newThread(driver).start();
      }
      catch (Throwable t) {
        _log.error("Job launch failed: " + t, t);
      }
    }
  }

  /** Listens to changes in the Job catalog and schedules/un-schedules jobs. */
  protected class JobSpecListener extends DefaultJobCatalogListenerImpl {
    public JobSpecListener() {
      super(LoggerFactory.getLogger(DefaultGobblinInstanceDriverImpl.this._log.getName() +
                                  "_jobSpecListener"));
    }

    @Override public String toString() {
      return _log.get().getName();
    }

    @Override public void onAddJob(JobSpec addedJob) {
      super.onAddJob(addedJob);
      _jobScheduler.scheduleJob(addedJob, createJobSpecRunnable(addedJob));
    }

    @Override public void onDeleteJob(URI deletedJobURI, String deletedJobVersion) {
      super.onDeleteJob(deletedJobURI, deletedJobVersion);
      _jobScheduler.unscheduleJob(deletedJobURI);
    }

    @Override public void onUpdateJob(JobSpec updatedJob) {
      super.onUpdateJob(updatedJob);
      _jobScheduler.scheduleJob(updatedJob, createJobSpecRunnable(updatedJob));
    }
  }

  @VisibleForTesting JobSpecRunnable createJobSpecRunnable(JobSpec addedJob) {
    return new JobSpecRunnable(addedJob, this);
  }

  ConfigAccessor getInstanceCfg() {
    return _instanceCfg;
  }

  @Override
  public void registerJobLifecycleListener(JobLifecycleListener listener) {
    _callbacksDispatcher.registerJobLifecycleListener(listener);
  }

  @Override
  public void unregisterJobLifecycleListener(JobLifecycleListener listener) {
    _callbacksDispatcher.unregisterJobLifecycleListener(listener);
  }

  @Override
  public List<JobLifecycleListener> getJobLifecycleListeners() {
    return _callbacksDispatcher.getJobLifecycleListeners();
  }

  @Override
  public void registerWeakJobLifecycleListener(JobLifecycleListener listener) {
    _callbacksDispatcher.registerWeakJobLifecycleListener(listener);
  }

  @Override public MetricContext getMetricContext() {
    return _metricCtx;
  }

  @Override public boolean isInstrumentationEnabled() {
    return _instrumentationEnabled;
  }

  @Override public List<Tag<?>> generateTags(gobblin.configuration.State state) {
    return Collections.emptyList();
  }

  @Override public void switchMetricContext(List<Tag<?>> tags) {
    throw new UnsupportedOperationException();
  }

  @Override
  public void switchMetricContext(MetricContext context) {
    throw new UnsupportedOperationException();
  }

  @Override public StandardMetrics getMetrics() {
    return _metrics;
  }

  @Override
  public String getInstanceName() {
    return _instanceName;
  }

}