TestExternalTezServicesErrors.java

/*
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.tez.tests;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;

import java.io.IOException;
import java.util.EnumSet;
import java.util.List;

import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.tez.client.TezClient;
import org.apache.tez.common.TezUtils;
import org.apache.tez.dag.api.DAG;
import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.api.TezException;
import org.apache.tez.dag.api.UserPayload;
import org.apache.tez.dag.api.Vertex;
import org.apache.tez.dag.api.client.DAGClient;
import org.apache.tez.dag.api.client.DAGStatus;
import org.apache.tez.dag.api.client.StatusGetOpts;
import org.apache.tez.dag.app.ErrorPluginConfiguration;
import org.apache.tez.dag.app.dag.event.DAGAppMasterEventType;
import org.apache.tez.dag.app.launcher.TezTestServiceContainerLauncherWithErrors;
import org.apache.tez.dag.app.launcher.TezTestServiceNoOpContainerLauncher;
import org.apache.tez.dag.app.rm.TezTestServiceTaskSchedulerService;
import org.apache.tez.dag.app.rm.TezTestServiceTaskSchedulerServiceWithErrors;
import org.apache.tez.dag.app.taskcomm.TezTestServiceTaskCommunicatorImpl;
import org.apache.tez.dag.app.taskcomm.TezTestServiceTaskCommunicatorWithErrors;
import org.apache.tez.examples.JoinValidateConfigured;
import org.apache.tez.serviceplugins.api.ContainerLauncherDescriptor;
import org.apache.tez.serviceplugins.api.ServicePluginErrorDefaults;
import org.apache.tez.serviceplugins.api.ServicePluginsDescriptor;
import org.apache.tez.serviceplugins.api.TaskCommunicatorDescriptor;
import org.apache.tez.serviceplugins.api.TaskSchedulerDescriptor;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class TestExternalTezServicesErrors {

  private static final Logger LOG = LoggerFactory.getLogger(TestExternalTezServicesErrors.class);

  private static final String EXT_PUSH_ENTITY_NAME = "ExtServiceTestPush";
  private static final String EXT_THROW_ERROR_ENTITY_NAME = "ExtServiceTestThrowErrors";
  private static final String EXT_REPORT_NON_FATAL_ERROR_ENTITY_NAME = "ExtServiceTestReportNonFatalErrors";
  private static final String EXT_REPORT_FATAL_ERROR_ENTITY_NAME = "ExtServiceTestReportFatalErrors";

  private static final String SUFFIX_LAUNCHER = "ContainerLauncher";
  private static final String SUFFIX_TASKCOMM = "TaskCommunicator";
  private static final String SUFFIX_SCHEDULER = "TaskScheduler";

  private static ExternalTezServiceTestHelper extServiceTestHelper;

  private static ServicePluginsDescriptor servicePluginsDescriptor;

  private static final Path SRC_DATA_DIR = new Path("/tmp/" + TestExternalTezServicesErrors.class.getSimpleName());
  private static final Path HASH_JOIN_EXPECTED_RESULT_PATH = new Path(SRC_DATA_DIR, "expectedOutputPath");
  private static final Path HASH_JOIN_OUTPUT_PATH = new Path(SRC_DATA_DIR, "outPath");

  private static final Vertex.VertexExecutionContext EXECUTION_CONTEXT_EXT_SERVICE_PUSH =
      Vertex.VertexExecutionContext.create(
          EXT_PUSH_ENTITY_NAME, EXT_PUSH_ENTITY_NAME, EXT_PUSH_ENTITY_NAME);
  // Throw error contexts
  private static final Vertex.VertexExecutionContext EXECUTION_CONTEXT_LAUNCHER_THROW =
      Vertex.VertexExecutionContext.create(EXT_PUSH_ENTITY_NAME, EXT_THROW_ERROR_ENTITY_NAME, EXT_PUSH_ENTITY_NAME);
  private static final Vertex.VertexExecutionContext EXECUTION_CONTEXT_TASKCOMM_THROW =
      Vertex.VertexExecutionContext.create(EXT_PUSH_ENTITY_NAME, EXT_PUSH_ENTITY_NAME,
          EXT_THROW_ERROR_ENTITY_NAME);
  private static final Vertex.VertexExecutionContext EXECUTION_CONTEXT_SCHEDULER_THROW =
      Vertex.VertexExecutionContext.create(EXT_THROW_ERROR_ENTITY_NAME, EXT_PUSH_ENTITY_NAME, EXT_PUSH_ENTITY_NAME);

  // Report-non-fatal contexts
  private static final Vertex.VertexExecutionContext EXECUTION_CONTEXT_LAUNCHER_REPORT_NON_FATAL =
      Vertex.VertexExecutionContext.create(EXT_PUSH_ENTITY_NAME, EXT_REPORT_NON_FATAL_ERROR_ENTITY_NAME, EXT_PUSH_ENTITY_NAME);
  private static final Vertex.VertexExecutionContext EXECUTION_CONTEXT_TASKCOMM_REPORT_NON_FATAL =
      Vertex.VertexExecutionContext.create(EXT_PUSH_ENTITY_NAME, EXT_PUSH_ENTITY_NAME,
          EXT_REPORT_NON_FATAL_ERROR_ENTITY_NAME);
  private static final Vertex.VertexExecutionContext EXECUTION_CONTEXT_SCHEDULER_REPORT_NON_FATAL =
      Vertex.VertexExecutionContext.create(EXT_REPORT_NON_FATAL_ERROR_ENTITY_NAME, EXT_PUSH_ENTITY_NAME, EXT_PUSH_ENTITY_NAME);

  // Report fatal contexts
  private static final Vertex.VertexExecutionContext EXECUTION_CONTEXT_LAUNCHER_REPORT_FATAL =
      Vertex.VertexExecutionContext.create(EXT_PUSH_ENTITY_NAME, EXT_REPORT_FATAL_ERROR_ENTITY_NAME, EXT_PUSH_ENTITY_NAME);
  private static final Vertex.VertexExecutionContext EXECUTION_CONTEXT_TASKCOMM_REPORT_FATAL =
      Vertex.VertexExecutionContext.create(EXT_PUSH_ENTITY_NAME, EXT_PUSH_ENTITY_NAME,
          EXT_REPORT_FATAL_ERROR_ENTITY_NAME);
  private static final Vertex.VertexExecutionContext EXECUTION_CONTEXT_SCHEDULER_REPORT_FATAL =
      Vertex.VertexExecutionContext.create(EXT_REPORT_FATAL_ERROR_ENTITY_NAME, EXT_PUSH_ENTITY_NAME, EXT_PUSH_ENTITY_NAME);


  private static final Vertex.VertexExecutionContext EXECUTION_CONTEXT_DEFAULT = EXECUTION_CONTEXT_EXT_SERVICE_PUSH;

  private static String TEST_ROOT_DIR = "target" + Path.SEPARATOR + TestExternalTezServicesErrors.class.getName()
      + "-tmpDir";

  @BeforeClass
  public static void setup() throws Exception {

    extServiceTestHelper = new ExternalTezServiceTestHelper(TEST_ROOT_DIR);
    UserPayload userPayload =
        TezUtils.createUserPayloadFromConf(extServiceTestHelper.getConfForJobs());
    UserPayload userPayloadThrowError =
        ErrorPluginConfiguration.toUserPayload(ErrorPluginConfiguration.createThrowErrorConf());

    UserPayload userPayloadReportFatalErrorLauncher = ErrorPluginConfiguration
        .toUserPayload(ErrorPluginConfiguration.createReportFatalErrorConf(SUFFIX_LAUNCHER));
    UserPayload userPayloadReportFatalErrorTaskComm = ErrorPluginConfiguration
        .toUserPayload(ErrorPluginConfiguration.createReportFatalErrorConf(SUFFIX_TASKCOMM));
    UserPayload userPayloadReportFatalErrorScheduler = ErrorPluginConfiguration
        .toUserPayload(ErrorPluginConfiguration.createReportFatalErrorConf(SUFFIX_SCHEDULER));

    UserPayload userPayloadReportNonFatalErrorLauncher = ErrorPluginConfiguration
        .toUserPayload(ErrorPluginConfiguration.createReportNonFatalErrorConf(SUFFIX_LAUNCHER));
    UserPayload userPayloadReportNonFatalErrorTaskComm = ErrorPluginConfiguration
        .toUserPayload(ErrorPluginConfiguration.createReportNonFatalErrorConf(SUFFIX_TASKCOMM));
    UserPayload userPayloadReportNonFatalErrorScheduler = ErrorPluginConfiguration
        .toUserPayload(ErrorPluginConfiguration.createReportNonFatalErrorConf(SUFFIX_SCHEDULER));

    TaskSchedulerDescriptor[] taskSchedulerDescriptors = new TaskSchedulerDescriptor[]{
        TaskSchedulerDescriptor
            .create(EXT_PUSH_ENTITY_NAME, TezTestServiceTaskSchedulerService.class.getName())
            .setUserPayload(userPayload),
        TaskSchedulerDescriptor.create(EXT_THROW_ERROR_ENTITY_NAME,
            TezTestServiceTaskSchedulerServiceWithErrors.class.getName()).setUserPayload(
            userPayloadThrowError),
        TaskSchedulerDescriptor.create(EXT_REPORT_FATAL_ERROR_ENTITY_NAME,
            TezTestServiceTaskSchedulerServiceWithErrors.class.getName()).setUserPayload(
            userPayloadReportFatalErrorScheduler),
        TaskSchedulerDescriptor.create(EXT_REPORT_NON_FATAL_ERROR_ENTITY_NAME,
            TezTestServiceTaskSchedulerServiceWithErrors.class.getName()).setUserPayload(
            userPayloadReportNonFatalErrorScheduler),
    };

    ContainerLauncherDescriptor[] containerLauncherDescriptors = new ContainerLauncherDescriptor[]{
        ContainerLauncherDescriptor
            .create(EXT_PUSH_ENTITY_NAME, TezTestServiceNoOpContainerLauncher.class.getName())
            .setUserPayload(userPayload),
        ContainerLauncherDescriptor.create(EXT_THROW_ERROR_ENTITY_NAME,
            TezTestServiceContainerLauncherWithErrors.class.getName()).setUserPayload(userPayloadThrowError),
        ContainerLauncherDescriptor.create(EXT_REPORT_FATAL_ERROR_ENTITY_NAME,
            TezTestServiceContainerLauncherWithErrors.class.getName()).setUserPayload(userPayloadReportFatalErrorLauncher),
        ContainerLauncherDescriptor.create(EXT_REPORT_NON_FATAL_ERROR_ENTITY_NAME,
            TezTestServiceContainerLauncherWithErrors.class.getName()).setUserPayload(userPayloadReportNonFatalErrorLauncher)
    };

    TaskCommunicatorDescriptor[] taskCommunicatorDescriptors = new TaskCommunicatorDescriptor[]{
        TaskCommunicatorDescriptor
            .create(EXT_PUSH_ENTITY_NAME, TezTestServiceTaskCommunicatorImpl.class.getName())
            .setUserPayload(userPayload),
        TaskCommunicatorDescriptor.create(EXT_THROW_ERROR_ENTITY_NAME,
            TezTestServiceTaskCommunicatorWithErrors.class.getName()).setUserPayload(userPayloadThrowError),
        TaskCommunicatorDescriptor.create(EXT_REPORT_FATAL_ERROR_ENTITY_NAME,
            TezTestServiceTaskCommunicatorWithErrors.class.getName()).setUserPayload(userPayloadReportFatalErrorTaskComm),
        TaskCommunicatorDescriptor.create(EXT_REPORT_NON_FATAL_ERROR_ENTITY_NAME,
            TezTestServiceTaskCommunicatorWithErrors.class.getName()).setUserPayload(userPayloadReportNonFatalErrorTaskComm)
    };

    servicePluginsDescriptor = ServicePluginsDescriptor.create(true, true,
        taskSchedulerDescriptors, containerLauncherDescriptors, taskCommunicatorDescriptors);

    extServiceTestHelper.setupSharedTezClient(servicePluginsDescriptor);

    // Generate the join data set used for each run.
    // Can a timeout be enforced here ?
    Path dataPath1 = new Path(SRC_DATA_DIR, "inPath1");
    Path dataPath2 = new Path(SRC_DATA_DIR, "inPath2");
    extServiceTestHelper
        .setupHashJoinData(SRC_DATA_DIR, dataPath1, dataPath2, HASH_JOIN_EXPECTED_RESULT_PATH, HASH_JOIN_OUTPUT_PATH);

    extServiceTestHelper.shutdownSharedTezClient();
  }

  @AfterClass
  public static void tearDown() throws IOException, TezException {
    extServiceTestHelper.tearDownAll();
  }

  @Test(timeout = 90000)
  public void testContainerLauncherThrowError() throws Exception {
    testFatalError("_testContainerLauncherError_", EXECUTION_CONTEXT_LAUNCHER_THROW,
        SUFFIX_LAUNCHER, Lists.newArrayList("Service Error",
            DAGAppMasterEventType.CONTAINER_LAUNCHER_SERVICE_FATAL_ERROR.name()));
  }

  @Test(timeout = 90000)
  public void testTaskCommunicatorThrowError() throws Exception {
    testFatalError("_testContainerLauncherError_", EXECUTION_CONTEXT_TASKCOMM_THROW,
        SUFFIX_TASKCOMM, Lists.newArrayList("Service Error",
            DAGAppMasterEventType.TASK_COMMUNICATOR_SERVICE_FATAL_ERROR.name()));
  }

  @Test(timeout = 90000)
  public void testTaskSchedulerThrowError() throws Exception {
    testFatalError("_testContainerLauncherError_", EXECUTION_CONTEXT_SCHEDULER_THROW,
        SUFFIX_SCHEDULER, Lists.newArrayList("Service Error",
            DAGAppMasterEventType.TASK_SCHEDULER_SERVICE_FATAL_ERROR.name()));
  }

  @Test (timeout = 150000)
  public void testNonFatalErrors() throws IOException, TezException, InterruptedException {
    String methodName = "testNonFatalErrors";
    TezConfiguration tezClientConf = new TezConfiguration(extServiceTestHelper.getConfForJobs());
    TezClient tezClient = TezClient
        .newBuilder(TestExternalTezServicesErrors.class.getSimpleName() + methodName + "_session",
            tezClientConf)
        .setIsSession(true).setServicePluginDescriptor(servicePluginsDescriptor).build();
    try {
      tezClient.start();
      LOG.info("TezSessionStarted for " + methodName);
      tezClient.waitTillReady();
      LOG.info("TezSession ready for submission for " + methodName);


      runAndVerifyForNonFatalErrors(tezClient, SUFFIX_LAUNCHER, EXECUTION_CONTEXT_LAUNCHER_REPORT_NON_FATAL);
      runAndVerifyForNonFatalErrors(tezClient, SUFFIX_TASKCOMM, EXECUTION_CONTEXT_TASKCOMM_REPORT_NON_FATAL);
      runAndVerifyForNonFatalErrors(tezClient, SUFFIX_SCHEDULER, EXECUTION_CONTEXT_SCHEDULER_REPORT_NON_FATAL);

    } finally {
      tezClient.stop();
    }
  }

  @Test(timeout = 90000)
  public void testContainerLauncherReportFatalError() throws Exception {
    testFatalError("_testContainerLauncherReportFatalError_",
        EXECUTION_CONTEXT_LAUNCHER_REPORT_FATAL, SUFFIX_LAUNCHER, Lists
            .newArrayList(ErrorPluginConfiguration.REPORT_FATAL_ERROR_MESSAGE,
                ServicePluginErrorDefaults.INCONSISTENT_STATE.name()));
  }

  @Test(timeout = 90000)
  public void testTaskCommReportFatalError() throws Exception {
    testFatalError("_testTaskCommReportFatalError_", EXECUTION_CONTEXT_TASKCOMM_REPORT_FATAL,
        SUFFIX_TASKCOMM, Lists.newArrayList(ErrorPluginConfiguration.REPORT_FATAL_ERROR_MESSAGE,
            ServicePluginErrorDefaults.INCONSISTENT_STATE.name()));
  }

  @Test(timeout = 90000)
  public void testTaskSchedulerReportFatalError() throws Exception {
    testFatalError("_testTaskSchedulerReportFatalError_",
        EXECUTION_CONTEXT_SCHEDULER_REPORT_FATAL, SUFFIX_SCHEDULER,
        Lists.newArrayList(ErrorPluginConfiguration.REPORT_FATAL_ERROR_MESSAGE,
            ServicePluginErrorDefaults.INCONSISTENT_STATE.name()));
  }


  private void testFatalError(String methodName,
                              Vertex.VertexExecutionContext lhsExecutionContext,
                              String dagNameSuffix, List<String> expectedDiagMessages) throws
      IOException, TezException, YarnException, InterruptedException {
    TezConfiguration tezClientConf = new TezConfiguration(extServiceTestHelper.getConfForJobs());
    TezClient tezClient = TezClient
        .newBuilder(TestExternalTezServicesErrors.class.getSimpleName() + methodName + "_session",
            tezClientConf)
        .setIsSession(true).setServicePluginDescriptor(servicePluginsDescriptor).build();

    ApplicationId appId= null;
    try {
      tezClient.start();
      LOG.info("TezSessionStarted for " + methodName);
      tezClient.waitTillReady();
      LOG.info("TezSession ready for submission for " + methodName);

      JoinValidateConfigured joinValidate =
          new JoinValidateConfigured(EXECUTION_CONTEXT_DEFAULT, lhsExecutionContext,
              EXECUTION_CONTEXT_EXT_SERVICE_PUSH,
              EXECUTION_CONTEXT_EXT_SERVICE_PUSH, dagNameSuffix);

      DAG dag = joinValidate
          .createDag(new TezConfiguration(extServiceTestHelper.getConfForJobs()),
              HASH_JOIN_EXPECTED_RESULT_PATH,
              HASH_JOIN_OUTPUT_PATH, 3);

      DAGClient dagClient = tezClient.submitDAG(dag);

      DAGStatus dagStatus =
          dagClient.waitForCompletionWithStatusUpdates(Sets.newHashSet(StatusGetOpts.GET_COUNTERS));
      assertEquals(DAGStatus.State.ERROR, dagStatus.getState());
      boolean foundDiag = false;
      for (String diag : dagStatus.getDiagnostics()) {
        foundDiag = checkDiag(diag, expectedDiagMessages);
        if (foundDiag) {
          break;
        }
      }
      appId = tezClient.getAppMasterApplicationId();
      assertTrue(foundDiag);
    } catch (InterruptedException e) {
      e.printStackTrace();
    } finally {
      tezClient.stop();
    }
    // Verify the state of the application.
    if (appId != null) {
      YarnClient yarnClient = YarnClient.createYarnClient();
      try {
        yarnClient.init(tezClientConf);
        yarnClient.start();

        ApplicationReport appReport = yarnClient.getApplicationReport(appId);
        YarnApplicationState appState = appReport.getYarnApplicationState();
        while (!EnumSet.of(YarnApplicationState.FINISHED, YarnApplicationState.FAILED,
            YarnApplicationState.KILLED).contains(appState)) {
          Thread.sleep(200L);
          appReport = yarnClient.getApplicationReport(appId);
          appState = appReport.getYarnApplicationState();
        }

        // TODO Workaround for YARN-4554. AppReport does not provide diagnostics - need to fetch them from ApplicationAttemptReport
        ApplicationAttemptId appAttemptId = appReport.getCurrentApplicationAttemptId();
        ApplicationAttemptReport appAttemptReport =
            yarnClient.getApplicationAttemptReport(appAttemptId);
        String diag = appAttemptReport.getDiagnostics();
        assertEquals(FinalApplicationStatus.FAILED, appReport.getFinalApplicationStatus());
        assertEquals(YarnApplicationState.FINISHED, appReport.getYarnApplicationState());
        checkDiag(diag, expectedDiagMessages);
      } finally {
        yarnClient.stop();
      }
    }
  }

  private boolean checkDiag(String diag, List<String> expected) {
    boolean found = true;
    for (String exp : expected) {
      if (diag.contains(exp)) {
        found = true;
        continue;
      } else {
        found = false;
        break;
      }
    }
    return found;
  }


  private void runAndVerifyForNonFatalErrors(TezClient tezClient, String componentName,
                                             Vertex.VertexExecutionContext lhsContext) throws
      TezException,
      InterruptedException, IOException {
    LOG.info("Running JoinValidate with componentName reportNonFatalException");
    JoinValidateConfigured joinValidate =
        new JoinValidateConfigured(EXECUTION_CONTEXT_DEFAULT, lhsContext,
            EXECUTION_CONTEXT_EXT_SERVICE_PUSH,
            EXECUTION_CONTEXT_EXT_SERVICE_PUSH, componentName);

    DAG dag = joinValidate
        .createDag(new TezConfiguration(extServiceTestHelper.getConfForJobs()),
            HASH_JOIN_EXPECTED_RESULT_PATH,
            HASH_JOIN_OUTPUT_PATH, 3);

    DAGClient dagClient = tezClient.submitDAG(dag);

    DAGStatus dagStatus =
        dagClient.waitForCompletionWithStatusUpdates(Sets.newHashSet(StatusGetOpts.GET_COUNTERS));
    assertEquals(DAGStatus.State.FAILED, dagStatus.getState());

    boolean foundDiag = false;
    for (String diag : dagStatus.getDiagnostics()) {
      if (diag.contains(ErrorPluginConfiguration.REPORT_NONFATAL_ERROR_MESSAGE) &&
          diag.contains(ServicePluginErrorDefaults.SERVICE_UNAVAILABLE.name())) {
        foundDiag = true;
        break;
      }
    }
    assertTrue(foundDiag);
  }

}