/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.tez.mapreduce.hadoop;
import java.io.File;
import java.util.HashMap;
import java.util.Map;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.tez.dag.api.TezConstants;
import org.apache.tez.runtime.library.api.TezRuntimeConfiguration;
import org.junit.Assert;
import org.junit.Test;
public class TestMRHelpers {
private Configuration createConfForJavaOptsTest() {
Configuration conf = new Configuration(false);
conf.set(MRJobConfig.MAPRED_MAP_ADMIN_JAVA_OPTS, "fooMapAdminOpts");
conf.set(MRJobConfig.MAP_JAVA_OPTS, "fooMapJavaOpts");
conf.set(MRJobConfig.MAP_LOG_LEVEL, "FATAL");
conf.set(MRJobConfig.MAPRED_REDUCE_ADMIN_JAVA_OPTS, "fooReduceAdminOpts");
conf.set(MRJobConfig.REDUCE_JAVA_OPTS, "fooReduceJavaOpts");
conf.set(MRJobConfig.REDUCE_LOG_LEVEL, "TRACE");
return conf;
}
@Test(timeout = 5000)
public void testMapJavaOptions() {
Configuration conf = createConfForJavaOptsTest();
String opts = MRHelpers.getJavaOptsForMRMapper(conf);
Assert.assertTrue(opts.contains("fooMapAdminOpts"));
Assert.assertTrue(opts.contains(" fooMapJavaOpts "));
Assert.assertFalse(opts.contains("fooReduceAdminOpts "));
Assert.assertFalse(opts.contains(" fooReduceJavaOpts "));
Assert.assertTrue(opts.indexOf("fooMapAdminOpts")
< opts.indexOf("fooMapJavaOpts"));
Assert.assertTrue(opts.contains(" -D"
+ TezConstants.TEZ_ROOT_LOGGER_NAME + "=FATAL"));
Assert.assertFalse(opts.contains(" -D"
+ TezConstants.TEZ_ROOT_LOGGER_NAME + "=TRACE"));
}
@Test(timeout = 5000)
public void testReduceJavaOptions() {
Configuration conf = createConfForJavaOptsTest();
String opts = MRHelpers.getJavaOptsForMRReducer(conf);
Assert.assertFalse(opts.contains("fooMapAdminOpts"));
Assert.assertFalse(opts.contains(" fooMapJavaOpts "));
Assert.assertTrue(opts.contains("fooReduceAdminOpts"));
Assert.assertTrue(opts.contains(" fooReduceJavaOpts "));
Assert.assertTrue(opts.indexOf("fooReduceAdminOpts")
< opts.indexOf("fooReduceJavaOpts"));
Assert.assertFalse(opts.contains(" -D"
+ TezConstants.TEZ_ROOT_LOGGER_NAME + "=FATAL"));
Assert.assertTrue(opts.contains(" -D"
+ TezConstants.TEZ_ROOT_LOGGER_NAME + "=TRACE"));
}
@Test(timeout = 5000)
public void testContainerResourceConstruction() {
JobConf conf = new JobConf(new Configuration());
Resource mapResource = MRHelpers.getResourceForMRMapper(conf);
Resource reduceResource = MRHelpers.getResourceForMRReducer(conf);
Assert.assertEquals(MRJobConfig.DEFAULT_MAP_CPU_VCORES,
mapResource.getVirtualCores());
Assert.assertEquals(MRJobConfig.DEFAULT_MAP_MEMORY_MB,
mapResource.getMemory());
Assert.assertEquals(MRJobConfig.DEFAULT_REDUCE_CPU_VCORES,
reduceResource.getVirtualCores());
Assert.assertEquals(MRJobConfig.DEFAULT_REDUCE_MEMORY_MB,
reduceResource.getMemory());
conf.setInt(MRJobConfig.MAP_CPU_VCORES, 2);
conf.setInt(MRJobConfig.MAP_MEMORY_MB, 123);
conf.setInt(MRJobConfig.REDUCE_CPU_VCORES, 20);
conf.setInt(MRJobConfig.REDUCE_MEMORY_MB, 1234);
mapResource = MRHelpers.getResourceForMRMapper(conf);
reduceResource = MRHelpers.getResourceForMRReducer(conf);
Assert.assertEquals(2, mapResource.getVirtualCores());
Assert.assertEquals(123, mapResource.getMemory());
Assert.assertEquals(20, reduceResource.getVirtualCores());
Assert.assertEquals(1234, reduceResource.getMemory());
}
private Configuration setupConfigForMREnvTest() {
JobConf conf = new JobConf(new Configuration());
conf.set(MRJobConfig.MAP_ENV, "foo=map1,bar=map2");
conf.set(MRJobConfig.REDUCE_ENV, "foo=red1,bar=red2");
conf.set(MRJobConfig.MAP_LOG_LEVEL, "TRACE");
conf.set(MRJobConfig.REDUCE_LOG_LEVEL, "FATAL");
final String mapredAdminUserEnv = Shell.WINDOWS ?
"PATH=%PATH%" + File.pathSeparator + "%TEZ_ADMIN_ENV%\\bin":
"LD_LIBRARY_PATH=$TEZ_ADMIN_ENV_TEST/lib/native";
conf.set(MRJobConfig.MAPRED_ADMIN_USER_ENV, mapredAdminUserEnv);
return conf;
}
private void testCommonEnvSettingsForMRTasks(Map<String, String> env) {
Assert.assertTrue(env.containsKey("foo"));
Assert.assertTrue(env.containsKey("bar"));
Assert.assertTrue(env.containsKey(Environment.LD_LIBRARY_PATH.name()));
Assert.assertTrue(env.containsKey(Environment.SHELL.name()));
Assert.assertTrue(env.containsKey("HADOOP_ROOT_LOGGER"));
/* On non-windows platform ensure that LD_LIBRARY_PATH is being set and PWD is present.
* on windows platform LD_LIBRARY_PATH is not applicable. check the PATH is being appended
* by the user setting (ex user may set HADOOP_HOME\\bin.
*/
if (!Shell.WINDOWS) {
Assert.assertEquals("$PWD:$TEZ_ADMIN_ENV_TEST/lib/native",
env.get(Environment.LD_LIBRARY_PATH.name()));
} else {
Assert.assertTrue(env.get(Environment.PATH.name()).contains(";%TEZ_ADMIN_ENV%\\bin"));
}
// TEZ-273 will reinstate this or similar.
// for (String val : YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH) {
// Assert.assertTrue(env.get(Environment.CLASSPATH.name()).contains(val));
// }
// Assert.assertTrue(0 ==
// env.get(Environment.CLASSPATH.name()).indexOf(Environment.PWD.$()));
}
@Test(timeout = 5000)
public void testMREnvSetupForMap() {
Configuration conf = setupConfigForMREnvTest();
Map<String, String> env = new HashMap<String, String>();
MRHelpers.updateEnvBasedOnMRTaskEnv(conf, env, true);
testCommonEnvSettingsForMRTasks(env);
Assert.assertEquals("map1", env.get("foo"));
Assert.assertEquals("map2", env.get("bar"));
}
@Test(timeout = 5000)
public void testMREnvSetupForReduce() {
Configuration conf = setupConfigForMREnvTest();
Map<String, String> env = new HashMap<String, String>();
MRHelpers.updateEnvBasedOnMRTaskEnv(conf, env, false);
testCommonEnvSettingsForMRTasks(env);
Assert.assertEquals("red1", env.get("foo"));
Assert.assertEquals("red2", env.get("bar"));
}
@Test(timeout = 5000)
public void testMRAMJavaOpts() {
Configuration conf = new Configuration();
conf.set(MRJobConfig.MR_AM_ADMIN_COMMAND_OPTS, " -Dadminfoobar ");
conf.set(MRJobConfig.MR_AM_COMMAND_OPTS, " -Duserfoo ");
String opts = MRHelpers.getJavaOptsForMRAM(conf);
Assert.assertEquals("-Dadminfoobar -Duserfoo", opts);
}
@Test(timeout = 5000)
public void testMRAMEnvironmentSetup() {
Configuration conf = new Configuration();
conf.set(MRJobConfig.MR_AM_ADMIN_USER_ENV, "foo=bar,admin1=foo1");
conf.set(MRJobConfig.MR_AM_ENV, "foo=bar2,user=foo2");
Map<String, String> env =
new HashMap<String, String>();
MRHelpers.updateEnvBasedOnMRAMEnv(conf, env);
Assert.assertEquals("foo1", env.get("admin1"));
Assert.assertEquals("foo2", env.get("user"));
Assert.assertEquals(("bar" + File.pathSeparator + "bar2"), env.get("foo"));
}
@Test(timeout = 5000)
public void testTranslateMRConfToTez() {
Configuration conf = new Configuration(false);
conf.setLong(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB, 1000);
conf.setLong(org.apache.tez.mapreduce.hadoop.MRJobConfig.IO_SORT_MB, 500);
Configuration conf1 = new Configuration(conf);
MRHelpers.translateMRConfToTez(conf1);
Assert.assertNull(conf1.get(org.apache.tez.mapreduce.hadoop.MRJobConfig.IO_SORT_MB));
Assert.assertEquals(1000, conf1.getLong(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB, 0));
Configuration conf2 = new Configuration(conf);
MRHelpers.translateMRConfToTez(conf2, true);
Assert.assertNull(conf2.get(org.apache.tez.mapreduce.hadoop.MRJobConfig.IO_SORT_MB));
Assert.assertEquals(1000, conf2.getLong(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB, 0));
Configuration conf3 = new Configuration(conf);
MRHelpers.translateMRConfToTez(conf3, false);
Assert.assertNull(conf3.get(org.apache.tez.mapreduce.hadoop.MRJobConfig.IO_SORT_MB));
Assert.assertEquals(500, conf3.getLong(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB, 0));
}
}