/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.tez.mapreduce.grouper;
import javax.annotation.Nullable;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import com.google.common.base.Preconditions;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.util.RackResolver;
import org.apache.tez.dag.api.TezUncheckedException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public abstract class TezSplitGrouper {
private static final Logger LOG = LoggerFactory.getLogger(TezSplitGrouper.class);
/**
* Specify the number of splits desired to be created
*/
public static final String TEZ_GROUPING_SPLIT_COUNT = "tez.grouping.split-count";
/**
* Limit the number of splits in a group by the total length of the splits in the group
*/
public static final String TEZ_GROUPING_SPLIT_BY_LENGTH = "tez.grouping.by-length";
public static final boolean TEZ_GROUPING_SPLIT_BY_LENGTH_DEFAULT = true;
/**
* Limit the number of splits in a group by the number of splits in the group
*/
public static final String TEZ_GROUPING_SPLIT_BY_COUNT = "tez.grouping.by-count";
public static final boolean TEZ_GROUPING_SPLIT_BY_COUNT_DEFAULT = false;
/**
* The multiplier for available queue capacity when determining number of
* tasks for a Vertex. 1.7 with 100% queue available implies generating a
* number of tasks roughly equal to 170% of the available containers on the
* queue. This enables multiple waves of mappers where the final wave is slightly smaller
* than the remaining waves. The gap helps overlap the final wave with any slower tasks
* from previous waves and tries to hide the delays from the slower tasks. Good values for
* this are 1.7, 2.7, 3.7 etc. Increase the number of waves to make the tasks smaller or
* shorter.
*/
public static final String TEZ_GROUPING_SPLIT_WAVES = "tez.grouping.split-waves";
public static final float TEZ_GROUPING_SPLIT_WAVES_DEFAULT = 1.7f;
/**
* Upper bound on the size (in bytes) of a grouped split, to avoid generating excessively large splits.
*/
public static final String TEZ_GROUPING_SPLIT_MAX_SIZE = "tez.grouping.max-size";
public static final long TEZ_GROUPING_SPLIT_MAX_SIZE_DEFAULT = 1024*1024*1024L;
/**
* Lower bound on the size (in bytes) of a grouped split, to avoid generating too many small splits.
*/
public static final String TEZ_GROUPING_SPLIT_MIN_SIZE = "tez.grouping.min-size";
public static final long TEZ_GROUPING_SPLIT_MIN_SIZE_DEFAULT = 50*1024*1024L;
/**
* This factor is used to decrease the per group desired (length and count) limits for groups
* created by combining splits within a rack. Since reading this split involves reading data intra
* rack, the group is made smaller to cover up for the increased latencies in doing intra rack
* reads. The value should be a fraction <= 1.
*/
public static final String TEZ_GROUPING_RACK_SPLIT_SIZE_REDUCTION =
"tez.grouping.rack-split-reduction";
public static final float TEZ_GROUPING_RACK_SPLIT_SIZE_REDUCTION_DEFAULT = 0.75f;
/**
* Repeated invocations of grouping on the same splits with the same parameters will produce the
* same groups. This may help in cache reuse but may cause hot-spotting on nodes when there are a
* large number of jobs reading the same hot data. True by default.
*/
public static final String TEZ_GROUPING_REPEATABLE = "tez.grouping.repeatable";
public static final boolean TEZ_GROUPING_REPEATABLE_DEFAULT = true;
/**
* Generate node local splits only. This prevents fallback to rack locality etc, and overrides
* the target size for small splits.
*/
public static final String TEZ_GROUPING_NODE_LOCAL_ONLY = "tez.grouping.node.local.only";
public static final boolean TEZ_GROUPING_NODE_LOCAL_ONLY_DEFAULT = false;
static class LocationHolder {
List<SplitContainer> splits;
int headIndex = 0;
LocationHolder(int capacity) {
splits = new ArrayList<SplitContainer>(capacity);
}
boolean isEmpty() {
return (headIndex == splits.size());
}
SplitContainer getUnprocessedHeadSplit() {
while (!isEmpty()) {
SplitContainer holder = splits.get(headIndex);
if (!holder.isProcessed()) {
return holder;
}
incrementHeadIndex();
}
return null;
}
void incrementHeadIndex() {
headIndex++;
}
}
private static final SplitSizeEstimatorWrapper DEFAULT_SPLIT_ESTIMATOR = new DefaultSplitSizeEstimatorWrapper();
static final class DefaultSplitSizeEstimatorWrapper implements SplitSizeEstimatorWrapper {
@Override
public long getEstimatedSize(SplitContainer splitContainer) throws IOException,
InterruptedException {
return splitContainer.getLength();
}
}
private static final SplitLocationProviderWrapper DEFAULT_SPLIT_LOCATION_PROVIDER = new DefaultSplitLocationProvider();
static final class DefaultSplitLocationProvider implements SplitLocationProviderWrapper {
@Override
public String[] getPreferredLocations(SplitContainer splitContainer) throws IOException,
InterruptedException {
return splitContainer.getPreferredLocations();
}
}
Map<String, LocationHolder> createLocationsMap(Configuration conf) {
if (conf.getBoolean(TEZ_GROUPING_REPEATABLE,
TEZ_GROUPING_REPEATABLE_DEFAULT)) {
return new TreeMap<String, LocationHolder>();
}
return new HashMap<String, LocationHolder>();
}
public List<GroupedSplitContainer> getGroupedSplits(Configuration conf,
List<SplitContainer> originalSplits,
int desiredNumSplits,
String wrappedInputFormatName,
SplitSizeEstimatorWrapper estimator,
SplitLocationProviderWrapper locationProvider) throws
IOException, InterruptedException {
LOG.info("Grouping splits in Tez");
Preconditions.checkArgument(originalSplits != null, "Splits must be specified");
int configNumSplits = conf.getInt(TEZ_GROUPING_SPLIT_COUNT, 0);
if (configNumSplits > 0) {
// always use config override if specified
desiredNumSplits = configNumSplits;
LOG.info("Desired numSplits overridden by config to: " + desiredNumSplits);
}
if (estimator == null) {
estimator = DEFAULT_SPLIT_ESTIMATOR;
}
if (locationProvider == null) {
locationProvider = DEFAULT_SPLIT_LOCATION_PROVIDER;
}
List<GroupedSplitContainer> groupedSplits = null;
String emptyLocation = "EmptyLocation";
String localhost = "localhost";
String[] emptyLocations = {emptyLocation};
groupedSplits = new ArrayList<GroupedSplitContainer>(desiredNumSplits);
boolean allSplitsHaveLocalhost = true;
long totalLength = 0;
Map<String, LocationHolder> distinctLocations = createLocationsMap(conf);
// go through splits and add them to locations
for (SplitContainer split : originalSplits) {
totalLength += estimator.getEstimatedSize(split);
String[] locations = locationProvider.getPreferredLocations(split);
if (locations == null || locations.length == 0) {
locations = emptyLocations;
allSplitsHaveLocalhost = false;
}
for (String location : locations ) {
if (location == null) {
location = emptyLocation;
allSplitsHaveLocalhost = false;
}
if (!location.equalsIgnoreCase(localhost)) {
allSplitsHaveLocalhost = false;
}
distinctLocations.put(location, null);
}
}
if (! (configNumSplits > 0 ||
originalSplits.size() == 0)) {
// numSplits has not been overridden by config
// numSplits has been set at runtime
// there are splits generated
// desired splits is less than number of splits generated
// Do sanity checks
int splitCount = desiredNumSplits>0?desiredNumSplits:originalSplits.size();
long lengthPerGroup = totalLength/splitCount;
long maxLengthPerGroup = conf.getLong(
TEZ_GROUPING_SPLIT_MAX_SIZE,
TEZ_GROUPING_SPLIT_MAX_SIZE_DEFAULT);
long minLengthPerGroup = conf.getLong(
TEZ_GROUPING_SPLIT_MIN_SIZE,
TEZ_GROUPING_SPLIT_MIN_SIZE_DEFAULT);
if (maxLengthPerGroup < minLengthPerGroup ||
minLengthPerGroup <=0) {
throw new TezUncheckedException(
"Invalid max/min group lengths. Required min>0, max>=min. " +
" max: " + maxLengthPerGroup + " min: " + minLengthPerGroup);
}
if (lengthPerGroup > maxLengthPerGroup) {
// splits too big to work. Need to override with max size.
int newDesiredNumSplits = (int)(totalLength/maxLengthPerGroup) + 1;
LOG.info("Desired splits: " + desiredNumSplits + " too small. " +
" Desired splitLength: " + lengthPerGroup +
" Max splitLength: " + maxLengthPerGroup +
" New desired splits: " + newDesiredNumSplits +
" Total length: " + totalLength +
" Original splits: " + originalSplits.size());
desiredNumSplits = newDesiredNumSplits;
} else if (lengthPerGroup < minLengthPerGroup) {
// splits too small to work. Need to override with size.
int newDesiredNumSplits = (int)(totalLength/minLengthPerGroup) + 1;
/**
* This is a workaround for systems like S3 that pass the same
* fake hostname for all splits.
*/
if (!allSplitsHaveLocalhost) {
desiredNumSplits = newDesiredNumSplits;
}
LOG.info("Desired splits: " + desiredNumSplits + " too large. " +
" Desired splitLength: " + lengthPerGroup +
" Min splitLength: " + minLengthPerGroup +
" New desired splits: " + newDesiredNumSplits +
" Final desired splits: " + desiredNumSplits +
" All splits have localhost: " + allSplitsHaveLocalhost +
" Total length: " + totalLength +
" Original splits: " + originalSplits.size());
}
}
if (desiredNumSplits == 0 ||
originalSplits.size() == 0 ||
desiredNumSplits >= originalSplits.size()) {
// nothing set. so return all the splits as is
LOG.info("Using original number of splits: " + originalSplits.size() +
" desired splits: " + desiredNumSplits);
groupedSplits = new ArrayList<GroupedSplitContainer>(originalSplits.size());
for (SplitContainer split : originalSplits) {
GroupedSplitContainer newSplit =
new GroupedSplitContainer(1, wrappedInputFormatName, cleanupLocations(locationProvider.getPreferredLocations(split)),
null);
newSplit.addSplit(split);
groupedSplits.add(newSplit);
}
return groupedSplits;
}
long lengthPerGroup = totalLength/desiredNumSplits;
int numNodeLocations = distinctLocations.size();
int numSplitsPerLocation = originalSplits.size()/numNodeLocations;
int numSplitsInGroup = originalSplits.size()/desiredNumSplits;
// allocation loop here so that we have a good initial size for the lists
for (String location : distinctLocations.keySet()) {
distinctLocations.put(location, new LocationHolder(numSplitsPerLocation+1));
}
Set<String> locSet = new HashSet<String>();
for (SplitContainer split : originalSplits) {
locSet.clear();
String[] locations = locationProvider.getPreferredLocations(split);
if (locations == null || locations.length == 0) {
locations = emptyLocations;
}
for (String location : locations) {
if (location == null) {
location = emptyLocation;
}
locSet.add(location);
}
for (String location : locSet) {
LocationHolder holder = distinctLocations.get(location);
holder.splits.add(split);
}
}
boolean groupByLength = conf.getBoolean(
TEZ_GROUPING_SPLIT_BY_LENGTH,
TEZ_GROUPING_SPLIT_BY_LENGTH_DEFAULT);
boolean groupByCount = conf.getBoolean(
TEZ_GROUPING_SPLIT_BY_COUNT,
TEZ_GROUPING_SPLIT_BY_COUNT_DEFAULT);
boolean nodeLocalOnly = conf.getBoolean(
TEZ_GROUPING_NODE_LOCAL_ONLY,
TEZ_GROUPING_NODE_LOCAL_ONLY_DEFAULT);
if (!(groupByLength || groupByCount)) {
throw new TezUncheckedException(
"None of the grouping parameters are true: "
+ TEZ_GROUPING_SPLIT_BY_LENGTH + ", "
+ TEZ_GROUPING_SPLIT_BY_COUNT);
}
LOG.info("Desired numSplits: " + desiredNumSplits +
" lengthPerGroup: " + lengthPerGroup +
" numLocations: " + numNodeLocations +
" numSplitsPerLocation: " + numSplitsPerLocation +
" numSplitsInGroup: " + numSplitsInGroup +
" totalLength: " + totalLength +
" numOriginalSplits: " + originalSplits.size() +
" . Grouping by length: " + groupByLength +
" count: " + groupByCount +
" nodeLocalOnly: " + nodeLocalOnly);
// go through locations and group splits
int splitsProcessed = 0;
List<SplitContainer> group = new ArrayList<SplitContainer>(numSplitsInGroup);
Set<String> groupLocationSet = new HashSet<String>(10);
boolean allowSmallGroups = false;
boolean doingRackLocal = false;
int iterations = 0;
while (splitsProcessed < originalSplits.size()) {
iterations++;
int numFullGroupsCreated = 0;
for (Map.Entry<String, LocationHolder> entry : distinctLocations.entrySet()) {
group.clear();
groupLocationSet.clear();
String location = entry.getKey();
LocationHolder holder = entry.getValue();
SplitContainer splitContainer = holder.getUnprocessedHeadSplit();
if (splitContainer == null) {
// all splits on node processed
continue;
}
int oldHeadIndex = holder.headIndex;
long groupLength = 0;
int groupNumSplits = 0;
do {
group.add(splitContainer);
groupLength += estimator.getEstimatedSize(splitContainer);
groupNumSplits++;
holder.incrementHeadIndex();
splitContainer = holder.getUnprocessedHeadSplit();
} while(splitContainer != null
&& (!groupByLength ||
(groupLength + estimator.getEstimatedSize(splitContainer) <= lengthPerGroup))
&& (!groupByCount ||
(groupNumSplits + 1 <= numSplitsInGroup)));
if (holder.isEmpty()
&& !allowSmallGroups
&& (!groupByLength || groupLength < lengthPerGroup/2)
&& (!groupByCount || groupNumSplits < numSplitsInGroup/2)) {
// group too small, reset it
holder.headIndex = oldHeadIndex;
continue;
}
numFullGroupsCreated++;
// One split group created
String[] groupLocation = {location};
if (location == emptyLocation) {
groupLocation = null;
} else if (doingRackLocal) {
for (SplitContainer splitH : group) {
String[] locations = locationProvider.getPreferredLocations(splitH);
if (locations != null) {
for (String loc : locations) {
if (loc != null) {
groupLocationSet.add(loc);
}
}
}
}
groupLocation = groupLocationSet.toArray(groupLocation);
}
GroupedSplitContainer groupedSplit =
new GroupedSplitContainer(group.size(), wrappedInputFormatName,
groupLocation,
// pass rack local hint directly to AM
((doingRackLocal && location != emptyLocation)?location:null));
for (SplitContainer groupedSplitContainer : group) {
groupedSplit.addSplit(groupedSplitContainer);
Preconditions.checkState(groupedSplitContainer.isProcessed() == false,
"Duplicates in grouping at location: " + location);
groupedSplitContainer.setIsProcessed(true);
splitsProcessed++;
}
if (LOG.isDebugEnabled()) {
LOG.debug("Grouped " + group.size()
+ " length: " + groupedSplit.getLength()
+ " split at: " + location);
}
groupedSplits.add(groupedSplit);
}
if (!doingRackLocal && numFullGroupsCreated < 1) {
// no node could create a regular node-local group.
// Allow small groups if that is configured.
if (nodeLocalOnly && !allowSmallGroups) {
LOG.info(
"Allowing small groups early after attempting to create full groups at iteration: {}, groupsCreatedSoFar={}",
iterations, groupedSplits.size());
allowSmallGroups = true;
continue;
}
// else go rack-local
doingRackLocal = true;
// re-create locations
int numRemainingSplits = originalSplits.size() - splitsProcessed;
Set<SplitContainer> remainingSplits = new HashSet<SplitContainer>(numRemainingSplits);
// gather remaining splits.
for (Map.Entry<String, LocationHolder> entry : distinctLocations.entrySet()) {
LocationHolder locHolder = entry.getValue();
while (!locHolder.isEmpty()) {
SplitContainer splitHolder = locHolder.getUnprocessedHeadSplit();
if (splitHolder != null) {
remainingSplits.add(splitHolder);
locHolder.incrementHeadIndex();
}
}
}
if (remainingSplits.size() != numRemainingSplits) {
throw new TezUncheckedException("Expected: " + numRemainingSplits
+ " got: " + remainingSplits.size());
}
// doing all this now instead of up front because the number of remaining
// splits is expected to be much smaller
RackResolver.init(conf);
Map<String, String> locToRackMap = new HashMap<String, String>(distinctLocations.size());
Map<String, LocationHolder> rackLocations = createLocationsMap(conf);
for (String location : distinctLocations.keySet()) {
String rack = emptyLocation;
if (location != emptyLocation) {
rack = RackResolver.resolve(location).getNetworkLocation();
}
locToRackMap.put(location, rack);
if (rackLocations.get(rack) == null) {
// splits will probably be located in all racks
rackLocations.put(rack, new LocationHolder(numRemainingSplits));
}
}
distinctLocations.clear();
HashSet<String> rackSet = new HashSet<String>(rackLocations.size());
int numRackSplitsToGroup = remainingSplits.size();
for (SplitContainer split : originalSplits) {
if (numRackSplitsToGroup == 0) {
break;
}
// Iterate through the original splits in their order and consider them for grouping.
// This maintains the original ordering in the list and thus subsequent grouping will
// maintain that order
if (!remainingSplits.contains(split)) {
continue;
}
numRackSplitsToGroup--;
rackSet.clear();
String[] locations = locationProvider.getPreferredLocations(split);
if (locations == null || locations.length == 0) {
locations = emptyLocations;
}
for (String location : locations ) {
if (location == null) {
location = emptyLocation;
}
rackSet.add(locToRackMap.get(location));
}
for (String rack : rackSet) {
rackLocations.get(rack).splits.add(split);
}
}
remainingSplits.clear();
distinctLocations = rackLocations;
// adjust split length to be smaller because the data is non local
float rackSplitReduction = conf.getFloat(
TEZ_GROUPING_RACK_SPLIT_SIZE_REDUCTION,
TEZ_GROUPING_RACK_SPLIT_SIZE_REDUCTION_DEFAULT);
if (rackSplitReduction > 0) {
long newLengthPerGroup = (long)(lengthPerGroup*rackSplitReduction);
int newNumSplitsInGroup = (int) (numSplitsInGroup*rackSplitReduction);
if (newLengthPerGroup > 0) {
lengthPerGroup = newLengthPerGroup;
}
if (newNumSplitsInGroup > 0) {
numSplitsInGroup = newNumSplitsInGroup;
}
}
LOG.info("Doing rack local after iteration: " + iterations +
" splitsProcessed: " + splitsProcessed +
" numFullGroupsInRound: " + numFullGroupsCreated +
" totalGroups: " + groupedSplits.size() +
" lengthPerGroup: " + lengthPerGroup +
" numSplitsInGroup: " + numSplitsInGroup);
// dont do smallGroups for the first pass
continue;
}
if (!allowSmallGroups && numFullGroupsCreated <= numNodeLocations/10) {
// a few nodes have a lot of data or data is thinly spread across nodes
// so allow small groups now
allowSmallGroups = true;
LOG.info("Allowing small groups after iteration: " + iterations +
" splitsProcessed: " + splitsProcessed +
" numFullGroupsInRound: " + numFullGroupsCreated +
" totalGroups: " + groupedSplits.size());
}
if (LOG.isDebugEnabled()) {
LOG.debug("Iteration: " + iterations +
" splitsProcessed: " + splitsProcessed +
" numFullGroupsInRound: " + numFullGroupsCreated +
" totalGroups: " + groupedSplits.size());
}
}
LOG.info("Number of splits desired: " + desiredNumSplits +
" created: " + groupedSplits.size() +
" splitsProcessed: " + splitsProcessed);
return groupedSplits;
}
private String[] cleanupLocations(String[] locations) {
if (locations == null || locations.length == 0) {
return null;
}
boolean nullLocationFound = false;
for (String location : locations) {
if (location == null) {
nullLocationFound = true;
break;
}
}
if (!nullLocationFound) {
return locations;
} else {
List<String> newLocations = new LinkedList<>();
for (String location : locations) {
if (location != null) {
newLocations.add(location);
}
}
if (newLocations.size() == 0) {
return null;
} else {
return newLocations.toArray(new String[newLocations.size()]);
}
}
}
/**
* Builder that can be used to configure grouping in Tez
*
* @param conf
* {@link Configuration} This will be modified in place. If
* configuration values may be changed at runtime via a config file
* then pass in a {@link Configuration} that is initialized from a
* config file. The parameters that are not overridden in code will
* be derived from the Configuration object.
* @return {@link org.apache.tez.mapreduce.grouper.TezSplitGrouper.TezMRSplitsGrouperConfigBuilder}
*/
public static TezMRSplitsGrouperConfigBuilder newConfigBuilder(Configuration conf) {
return new TezMRSplitsGrouperConfigBuilder(conf);
}
public static final class TezMRSplitsGrouperConfigBuilder {
private final Configuration conf;
/**
* This configuration will be modified in place
*/
private TezMRSplitsGrouperConfigBuilder(@Nullable Configuration conf) {
if (conf == null) {
conf = new Configuration(false);
}
this.conf = conf;
}
public TezMRSplitsGrouperConfigBuilder setGroupSplitCount(int count) {
this.conf.setInt(TEZ_GROUPING_SPLIT_COUNT, count);
return this;
}
public TezMRSplitsGrouperConfigBuilder setGroupSplitByCount(boolean enabled) {
this.conf.setBoolean(TEZ_GROUPING_SPLIT_BY_COUNT, enabled);
return this;
}
public TezMRSplitsGrouperConfigBuilder setGroupSplitByLength(boolean enabled) {
this.conf.setBoolean(TEZ_GROUPING_SPLIT_BY_LENGTH, enabled);
return this;
}
public TezMRSplitsGrouperConfigBuilder setGroupSplitWaves(float multiplier) {
this.conf.setFloat(TEZ_GROUPING_SPLIT_WAVES, multiplier);
return this;
}
public TezMRSplitsGrouperConfigBuilder setGroupingRackSplitSizeReduction(float rackSplitSizeReduction) {
this.conf.setFloat(TEZ_GROUPING_RACK_SPLIT_SIZE_REDUCTION, rackSplitSizeReduction);
return this;
}
public TezMRSplitsGrouperConfigBuilder setNodeLocalGroupsOnly(boolean nodeLocalGroupsOnly) {
this.conf.setBoolean(TEZ_GROUPING_NODE_LOCAL_ONLY, nodeLocalGroupsOnly);
return this;
}
/**
* upper and lower bounds for the splits
*/
public TezMRSplitsGrouperConfigBuilder setGroupingSplitSize(long lowerBound, long upperBound) {
this.conf.setLong(TEZ_GROUPING_SPLIT_MIN_SIZE, lowerBound);
this.conf.setLong(TEZ_GROUPING_SPLIT_MAX_SIZE, upperBound);
return this;
}
public Configuration build() {
return this.conf;
}
}
}