/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.carbondata.spark.util; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import org.apache.carbondata.core.constants.CarbonCommonConstants; import org.apache.carbondata.core.scan.model.CarbonQueryPlan; import org.apache.carbondata.core.statusmanager.LoadMetadataDetails; import org.apache.carbondata.spark.partition.api.Partition; import org.apache.carbondata.spark.partition.api.impl.DefaultLoadBalancer; import org.apache.carbondata.spark.partition.api.impl.PartitionMultiFileImpl; import org.apache.carbondata.spark.partition.api.impl.QueryPartitionHelper; import org.apache.carbondata.spark.splits.TableSplit; import org.apache.commons.lang3.StringUtils; /** * This utilty parses the Carbon query plan to actual query model object. */ public class CarbonQueryUtil { private CarbonQueryUtil() { } /** * It creates the one split for each region server. */ public static synchronized TableSplit[] getTableSplits(String databaseName, String tableName, CarbonQueryPlan queryPlan) { //Just create splits depends on locations of region servers List<Partition> allPartitions = null; if (queryPlan == null) { allPartitions = QueryPartitionHelper.getInstance().getAllPartitions(databaseName, tableName); } else { allPartitions = QueryPartitionHelper.getInstance().getPartitionsForQuery(queryPlan); } TableSplit[] splits = new TableSplit[allPartitions.size()]; for (int i = 0; i < splits.length; i++) { splits[i] = new TableSplit(); List<String> locations = new ArrayList<String>(CarbonCommonConstants.CONSTANT_SIZE_TEN); Partition partition = allPartitions.get(i); String location = QueryPartitionHelper.getInstance() .getLocation(partition, databaseName, tableName); locations.add(location); splits[i].setPartition(partition); splits[i].setLocations(locations); } return splits; } /** * It creates the one split for each region server. */ public static TableSplit[] getTableSplitsForDirectLoad(String sourcePath) { //Just create splits depends on locations of region servers DefaultLoadBalancer loadBalancer = null; List<Partition> allPartitions = getAllFilesForDataLoad(sourcePath); loadBalancer = new DefaultLoadBalancer(new ArrayList<String>(), allPartitions); TableSplit[] tblSplits = new TableSplit[allPartitions.size()]; for (int i = 0; i < tblSplits.length; i++) { tblSplits[i] = new TableSplit(); List<String> locations = new ArrayList<String>(CarbonCommonConstants.CONSTANT_SIZE_TEN); Partition partition = allPartitions.get(i); String location = loadBalancer.getNodeForPartitions(partition); locations.add(location); tblSplits[i].setPartition(partition); tblSplits[i].setLocations(locations); } return tblSplits; } /** * split sourcePath by comma */ public static void splitFilePath(String sourcePath, List<String> partitionsFiles, String separator) { if (StringUtils.isNotEmpty(sourcePath)) { String[] files = sourcePath.split(separator); Collections.addAll(partitionsFiles, files); } } private static List<Partition> getAllFilesForDataLoad(String sourcePath) { List<String> files = new ArrayList<String>(CarbonCommonConstants.CONSTANT_SIZE_TEN); splitFilePath(sourcePath, files, CarbonCommonConstants.COMMA); List<Partition> partitionList = new ArrayList<Partition>(CarbonCommonConstants.CONSTANT_SIZE_TEN); Map<Integer, List<String>> partitionFiles = new HashMap<Integer, List<String>>(); partitionFiles.put(0, new ArrayList<String>(CarbonCommonConstants.CONSTANT_SIZE_TEN)); partitionList.add(new PartitionMultiFileImpl(0 + "", partitionFiles.get(0))); for (int i = 0; i < files.size(); i++) { partitionFiles.get(0).add(files.get(i)); } return partitionList; } public static List<String> getListOfSlices(LoadMetadataDetails[] details) { List<String> slices = new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE); if (null != details) { for (LoadMetadataDetails oneLoad : details) { if (!CarbonCommonConstants.STORE_LOADSTATUS_FAILURE.equals(oneLoad.getLoadStatus())) { String loadName = CarbonCommonConstants.LOAD_FOLDER + oneLoad.getLoadName(); slices.add(loadName); } } } return slices; } }