/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.tajo;
import com.google.common.base.Function;
import com.google.common.base.Preconditions;
import com.google.common.collect.Collections2;
import com.google.common.collect.Lists;
import com.google.protobuf.ServiceException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.tajo.algebra.*;
import org.apache.tajo.annotation.Nullable;
import org.apache.tajo.catalog.CatalogService;
import org.apache.tajo.catalog.TableDesc;
import org.apache.tajo.cli.tsql.InvalidStatementException;
import org.apache.tajo.cli.tsql.ParsedResult;
import org.apache.tajo.cli.tsql.SimpleParser;
import org.apache.tajo.client.TajoClient;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.engine.query.QueryContext;
import org.apache.tajo.exception.InsufficientPrivilegeException;
import org.apache.tajo.exception.TajoException;
import org.apache.tajo.exception.UndefinedTableException;
import org.apache.tajo.jdbc.FetchResultSet;
import org.apache.tajo.jdbc.TajoMemoryResultSet;
import org.apache.tajo.jdbc.TajoResultSetBase;
import org.apache.tajo.master.GlobalEngine;
import org.apache.tajo.parser.sql.SQLAnalyzer;
import org.apache.tajo.plan.LogicalOptimizer;
import org.apache.tajo.plan.LogicalPlan;
import org.apache.tajo.plan.LogicalPlanner;
import org.apache.tajo.plan.verifier.LogicalPlanVerifier;
import org.apache.tajo.plan.verifier.PreLogicalPlanVerifier;
import org.apache.tajo.plan.verifier.VerificationState;
import org.apache.tajo.schema.IdentifierUtil;
import org.apache.tajo.storage.BufferPool;
import org.apache.tajo.storage.StorageUtil;
import org.apache.tajo.util.FileUtil;
import org.junit.*;
import org.junit.rules.TestName;
import org.junit.rules.TestRule;
import org.junit.rules.TestWatcher;
import org.junit.runner.Description;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.lang.annotation.*;
import java.lang.management.BufferPoolMXBean;
import java.lang.reflect.Method;
import java.net.URL;
import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.util.*;
import static org.junit.Assert.*;
/**
* (Note that this class is not thread safe. Do not execute maven test in any parallel mode.)
* <br />
* <code>QueryTestCaseBase</code> provides useful methods to easily execute queries and verify their results.
*
* This class basically uses four resource directories:
* <ul>
* <li>src/test/resources/dataset - contains a set of data files. It contains sub directories, each of which
* corresponds each test class. All data files in each sub directory can be used in the corresponding test class.</li>
*
* <li>src/test/resources/queries - This is the query directory. It contains sub directories, each of which
* corresponds each test class. All query files in each sub directory can be used in the corresponding test
* class. This directory also can include <code>positive</code> and <code>negative</code> directories, each
* of which include multiple files containing multiple queries. They are used by <code>runPositiveTests()</code> and
* <code>runNegativeTests()</code>, which test just successfully completed and failed respectively
* without compararing query results.</li>
*
* <li>src/test/resources/results - This is the result directory. It contains sub directories, each of which
* corresponds each test class. All result files in each sub directory can be used in the corresponding test class.
* </li>
* </ul>
*
* For example, if you create a test class named <code>TestJoinQuery</code>, you should create a pair of query and
* result set directories as follows:
*
* <pre>
* src-|
* |- resources
* |- dataset
* | |- TestJoinQuery
* | |- table1.tbl
* | |- table2.tbl
* |
* |- queries
* | |- TestJoinQuery
* | |- positive
* | |- valid_join_conditions.sql
* | |- negative
* | |- invalid_join_conditions.sql
* | |- TestInnerJoin.sql
* | |- table1_ddl.sql
* | |- table2_ddl.sql
* |
* |- results
* |- TestJoinQuery
* |- TestInnerJoin.result
* </pre>
*
* <code>QueryTestCaseBase</code> basically provides the following methods:
* <ul>
* <li><code>{@link #executeQuery()}</code> - executes a corresponding query and returns an ResultSet instance</li>
* <li><code>{@link #executeFile(String)}</code> - executes a given query file included in the corresponding query
* file in the current class's query directory</li>
* <li><code>assertResultSet()</code> - check if the query result is equivalent to the expected result included
* in the corresponding result file in the current class's result directory.</li>
* <li><code>cleanQuery()</code> - clean up all resources</li>
* <li><code>executeDDL()</code> - execute a DDL query like create or drop table.</li>
* </ul>
*
* In order to make use of the above methods, query files and results file must be as follows:
* <ul>
* <li>Each query file must be located on the subdirectory whose structure must be src/resources/queries/${ClassName},
* where ${ClassName} indicates an actual test class's simple name.</li>
* <li>Each result file must be located on the subdirectory whose structure must be src/resources/results/${ClassName},
* where ${ClassName} indicates an actual test class's simple name.</li>
* </ul>
*
* Especially, {@link #executeQuery() and {@link #assertResultSet(java.sql.ResultSet)} methods automatically finds
* a query file to be executed and a result to be compared, which are corresponding to the running class and method.
* For them, query and result files additionally must be follows as:
* <ul>
* <li>Each result file must have the file extension '.result'</li>
* <li>Each query file must have the file extension '.sql'.</li>
* </ul>
*/
public class QueryTestCaseBase {
private static final Log LOG = LogFactory.getLog(QueryTestCaseBase.class);
protected static final TpchTestBase testBase;
protected static final TajoTestingCluster testingCluster;
protected static TajoConf conf;
protected static TajoClient client;
protected static final CatalogService catalog;
protected static final SQLAnalyzer sqlParser;
protected static PreLogicalPlanVerifier verifier;
protected static LogicalPlanner planner;
protected static LogicalOptimizer optimizer;
protected static LogicalPlanVerifier postVerifier;
/** the base path of dataset directories */
protected static Path datasetBasePath;
/** the base path of query directories */
protected static Path queryBasePath;
/** the base path of result directories */
protected static Path resultBasePath;
static {
testBase = TpchTestBase.getInstance();
testingCluster = testBase.getTestingCluster();
conf = testBase.getTestingCluster().getConfiguration();
catalog = testBase.getTestingCluster().getMaster().getCatalog();
GlobalEngine engine = testingCluster.getMaster().getContext().getGlobalEngine();
sqlParser = engine.getAnalyzer();
verifier = engine.getPreLogicalPlanVerifier();
planner = engine.getLogicalPlanner();
optimizer = engine.getLogicalOptimizer();
postVerifier = engine.getLogicalPlanVerifier();
}
/** It transiently contains created tables for the running test class. */
private static String currentDatabase;
private static Set<String> createdTableGlobalSet = new HashSet<>();
// queries and results directory corresponding to subclass class.
protected Path currentQueryPath;
protected Path namedQueryPath;
protected Path currentResultPath;
protected Path currentDatasetPath;
protected Path namedDatasetPath;
protected FileSystem currentResultFS;
protected final String testParameter;
// for getting a method name
@Rule public TestName name = new TestName();
@BeforeClass
public static void setUpClass() throws Exception {
client = testBase.getTestingCluster().newTajoClient();
URL datasetBaseURL = ClassLoader.getSystemResource("dataset");
Preconditions.checkNotNull(datasetBaseURL, "dataset directory is absent.");
datasetBasePath = new Path(datasetBaseURL.toString());
URL queryBaseURL = ClassLoader.getSystemResource("queries");
Preconditions.checkNotNull(queryBaseURL, "queries directory is absent.");
queryBasePath = new Path(queryBaseURL.toString());
URL resultBaseURL = ClassLoader.getSystemResource("results");
Preconditions.checkNotNull(resultBaseURL, "results directory is absent.");
resultBasePath = new Path(resultBaseURL.toString());
}
@AfterClass
public static void tearDownClass() throws Exception {
for (String tableName : createdTableGlobalSet) {
client.updateQuery("DROP TABLE IF EXISTS " + IdentifierUtil.denormalizeIdentifier(tableName));
}
createdTableGlobalSet.clear();
// if the current database is "default", shouldn't drop it.
if (!currentDatabase.equals(TajoConstants.DEFAULT_DATABASE_NAME)) {
for (String tableName : catalog.getAllTableNames(currentDatabase)) {
try {
client.updateQuery("DROP TABLE IF EXISTS " + tableName);
} catch (InsufficientPrivilegeException i) {
LOG.warn("relation '" + tableName + "' is read only.");
}
}
client.selectDatabase(TajoConstants.DEFAULT_DATABASE_NAME);
try {
client.dropDatabase(currentDatabase);
} catch (InsufficientPrivilegeException e) {
LOG.warn("database '" + currentDatabase + "' is read only.");
}
}
client.close();
}
@Before
public void printTestName() {
/* protect a travis stalled build */
BufferPoolMXBean direct = BufferPool.getDirectBufferPool();
BufferPoolMXBean mapped = BufferPool.getMappedBufferPool();
System.out.println(String.format("Used heap: %s/%s, direct:%s/%s, mapped:%s/%s, Active Threads: %d, Run: %s.%s",
FileUtil.humanReadableByteCount(Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory(), false),
FileUtil.humanReadableByteCount(Runtime.getRuntime().maxMemory(), false),
FileUtil.humanReadableByteCount(direct.getMemoryUsed(), false),
FileUtil.humanReadableByteCount(direct.getTotalCapacity(), false),
FileUtil.humanReadableByteCount(mapped.getMemoryUsed(), false),
FileUtil.humanReadableByteCount(mapped.getTotalCapacity(), false),
Thread.activeCount(),
getClass().getSimpleName(),
name.getMethodName()));
}
@After
public void clear() {
getClient().unsetSessionVariables(Lists.newArrayList(SessionVars.TIMEZONE.name()));
}
public QueryTestCaseBase() {
// hive 0.12 does not support quoted identifier.
// So, we use lower case database names when Tajo uses HiveCatalogStore.
if (testingCluster.isHiveCatalogStoreRunning()) {
this.currentDatabase = getClass().getSimpleName().toLowerCase();
} else {
this.currentDatabase = getClass().getSimpleName();
}
testParameter = null;
init();
}
public QueryTestCaseBase(String currentDatabase) {
this(currentDatabase, null);
}
public QueryTestCaseBase(String currentDatabase, String testParameter) {
this.currentDatabase = currentDatabase;
this.testParameter = testParameter;
init();
}
private void init() {
String className = getClass().getSimpleName();
currentQueryPath = new Path(queryBasePath, className);
currentResultPath = new Path(resultBasePath, className);
currentDatasetPath = new Path(datasetBasePath, className);
NamedTest namedTest = getClass().getAnnotation(NamedTest.class);
if (namedTest != null) {
namedQueryPath = new Path(queryBasePath, namedTest.value());
namedDatasetPath = new Path(datasetBasePath, namedTest.value());
}
try {
// if the current database is "default", we don't need create it because it is already prepated at startup time.
if (!currentDatabase.equals(TajoConstants.DEFAULT_DATABASE_NAME)) {
client.updateQuery("CREATE DATABASE IF NOT EXISTS " + IdentifierUtil.denormalizeIdentifier(currentDatabase));
}
client.selectDatabase(currentDatabase);
currentResultFS = currentResultPath.getFileSystem(testBase.getTestingCluster().getConfiguration());
} catch (Exception e) {
throw new RuntimeException(e);
}
testingCluster.setAllTajoDaemonConfValue(TajoConf.ConfVars.$TEST_BROADCAST_JOIN_ENABLED.varname, "false");
}
protected TajoClient getClient() {
return client;
}
public String getCurrentDatabase() {
return currentDatabase;
}
private static VerificationState verify(String query) throws TajoException {
VerificationState state = new VerificationState();
QueryContext context = LocalTajoTestingUtility.createDummyContext(conf);
Expr expr = sqlParser.parse(query);
verifier.verify(context, state, expr);
if (state.getErrors().size() > 0) {
return state;
}
LogicalPlan plan = planner.createPlan(context, expr);
optimizer.optimize(plan);
postVerifier.verify(state, plan);
return state;
}
public void assertValidSQL(String query) throws IOException {
VerificationState state = null;
try {
state = verify(query);
if (state.getErrors().size() > 0) {
fail(state.getErrors().get(0).getMessage());
}
} catch (TajoException e) {
throw new RuntimeException(e);
}
}
public void assertValidSQLFromFile(String fileName) throws IOException {
Path queryFilePath = getQueryFilePath(fileName);
String query = FileUtil.readTextFile(new File(queryFilePath.toUri()));
assertValidSQL(query);
}
public void assertInvalidSQL(String query) throws IOException {
VerificationState state = null;
try {
state = verify(query);
if (state.getErrors().size() == 0) {
fail(PreLogicalPlanVerifier.class.getSimpleName() + " cannot catch any verification error: " + query);
}
} catch (TajoException e) {
throw new RuntimeException(e);
}
}
public void assertInvalidSQLFromFile(String fileName) throws IOException {
Path queryFilePath = getQueryFilePath(fileName);
String query = FileUtil.readTextFile(new File(queryFilePath.toUri()));
assertInvalidSQL(query);
}
public void assertPlanError(String fileName) throws IOException {
Path queryFilePath = getQueryFilePath(fileName);
String query = FileUtil.readTextFile(new File(queryFilePath.toUri()));
try {
verify(query);
} catch (TajoException e) {
return;
}
fail("Cannot catch any planning error from: " + query);
}
protected ResultSet executeString(String sql) throws TajoException {
return client.executeQueryAndGetResult(sql);
}
/**
* It executes the query file and compare the result against the the result file.
*
* @throws Exception
*/
public void assertQuery() throws Exception {
ResultSet res = null;
try {
res = executeQuery();
assertResultSet(res);
} finally {
if (res != null) {
res.close();
}
}
}
/**
* It executes a given query statement and verifies the result against the the result file.
*
* @param query A query statement
* @throws Exception
*/
public void assertQueryStr(String query) throws Exception {
ResultSet res = null;
try {
res = executeString(query);
assertResultSet(res);
} finally {
if (res != null) {
res.close();
}
}
}
/**
* Execute a query contained in the file located in src/test/resources/results/<i>ClassName</i>/<i>MethodName</i>.
* <i>ClassName</i> and <i>MethodName</i> will be replaced by actual executed class and methods.
*
* @return ResultSet of query execution.
*/
public ResultSet executeQuery() throws Exception {
return executeFile(getMethodName() + ".sql");
}
private volatile Description current;
@Rule
public TestRule watcher = new TestWatcher() {
@Override
protected void starting(Description description) {
QueryTestCaseBase.this.current = description;
}
};
@Target({ElementType.METHOD})
@Retention(RetentionPolicy.RUNTIME)
protected @interface SimpleTest {
String[] prepare() default {};
QuerySpec[] queries() default {};
String[] cleanup() default {};
}
@Target({ElementType.METHOD})
@Retention(RetentionPolicy.RUNTIME)
protected @interface QuerySpec {
String value();
boolean override() default false;
Option option() default @Option;
}
@Target({ElementType.METHOD})
@Retention(RetentionPolicy.RUNTIME)
protected @interface Option {
boolean withExplain() default false;
boolean withExplainGlobal() default false;
boolean parameterized() default false;
boolean sort() default false;
boolean resultClose() default true;
}
private static class DummyQuerySpec implements QuerySpec {
private final String value;
private final Option option;
public DummyQuerySpec(String value, Option option) {
this.value = value;
this.option = option;
}
public Class<? extends Annotation> annotationType() { return QuerySpec.class; }
public String value() { return value; }
public boolean override() { return option != null; }
public Option option() { return option; }
}
private static class DummyOption implements Option {
private final boolean explain;
private final boolean withExplainGlobal;
private final boolean parameterized;
private final boolean sort;
private final boolean resultClose;
public DummyOption(boolean explain, boolean withExplainGlobal, boolean parameterized, boolean sort,
boolean resultClose) {
this.explain = explain;
this.withExplainGlobal = withExplainGlobal;
this.parameterized = parameterized;
this.sort = sort;
this.resultClose = resultClose;
}
public Class<? extends Annotation> annotationType() { return Option.class; }
public boolean withExplain() { return explain;}
public boolean withExplainGlobal() { return withExplainGlobal;}
public boolean parameterized() { return parameterized;}
public boolean sort() { return sort;}
@Override
public boolean resultClose() {
return resultClose;
}
}
protected Collection<String> getBatchQueries(Collection<Path> paths) throws IOException, InvalidStatementException {
List<String> queries = Lists.newArrayList();
for (Path p : paths) {
for (ParsedResult statement: SimpleParser.parseScript(FileUtil.readTextFile(new File(p.toUri())))) {
queries.add(statement.getStatement());
}
}
return queries;
}
/**
* Run all positive tests
*
* @throws Exception
*/
protected void runPositiveTests() throws Exception {
Collection<String> queries = getBatchQueries(getPositiveQueryFiles());
ResultSet result = null;
for (String query: queries) {
try {
result = client.executeQueryAndGetResult(query);
} catch (TajoException e) {
fail("Positive Test Failed: " + e.getMessage());
} finally {
if (result != null) {
result.close();
}
}
}
}
/**
* Run all negative tests
*
* @throws Exception
*/
protected void runNegativeTests() throws Exception {
Collection<String> queries = getBatchQueries(getNegativeQueryFiles());
ResultSet result = null;
for (String query: queries) {
try {
result = client.executeQueryAndGetResult(query);
fail("Negative Test Failed: " + query);
} catch (TajoException e) {
} finally {
if (result != null) {
result.close();
}
}
}
}
protected Optional<TajoResultSetBase[]> runSimpleTests() throws Exception {
String methodName = getMethodName();
Method method = current.getTestClass().getMethod(methodName);
SimpleTest annotation = method.getAnnotation(SimpleTest.class);
if (annotation == null) {
throw new IllegalStateException("Cannot find test annotation");
}
List<String> prepares = new ArrayList<>(Arrays.asList(annotation.prepare()));
QuerySpec[] queries = annotation.queries();
Option defaultOption = method.getAnnotation(Option.class);
if (defaultOption == null) {
defaultOption = new DummyOption(false, false, false, false, true);
}
boolean fromFile = false;
if (queries.length == 0) {
Path queryFilePath = getQueryFilePath(getMethodName() + ".sql");
List<ParsedResult> parsedResults = SimpleParser.parseScript(FileUtil.readTextFile(new File(queryFilePath.toUri())));
int i = 0;
for (; i < parsedResults.size() - 1; i++) {
prepares.add(parsedResults.get(i).getStatement());
}
queries = new QuerySpec[] {new DummyQuerySpec(parsedResults.get(i).getHistoryStatement(), null)};
fromFile = true; // do not append query index to result file
}
try {
for (String prepare : prepares) {
client.executeQueryAndGetResult(prepare).close();
}
List<TajoResultSetBase> resultSetBases = new ArrayList<>();
for (int i = 0; i < queries.length; i++) {
QuerySpec spec = queries[i];
Option option = spec.override() ? spec.option() : defaultOption;
String prefix = "";
testingCluster.getConfiguration().set(TajoConf.ConfVars.$TEST_PLAN_SHAPE_FIX_ENABLED.varname, "true");
if (option.withExplain()) {// Enable this option to fix the shape of the generated plans.
prefix += resultSetToString(executeString("explain " + spec.value()));
}
if (option.withExplainGlobal()) {
// Enable this option to fix the shape of the generated plans.
prefix += resultSetToString(executeString("explain global " + spec.value()));
}
// plan test
if (prefix.length() > 0) {
String planResultName = methodName + (fromFile ? "" : "." + (i + 1)) +
((option.parameterized() && testParameter != null) ? "." + testParameter : "") + ".plan";
Path resultPath = StorageUtil.concatPath(currentResultPath, planResultName);
if (currentResultFS.exists(resultPath)) {
assertEquals("Plan Verification for: " + (i + 1) + " th test",
FileUtil.readTextFromStream(currentResultFS.open(resultPath)), prefix);
} else if (prefix.length() > 0) {
// If there is no result file expected, create gold files for new tests.
FileUtil.writeTextToStream(prefix, currentResultFS.create(resultPath));
LOG.info("New test output for " + current.getDisplayName() + " is written to " + resultPath);
// should be copied to src directory
}
}
testingCluster.getConfiguration().set(TajoConf.ConfVars.$TEST_PLAN_SHAPE_FIX_ENABLED.varname, "false");
ResultSet result = client.executeQueryAndGetResult(spec.value());
resultSetBases.add((TajoResultSetBase) result);
// result test
String fileName = methodName + (fromFile ? "" : "." + (i + 1)) + ".result";
Path resultPath = StorageUtil.concatPath(currentResultPath, fileName);
if (currentResultFS.exists(resultPath)) {
assertEquals("Result Verification for: " + (i + 1) + " th test",
FileUtil.readTextFromStream(currentResultFS.open(resultPath)), resultSetToString(result, option.sort()));
} else if (!isNull(result)) {
// If there is no result file expected, create gold files for new tests.
FileUtil.writeTextToStream(resultSetToString(result, option.sort()), currentResultFS.create(resultPath));
LOG.info("New test output for " + current.getDisplayName() + " is written to " + resultPath);
// should be copied to src directory
}
if (option.resultClose()) {
result.close();
}
}
if (resultSetBases.size() > 0) {
return Optional.of(resultSetBases.toArray(new TajoResultSetBase[resultSetBases.size()]));
} else {
return Optional.empty();
}
} finally {
for (String cleanup : annotation.cleanup()) {
try {
client.executeQueryAndGetResult(cleanup).close();
} catch (SQLException e) {
// ignore
}
}
}
}
protected void closeResultSets(ResultSet... resultSets) throws SQLException {
for (ResultSet rs : resultSets) {
rs.close();
}
}
private boolean isNull(ResultSet result) throws SQLException {
return result.getMetaData().getColumnCount() == 0;
}
protected String getMethodName() {
String methodName = name.getMethodName();
// In the case of parameter execution name's pattern is methodName[0]
if (methodName.endsWith("]")) {
int index = methodName.indexOf('[');
methodName = methodName.substring(0, index);
}
return methodName;
}
public ResultSet executeJsonQuery() throws Exception {
return executeJsonFile(getMethodName() + ".json");
}
/**
* Execute a query contained in the given named file. This methods tries to find the given file within the directory
* src/test/resources/results/<i>ClassName</i>.
*
* @param queryFileName The file name to be used to execute a query.
* @return ResultSet of query execution.
*/
public ResultSet executeFile(String queryFileName) throws Exception {
Path queryFilePath = getQueryFilePath(queryFileName);
List<ParsedResult> parsedResults = SimpleParser.parseScript(FileUtil.readTextFile(new File(queryFilePath.toUri())));
if (parsedResults.size() > 1) {
assertNotNull("This script \"" + queryFileName + "\" includes two or more queries");
}
int idx = 0;
for (; idx < parsedResults.size() - 1; idx++) {
client.executeQueryAndGetResult(parsedResults.get(idx).getHistoryStatement()).close();
}
ResultSet result = client.executeQueryAndGetResult(parsedResults.get(idx).getHistoryStatement());
assertNotNull("Query succeeded test", result);
return result;
}
public ResultSet executeJsonFile(String jsonFileName) throws Exception {
Path queryFilePath = getQueryFilePath(jsonFileName);
ResultSet result = client.executeJsonQueryAndGetResult(FileUtil.readTextFile(new File(queryFilePath.toUri())));
assertNotNull("Query succeeded test", result);
return result;
}
/**
* Assert the equivalence between the expected result and an actual query result.
* If it isn't it throws an AssertionError.
*
* @param result Query result to be compared.
*/
public final void assertResultSet(ResultSet result) throws IOException {
assertResultSet("Result Verification", result, getMethodName() + ".result");
}
/**
* Assert the equivalence between the expected result and an actual query result.
* If it isn't it throws an AssertionError.
*
* @param result Query result to be compared.
* @param resultFileName The file name containing the result to be compared
*/
public final void assertResultSet(ResultSet result, String resultFileName) throws IOException {
assertResultSet("Result Verification", result, resultFileName);
}
/**
* Assert the equivalence between the expected result and an actual query result.
* If it isn't it throws an AssertionError with the given message.
*
* @param message message The message to printed if the assertion is failed.
* @param result Query result to be compared.
*/
public final void assertResultSet(String message, ResultSet result, String resultFileName) throws IOException {
Path resultFile = getResultFile(resultFileName);
try {
verifyResultText(message, result, resultFile);
} catch (SQLException e) {
throw new IOException(e);
}
}
public final void assertStrings(String actual) throws IOException {
assertStrings(actual, getMethodName() + ".result");
}
public final void assertStrings(String actual, String resultFileName) throws IOException {
assertStrings("Result Verification", actual, resultFileName);
}
public final void assertStrings(String message, String actual, String resultFileName) throws IOException {
Path resultFile = getResultFile(resultFileName);
String expectedResult = FileUtil.readTextFile(new File(resultFile.toUri()));
assertEquals(message, expectedResult, actual);
}
/**
* Release all resources
*
* @param resultSet ResultSet
*/
public final void cleanupQuery(ResultSet resultSet) throws IOException {
if (resultSet == null) {
return;
}
try {
resultSet.close();
} catch (SQLException e) {
throw new IOException(e);
}
}
/**
* Assert that the database exists.
* @param databaseName The database name to be checked. This name is case sensitive.
*/
public void assertDatabaseExists(String databaseName) throws SQLException {
assertTrue(client.existDatabase(databaseName));
}
/**
* Assert that the database does not exists.
* @param databaseName The database name to be checked. This name is case sensitive.
*/
public void assertDatabaseNotExists(String databaseName) {
assertTrue(!client.existDatabase(databaseName));
}
/**
* Assert that the table exists.
*
* @param tableName The table name to be checked. This name is case sensitive.
* @throws ServiceException
*/
public void assertTableExists(String tableName) {
assertTrue(client.existTable(tableName));
}
/**
* Assert that the table does not exist.
*
* @param tableName The table name to be checked. This name is case sensitive.
*/
public void assertTableNotExists(String tableName) {
assertTrue(!client.existTable(tableName));
}
public void assertColumnExists(String tableName,String columnName) throws UndefinedTableException {
TableDesc tableDesc = getTableDesc(tableName);
assertTrue(tableDesc.getSchema().containsByName(columnName));
}
private TableDesc getTableDesc(String tableName) throws UndefinedTableException {
return client.getTableDesc(tableName);
}
public void assertTablePropertyEquals(String tableName, String key, String expectedValue)
throws UndefinedTableException {
TableDesc tableDesc = getTableDesc(tableName);
assertEquals(expectedValue, tableDesc.getMeta().getProperty(key));
}
public String resultSetToString(ResultSet resultSet) throws SQLException {
return resultSetToString(resultSet, false);
}
/**
* It transforms a ResultSet instance to rows represented as strings.
*
* @param resultSet ResultSet that contains a query result
* @return String
* @throws SQLException
*/
public String resultSetToString(ResultSet resultSet, boolean sort) throws SQLException {
StringBuilder sb = new StringBuilder();
ResultSetMetaData rsmd = resultSet.getMetaData();
int numOfColumns = rsmd.getColumnCount();
for (int i = 1; i <= numOfColumns; i++) {
if (i > 1) sb.append(",");
String columnName = rsmd.getColumnName(i);
sb.append(columnName);
}
sb.append("\n-------------------------------\n");
List<String> results = new ArrayList<>();
while (resultSet.next()) {
StringBuilder line = new StringBuilder();
for (int i = 1; i <= numOfColumns; i++) {
if (i > 1) line.append(",");
String columnValue = resultSet.getString(i);
if (resultSet.wasNull()) {
columnValue = "null";
}
line.append(columnValue);
}
results.add(line.toString());
}
if (sort) {
Collections.sort(results);
}
for (String line : results) {
sb.append(line).append('\n');
}
return sb.toString();
}
private void verifyResultText(String message, ResultSet res, Path resultFile) throws SQLException, IOException {
String actualResult = resultSetToString(res);
String expectedResult = FileUtil.readTextFile(new File(resultFile.toUri()));
assertEquals(message, expectedResult.trim(), actualResult.trim());
}
private Collection<Path> getPositiveQueryFiles() throws IOException {
Path positiveQueryDir = StorageUtil.concatPath(currentQueryPath, "positive");
FileSystem fs = currentQueryPath.getFileSystem(testBase.getTestingCluster().getConfiguration());
if (!fs.exists(positiveQueryDir)) {
throw new IOException("Cannot find " + positiveQueryDir);
}
return Collections2.transform(Lists.newArrayList(fs.listStatus(positiveQueryDir)), new Function<FileStatus, Path>(){
@Override
public Path apply(@Nullable FileStatus fileStatus) {
return fileStatus.getPath();
}
});
}
private Collection<Path> getNegativeQueryFiles() throws IOException {
Path positiveQueryDir = StorageUtil.concatPath(currentQueryPath, "negative");
FileSystem fs = currentQueryPath.getFileSystem(testBase.getTestingCluster().getConfiguration());
if (!fs.exists(positiveQueryDir)) {
throw new IOException("Cannot find " + positiveQueryDir);
}
return Collections2.transform(Lists.newArrayList(fs.listStatus(positiveQueryDir)),new Function<FileStatus, Path>(){
@Override
public Path apply(@Nullable FileStatus fileStatus) {
return fileStatus.getPath();
}
});
}
private Path getQueryFilePath(String fileName) throws IOException {
Path queryFilePath = StorageUtil.concatPath(currentQueryPath, fileName);
FileSystem fs = currentQueryPath.getFileSystem(testBase.getTestingCluster().getConfiguration());
if (!fs.exists(queryFilePath)) {
if (namedQueryPath != null) {
queryFilePath = StorageUtil.concatPath(namedQueryPath, fileName);
fs = namedQueryPath.getFileSystem(testBase.getTestingCluster().getConfiguration());
if (!fs.exists(queryFilePath)) {
throw new IOException("Cannot find " + fileName + " at " + currentQueryPath + " and " + namedQueryPath);
}
} else {
throw new IOException("Cannot find " + fileName + " at " + currentQueryPath);
}
}
return queryFilePath;
}
protected String getResultContents(String fileName) throws IOException {
Path resultFile = getResultFile(getMethodName() + ".result");
return FileUtil.readTextFile(new File(resultFile.toUri()));
}
protected Path getResultFile(String fileName) throws IOException {
Path resultPath = StorageUtil.concatPath(currentResultPath, fileName);
FileSystem fs = currentResultPath.getFileSystem(testBase.getTestingCluster().getConfiguration());
assertTrue(resultPath.toString() + " existence check", fs.exists(resultPath));
return resultPath;
}
protected Path getDataSetFile(String fileName) throws IOException {
Path dataFilePath = StorageUtil.concatPath(currentDatasetPath, fileName);
FileSystem fs = currentDatasetPath.getFileSystem(testBase.getTestingCluster().getConfiguration());
if (!fs.exists(dataFilePath)) {
if (namedDatasetPath != null) {
dataFilePath = StorageUtil.concatPath(namedDatasetPath, fileName);
fs = namedDatasetPath.getFileSystem(testBase.getTestingCluster().getConfiguration());
if (!fs.exists(dataFilePath)) {
throw new IOException("Cannot find " + fileName + " at " + currentDatasetPath);
}
} else {
throw new IOException("Cannot find " + fileName + " at " + currentDatasetPath);
}
}
return dataFilePath;
}
public List<String> executeDDL(String ddlFileName, @Nullable String [] args) throws Exception {
return executeDDL(ddlFileName, null, true, args);
}
/**
*
* Execute a data definition language (DDL) template. A general SQL DDL statement can be included in this file. But,
* for user-specified table name or exact external table path, you must use some format string to indicate them.
* The format string will be replaced by the corresponding arguments.
*
* The below is predefined format strings:
* <ul>
* <li>${table.path} - It is replaced by the absolute file path that <code>dataFileName</code> points. </li>
* <li>${i} - It is replaced by the corresponding element of <code>args</code>. For example, ${0} and ${1} are
* replaced by the first and second elements of <code>args</code> respectively</li>. It uses zero-based index.
* </ul>
*
* Example ddl
* <pre>
* CREATE EXTERNAL TABLE ${0} (
* t_timestamp TIMESTAMP,
* t_date DATE
* ) USING TEXT LOCATION ${table.path}
* </pre>
*
* @param ddlFileName A file name, containing a data definition statement.
* @param dataFileName A file name, containing data rows, which columns have to be separated by vertical bar '|'.
* This file name is used for replacing some format string indicating an external table location.
* @param args A list of arguments, each of which is used to replace corresponding variable which has a form of ${i}.
* @return The table names created
*/
public List<String> executeDDL(String ddlFileName, @Nullable String dataFileName, @Nullable String ... args)
throws Exception {
return executeDDL(ddlFileName, dataFileName, true, args);
}
private List<String> executeDDL(String ddlFileName, @Nullable String dataFileName, boolean isLocalTable,
@Nullable String[] args) throws Exception {
Path ddlFilePath = getQueryFilePath(ddlFileName);
String template = FileUtil.readTextFile(new File(ddlFilePath.toUri()));
String dataFilePath = null;
if (dataFileName != null) {
dataFilePath = getDataSetFile(dataFileName).toString();
}
String compiled = compileTemplate(template, dataFilePath, args);
List<ParsedResult> parsedResults = SimpleParser.parseScript(compiled);
List<String> createdTableNames = new ArrayList<>();
for (ParsedResult parsedResult : parsedResults) {
// parse a statement
Expr expr = sqlParser.parse(parsedResult.getHistoryStatement());
assertNotNull(ddlFilePath + " cannot be parsed", expr);
if (expr.getType() == OpType.CreateTable) {
CreateTable createTable = (CreateTable) expr;
String tableName = createTable.getTableName();
assertTrue("Table [" + tableName + "] creation is failed.",
client.updateQuery(parsedResult.getHistoryStatement()));
TableDesc createdTable = client.getTableDesc(tableName);
String createdTableName = createdTable.getName();
assertTrue("table '" + createdTableName + "' creation check", client.existTable(createdTableName));
if (isLocalTable) {
createdTableGlobalSet.add(createdTableName);
createdTableNames.add(tableName);
}
} else if (expr.getType() == OpType.DropTable) {
DropTable dropTable = (DropTable) expr;
String tableName = dropTable.getTableName();
assertTrue("table '" + tableName + "' existence check",
client.existTable(IdentifierUtil.buildFQName(currentDatabase, tableName)));
assertTrue("table drop is failed.", client.updateQuery(parsedResult.getHistoryStatement()));
assertFalse("table '" + tableName + "' dropped check",
client.existTable(IdentifierUtil.buildFQName(currentDatabase, tableName)));
if (isLocalTable) {
createdTableGlobalSet.remove(tableName);
}
} else if (expr.getType() == OpType.AlterTable) {
AlterTable alterTable = (AlterTable) expr;
String tableName = alterTable.getTableName();
assertTrue("table '" + tableName + "' existence check", client.existTable(tableName));
client.updateQuery(compiled);
if (isLocalTable) {
createdTableGlobalSet.remove(tableName);
}
} else if (expr.getType() == OpType.CreateIndex) {
// TODO: index existence check
client.executeQuery(compiled);
} else {
assertTrue(ddlFilePath + " is not a Create or Drop Table statement", false);
}
}
return createdTableNames;
}
/**
* Replace format strings by a given parameters.
*
* @param template
* @param dataFileName The data file name to replace <code>${table.path}</code>
* @param args The list argument to replace each corresponding format string ${i}. ${i} uses zero-based index.
* @return A string compiled
*/
private String compileTemplate(String template, @Nullable String dataFileName, @Nullable String ... args) {
String result;
if (dataFileName != null) {
result = template.replace("${table.path}", "\'" + dataFileName + "'");
} else {
result = template;
}
if (args != null) {
for (int i = 0; i < args.length; i++) {
result = result.replace("${" + i + "}", args[i]);
}
}
return result;
}
/**
* Reads data file from Test Cluster's HDFS
* @param path data parent path
* @return data file's contents
* @throws Exception
*/
public String getTableFileContents(Path path) throws Exception {
FileSystem fs = path.getFileSystem(conf);
FileStatus[] files = fs.listStatus(path);
if (files == null || files.length == 0) {
return "";
}
StringBuilder sb = new StringBuilder();
byte[] buf = new byte[1024];
for (FileStatus file: files) {
if (file.isDirectory()) {
sb.append(getTableFileContents(file.getPath()));
continue;
}
try (InputStream in = fs.open(file.getPath())) {
while (true) {
int readBytes = in.read(buf);
if (readBytes <= 0) {
break;
}
sb.append(new String(buf, 0, readBytes));
}
}
}
return sb.toString();
}
/**
* Reads data file from Test Cluster's HDFS
* @param tableName
* @return data file's contents
* @throws Exception
*/
public String getTableFileContents(String tableName) throws Exception {
TableDesc tableDesc = testingCluster.getMaster().getCatalog().getTableDesc(getCurrentDatabase(), tableName);
if (tableDesc == null) {
return null;
}
Path path = new Path(tableDesc.getUri());
return getTableFileContents(path);
}
public List<Path> listTableFiles(String tableName) throws Exception {
TableDesc tableDesc = testingCluster.getMaster().getCatalog().getTableDesc(getCurrentDatabase(), tableName);
if (tableDesc == null) {
return null;
}
Path path = new Path(tableDesc.getUri());
FileSystem fs = path.getFileSystem(conf);
return listFiles(fs, path);
}
private List<Path> listFiles(FileSystem fs, Path path) throws Exception {
List<Path> result = new ArrayList<>();
FileStatus[] files = fs.listStatus(path);
if (files == null || files.length == 0) {
return result;
}
for (FileStatus eachFile: files) {
if (eachFile.isDirectory()) {
result.addAll(listFiles(fs, eachFile.getPath()));
} else {
result.add(eachFile.getPath());
}
}
return result;
}
public static QueryId getQueryId(ResultSet resultSet) {
if (resultSet instanceof TajoMemoryResultSet) {
return ((TajoMemoryResultSet) resultSet).getQueryId();
} else if (resultSet instanceof FetchResultSet) {
return ((FetchResultSet) resultSet).getQueryId();
} else {
throw new IllegalArgumentException(resultSet.toString());
}
}
}