/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.twitter.distributedlog;
import com.twitter.distributedlog.impl.BKLogSegmentEntryWriter;
import com.twitter.distributedlog.metadata.BKDLConfig;
import com.twitter.distributedlog.metadata.DLMetadata;
import com.twitter.distributedlog.namespace.DistributedLogNamespace;
import com.twitter.distributedlog.namespace.DistributedLogNamespaceBuilder;
import com.twitter.distributedlog.util.ConfUtils;
import com.twitter.distributedlog.util.FutureUtils;
import com.twitter.distributedlog.util.PermitLimiter;
import com.twitter.distributedlog.util.RetryPolicyUtils;
import com.twitter.distributedlog.util.Utils;
import com.twitter.util.Await;
import com.twitter.util.Duration;
import com.twitter.util.Future;
import org.apache.bookkeeper.client.BookKeeper;
import org.apache.bookkeeper.client.LedgerHandle;
import org.apache.bookkeeper.conf.ServerConfiguration;
import org.apache.bookkeeper.feature.SettableFeatureProvider;
import org.apache.bookkeeper.stats.NullStatsLogger;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.ZooDefs;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.net.URI;
import java.net.URL;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static com.google.common.base.Charsets.UTF_8;
import static org.junit.Assert.assertNotNull;
/**
* Utility class for setting up bookkeeper ensembles
* and bringing individual bookies up and down
*/
public class DLMTestUtil {
protected static final Logger LOG = LoggerFactory.getLogger(DLMTestUtil.class);
private final static byte[] payloadStatic = repeatString("abc", 512).getBytes();
static String repeatString(String s, int n) {
String ret = s;
for(int i = 1; i < n; i++) {
ret += s;
}
return ret;
}
public static Map<Long, LogSegmentMetadata> readLogSegments(ZooKeeperClient zkc, String ledgerPath) throws Exception {
List<String> children = zkc.get().getChildren(ledgerPath, false);
LOG.info("Children under {} : {}", ledgerPath, children);
Map<Long, LogSegmentMetadata> segments =
new HashMap<Long, LogSegmentMetadata>(children.size());
for (String child : children) {
LogSegmentMetadata segment =
FutureUtils.result(LogSegmentMetadata.read(zkc, ledgerPath + "/" + child));
LOG.info("Read segment {} : {}", child, segment);
segments.put(segment.getLogSegmentSequenceNumber(), segment);
}
return segments;
}
static void updateBKDLConfig(URI uri, String zkServers, String ledgersPath, boolean sanityCheckTxnID) throws Exception {
BKDLConfig bkdlConfig = new BKDLConfig(zkServers, ledgersPath).setSanityCheckTxnID(sanityCheckTxnID);
DLMetadata.create(bkdlConfig).update(uri);
}
public static URI createDLMURI(int port, String path) throws Exception {
return LocalDLMEmulator.createDLMURI("127.0.0.1:" + port, path);
}
public static DistributedLogManager createNewDLM(String name,
DistributedLogConfiguration conf,
URI uri) throws Exception {
DistributedLogNamespace namespace = DistributedLogNamespaceBuilder.newBuilder()
.conf(conf).uri(uri).build();
return namespace.openLog(name);
}
static MetadataAccessor createNewMetadataAccessor(DistributedLogConfiguration conf,
String name,
URI uri) throws Exception {
// TODO: Metadata Accessor seems to be a legacy object which only used by kestrel
// (we might consider deprecating this)
BKDistributedLogNamespace namespace = BKDistributedLogNamespace.newBuilder()
.conf(conf).uri(uri).build();
return namespace.createMetadataAccessor(name);
}
public static class BKLogPartitionWriteHandlerAndClients {
private BKLogWriteHandler writeHandler;
private ZooKeeperClient zooKeeperClient;
private BookKeeperClient bookKeeperClient;
public BKLogPartitionWriteHandlerAndClients(BKLogWriteHandler writeHandler, ZooKeeperClient zooKeeperClient, BookKeeperClient bookKeeperClient) {
this.writeHandler = writeHandler;
this.zooKeeperClient = zooKeeperClient;
this.bookKeeperClient = bookKeeperClient;
}
public void close() {
bookKeeperClient.close();
zooKeeperClient.close();
Utils.closeQuietly(writeHandler);
}
public BKLogWriteHandler getWriteHandler() {
return writeHandler;
}
}
static BKLogPartitionWriteHandlerAndClients createNewBKDLM(DistributedLogConfiguration conf,
String logName,
int zkPort) throws Exception {
URI uri = createDLMURI(zkPort, "/" + logName);
ZooKeeperClientBuilder zkcBuilder = TestZooKeeperClientBuilder.newBuilder(conf)
.name(String.format("dlzk:%s:handler_dedicated", logName))
.uri(uri);
ZooKeeperClient zkClient = zkcBuilder.build();
try {
zkClient.get().create(uri.getPath(), new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
} catch (KeeperException.NodeExistsException nee) {
// ignore
}
// resolve uri
BKDLConfig bkdlConfig = BKDLConfig.resolveDLConfig(zkClient, uri);
BKDLConfig.propagateConfiguration(bkdlConfig, conf);
BookKeeperClientBuilder bkcBuilder = BookKeeperClientBuilder.newBuilder()
.dlConfig(conf)
.name(String.format("bk:%s:handler_dedicated", logName))
.zkServers(bkdlConfig.getBkZkServersForWriter())
.ledgersPath(bkdlConfig.getBkLedgersPath())
.statsLogger(NullStatsLogger.INSTANCE);
BKDistributedLogManager bkdlm = new BKDistributedLogManager(
logName,
conf,
uri,
zkcBuilder,
zkcBuilder,
zkClient,
zkClient,
bkcBuilder,
bkcBuilder,
new SettableFeatureProvider("", 0),
PermitLimiter.NULL_PERMIT_LIMITER,
NullStatsLogger.INSTANCE);
BKLogWriteHandler writeHandler = bkdlm.createWriteHandler(true);
return new BKLogPartitionWriteHandlerAndClients(writeHandler, zkClient, bkcBuilder.build());
}
public static void fenceStream(DistributedLogConfiguration conf, URI uri, String name) throws Exception {
BKDistributedLogManager dlm = (BKDistributedLogManager) createNewDLM(name, conf, uri);
try {
BKLogReadHandler readHandler = dlm.createReadHandler();
List<LogSegmentMetadata> ledgerList = readHandler.getFullLedgerList(true, true);
LogSegmentMetadata lastSegment = ledgerList.get(ledgerList.size() - 1);
BookKeeperClient bkc = dlm.getWriterBKC();
LedgerHandle lh = bkc.get().openLedger(lastSegment.getLedgerId(),
BookKeeper.DigestType.CRC32, conf.getBKDigestPW().getBytes(UTF_8));
lh.close();
} finally {
dlm.close();
}
}
static long getNumberofLogRecords(DistributedLogManager bkdlm, long startTxId) throws IOException {
long numLogRecs = 0;
LogReader reader = bkdlm.getInputStream(startTxId);
LogRecord record = reader.readNext(false);
while (null != record) {
numLogRecs++;
verifyLogRecord(record);
record = reader.readNext(false);
}
reader.close();
return numLogRecs;
}
public static LogRecord getLogRecordInstance(long txId) {
return new LogRecord(txId, generatePayload(txId));
}
public static LogRecord getLogRecordInstance(long txId, int size) {
ByteBuffer buf = ByteBuffer.allocate(size);
return new LogRecord(txId, buf.array());
}
public static void verifyLogRecord(LogRecord record) {
assertEquals(generatePayload(record.getTransactionId()).length, record.getPayload().length);
assertArrayEquals(generatePayload(record.getTransactionId()), record.getPayload());
assert(!record.isControl());
verifyPayload(record.getTransactionId(), record.getPayload());
}
static byte[] generatePayload(long txId) {
return String.format("%d;%d", txId, txId).getBytes();
}
static void verifyPayload(long txId, byte[] payload) {
String[] txIds = new String(payload).split(";");
assertEquals(Long.valueOf(txIds[0]), Long.valueOf(txIds[0]));
}
static LogRecord getLargeLogRecordInstance(long txId, boolean control) {
LogRecord record = new LogRecord(txId, payloadStatic);
if (control) {
record.setControl();
}
return record;
}
static LogRecord getLargeLogRecordInstance(long txId) {
return new LogRecord(txId, payloadStatic);
}
static List<LogRecord> getLargeLogRecordInstanceList(long firstTxId, int count) {
List<LogRecord> logrecs = new ArrayList<LogRecord>(count);
for (long i = 0; i < count; i++) {
logrecs.add(getLargeLogRecordInstance(firstTxId + i));
}
return logrecs;
}
static List<LogRecord> getLogRecordInstanceList(long firstTxId, int count, int size) {
List<LogRecord> logrecs = new ArrayList<LogRecord>(count);
for (long i = 0; i < count; i++) {
logrecs.add(getLogRecordInstance(firstTxId + i, size));
}
return logrecs;
}
static void verifyLargeLogRecord(LogRecord record) {
verifyLargeLogRecord(record.getPayload());
}
static void verifyLargeLogRecord(byte[] payload) {
assertArrayEquals(payloadStatic, payload);
}
static LogRecord getEmptyLogRecordInstance(long txId) {
return new LogRecord(txId, new byte[0]);
}
static void verifyEmptyLogRecord(LogRecord record) {
assert(record.getPayload().length == 0);
}
public static LogRecordWithDLSN getLogRecordWithDLSNInstance(DLSN dlsn, long txId) {
return getLogRecordWithDLSNInstance(dlsn, txId, false);
}
public static LogRecordWithDLSN getLogRecordWithDLSNInstance(DLSN dlsn, long txId, boolean isControlRecord) {
LogRecordWithDLSN record = new LogRecordWithDLSN(dlsn, txId, generatePayload(txId), 1L);
record.setPositionWithinLogSegment((int)txId + 1);
if (isControlRecord) {
record.setControl();
}
return record;
}
public static String inprogressZNodeName(long logSegmentSeqNo) {
return String.format("%s_%018d", DistributedLogConstants.INPROGRESS_LOGSEGMENT_PREFIX, logSegmentSeqNo);
}
public static String completedLedgerZNodeNameWithVersion(long ledgerId, long firstTxId, long lastTxId, long logSegmentSeqNo) {
return String.format("%s_%018d_%018d_%018d_v%dl%d_%04d", DistributedLogConstants.COMPLETED_LOGSEGMENT_PREFIX,
firstTxId, lastTxId, logSegmentSeqNo, DistributedLogConstants.LOGSEGMENT_NAME_VERSION, ledgerId,
DistributedLogConstants.LOCAL_REGION_ID);
}
public static String completedLedgerZNodeNameWithTxID(long firstTxId, long lastTxId) {
return String.format("%s_%018d_%018d", DistributedLogConstants.COMPLETED_LOGSEGMENT_PREFIX, firstTxId, lastTxId);
}
public static String completedLedgerZNodeNameWithLogSegmentSequenceNumber(long logSegmentSeqNo) {
return String.format("%s_%018d", DistributedLogConstants.COMPLETED_LOGSEGMENT_PREFIX, logSegmentSeqNo);
}
public static LogSegmentMetadata inprogressLogSegment(String ledgerPath,
long ledgerId,
long firstTxId,
long logSegmentSeqNo) {
return inprogressLogSegment(ledgerPath, ledgerId, firstTxId, logSegmentSeqNo,
LogSegmentMetadata.LEDGER_METADATA_CURRENT_LAYOUT_VERSION);
}
public static LogSegmentMetadata inprogressLogSegment(String ledgerPath,
long ledgerId,
long firstTxId,
long logSegmentSeqNo,
int version) {
return new LogSegmentMetadata.LogSegmentMetadataBuilder(
ledgerPath + "/" + inprogressZNodeName(logSegmentSeqNo),
version,
ledgerId,
firstTxId)
.setLogSegmentSequenceNo(logSegmentSeqNo)
.build();
}
public static LogSegmentMetadata completedLogSegment(String ledgerPath,
long ledgerId,
long firstTxId,
long lastTxId,
int recordCount,
long logSegmentSeqNo,
long lastEntryId,
long lastSlotId) {
return completedLogSegment(ledgerPath, ledgerId, firstTxId, lastTxId,
recordCount, logSegmentSeqNo, lastEntryId, lastSlotId,
LogSegmentMetadata.LEDGER_METADATA_CURRENT_LAYOUT_VERSION);
}
public static LogSegmentMetadata completedLogSegment(String ledgerPath,
long ledgerId,
long firstTxId,
long lastTxId,
int recordCount,
long logSegmentSeqNo,
long lastEntryId,
long lastSlotId,
int version) {
LogSegmentMetadata metadata =
new LogSegmentMetadata.LogSegmentMetadataBuilder(
ledgerPath + "/" + inprogressZNodeName(logSegmentSeqNo),
version,
ledgerId,
firstTxId)
.setInprogress(false)
.setLogSegmentSequenceNo(logSegmentSeqNo)
.build();
return metadata.completeLogSegment(ledgerPath + "/" + completedLedgerZNodeNameWithLogSegmentSequenceNumber(logSegmentSeqNo),
lastTxId, recordCount, lastEntryId, lastSlotId, firstTxId);
}
public static void generateCompletedLogSegments(DistributedLogManager manager, DistributedLogConfiguration conf,
long numCompletedSegments, long segmentSize) throws Exception {
BKDistributedLogManager dlm = (BKDistributedLogManager) manager;
long txid = 1L;
for (long i = 0; i < numCompletedSegments; i++) {
BKSyncLogWriter writer = dlm.startLogSegmentNonPartitioned();
for (long j = 1; j <= segmentSize; j++) {
writer.write(DLMTestUtil.getLogRecordInstance(txid++));
}
writer.closeAndComplete();
}
}
public static long generateLogSegmentNonPartitioned(DistributedLogManager dlm, int controlEntries, int userEntries, long startTxid)
throws Exception {
return generateLogSegmentNonPartitioned(dlm, controlEntries, userEntries, startTxid, 1L);
}
public static long generateLogSegmentNonPartitioned(DistributedLogManager dlm, int controlEntries, int userEntries, long startTxid, long txidStep) throws Exception {
AsyncLogWriter out = dlm.startAsyncLogSegmentNonPartitioned();
long txid = startTxid;
for (int i = 0; i < controlEntries; ++i) {
LogRecord record = DLMTestUtil.getLargeLogRecordInstance(txid);
record.setControl();
Await.result(out.write(record));
txid += txidStep;
}
for (int i = 0; i < userEntries; ++i) {
LogRecord record = DLMTestUtil.getLargeLogRecordInstance(txid);
Await.result(out.write(record));
txid += txidStep;
}
Utils.close(out);
return txid - startTxid;
}
public static void injectLogSegmentWithGivenLogSegmentSeqNo(DistributedLogManager manager, DistributedLogConfiguration conf,
long logSegmentSeqNo, long startTxID, boolean writeEntries, long segmentSize,
boolean completeLogSegment)
throws Exception {
BKDistributedLogManager dlm = (BKDistributedLogManager) manager;
BKLogWriteHandler writeHandler = dlm.createWriteHandler(false);
FutureUtils.result(writeHandler.lockHandler());
// Start a log segment with a given ledger seq number.
BookKeeperClient bkc = dlm.getWriterBKC();
LedgerHandle lh = bkc.get().createLedger(conf.getEnsembleSize(), conf.getWriteQuorumSize(),
conf.getAckQuorumSize(), BookKeeper.DigestType.CRC32, conf.getBKDigestPW().getBytes());
String inprogressZnodeName = writeHandler.inprogressZNodeName(lh.getId(), startTxID, logSegmentSeqNo);
String znodePath = writeHandler.inprogressZNode(lh.getId(), startTxID, logSegmentSeqNo);
LogSegmentMetadata l =
new LogSegmentMetadata.LogSegmentMetadataBuilder(znodePath,
conf.getDLLedgerMetadataLayoutVersion(), lh.getId(), startTxID)
.setLogSegmentSequenceNo(logSegmentSeqNo)
.build();
l.write(dlm.writerZKC);
writeHandler.maxTxId.store(startTxID);
writeHandler.addLogSegmentToCache(inprogressZnodeName, l);
BKLogSegmentWriter writer = new BKLogSegmentWriter(
writeHandler.getFullyQualifiedName(),
inprogressZnodeName,
conf,
conf.getDLLedgerMetadataLayoutVersion(),
new BKLogSegmentEntryWriter(lh),
writeHandler.lock,
startTxID,
logSegmentSeqNo,
writeHandler.scheduler,
writeHandler.statsLogger,
writeHandler.statsLogger,
writeHandler.alertStatsLogger,
PermitLimiter.NULL_PERMIT_LIMITER,
new SettableFeatureProvider("", 0),
ConfUtils.getConstDynConf(conf));
if (writeEntries) {
long txid = startTxID;
for (long j = 1; j <= segmentSize; j++) {
writer.write(DLMTestUtil.getLogRecordInstance(txid++));
}
FutureUtils.result(writer.flushAndCommit());
}
if (completeLogSegment) {
FutureUtils.result(writeHandler.completeAndCloseLogSegment(writer));
}
FutureUtils.result(writeHandler.unlockHandler());
}
public static void injectLogSegmentWithLastDLSN(DistributedLogManager manager, DistributedLogConfiguration conf,
long logSegmentSeqNo, long startTxID, long segmentSize,
boolean recordWrongLastDLSN) throws Exception {
BKDistributedLogManager dlm = (BKDistributedLogManager) manager;
BKLogWriteHandler writeHandler = dlm.createWriteHandler(false);
FutureUtils.result(writeHandler.lockHandler());
// Start a log segment with a given ledger seq number.
BookKeeperClient bkc = dlm.getReaderBKC();
LedgerHandle lh = bkc.get().createLedger(conf.getEnsembleSize(), conf.getWriteQuorumSize(),
conf.getAckQuorumSize(), BookKeeper.DigestType.CRC32, conf.getBKDigestPW().getBytes());
String inprogressZnodeName = writeHandler.inprogressZNodeName(lh.getId(), startTxID, logSegmentSeqNo);
String znodePath = writeHandler.inprogressZNode(lh.getId(), startTxID, logSegmentSeqNo);
LogSegmentMetadata l =
new LogSegmentMetadata.LogSegmentMetadataBuilder(znodePath,
conf.getDLLedgerMetadataLayoutVersion(), lh.getId(), startTxID)
.setLogSegmentSequenceNo(logSegmentSeqNo)
.setInprogress(false)
.build();
l.write(dlm.writerZKC);
writeHandler.maxTxId.store(startTxID);
writeHandler.addLogSegmentToCache(inprogressZnodeName, l);
BKLogSegmentWriter writer = new BKLogSegmentWriter(
writeHandler.getFullyQualifiedName(),
inprogressZnodeName,
conf,
conf.getDLLedgerMetadataLayoutVersion(),
new BKLogSegmentEntryWriter(lh),
writeHandler.lock,
startTxID,
logSegmentSeqNo,
writeHandler.scheduler,
writeHandler.statsLogger,
writeHandler.statsLogger,
writeHandler.alertStatsLogger,
PermitLimiter.NULL_PERMIT_LIMITER,
new SettableFeatureProvider("", 0),
ConfUtils.getConstDynConf(conf));
long txid = startTxID;
DLSN wrongDLSN = null;
for (long j = 1; j <= segmentSize; j++) {
DLSN dlsn = Await.result(writer.asyncWrite(DLMTestUtil.getLogRecordInstance(txid++)));
if (j == (segmentSize - 1)) {
wrongDLSN = dlsn;
}
}
assertNotNull(wrongDLSN);
if (recordWrongLastDLSN) {
FutureUtils.result(writer.asyncClose());
writeHandler.completeAndCloseLogSegment(
writeHandler.inprogressZNodeName(writer.getLogSegmentId(), writer.getStartTxId(), writer.getLogSegmentSequenceNumber()),
writer.getLogSegmentSequenceNumber(),
writer.getLogSegmentId(),
writer.getStartTxId(),
startTxID + segmentSize - 2,
writer.getPositionWithinLogSegment() - 1,
wrongDLSN.getEntryId(),
wrongDLSN.getSlotId());
} else {
FutureUtils.result(writeHandler.completeAndCloseLogSegment(writer));
}
FutureUtils.result(writeHandler.unlockHandler());
}
public static void updateSegmentMetadata(ZooKeeperClient zkc, LogSegmentMetadata segment) throws Exception {
byte[] finalisedData = segment.getFinalisedData().getBytes(UTF_8);
zkc.get().setData(segment.getZkPath(), finalisedData, -1);
}
public static ServerConfiguration loadTestBkConf() {
ServerConfiguration conf = new ServerConfiguration();
ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
URL confUrl = classLoader.getResource("bk_server.conf");
try {
if (null != confUrl) {
conf.loadConf(confUrl);
LOG.info("loaded bk_server.conf from resources");
}
} catch (org.apache.commons.configuration.ConfigurationException ex) {
LOG.warn("loading conf failed", ex);
}
conf.setAllowLoopback(true);
return conf;
}
public static <T> void validateFutureFailed(Future<T> future, Class exClass) {
try {
Await.result(future);
} catch (Exception ex) {
LOG.info("Expected: {} Actual: {}", exClass.getName(), ex.getClass().getName());
assertTrue("exceptions types equal", exClass.isInstance(ex));
}
}
public static <T> T validateFutureSucceededAndGetResult(Future<T> future) throws Exception {
try {
return Await.result(future, Duration.fromSeconds(10));
} catch (Exception ex) {
fail("unexpected exception " + ex.getClass().getName());
throw ex;
}
}
}