/*
* Copyright (C) 2015 hops.io.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.hops.leaderElection;
import io.hops.exception.StorageException;
import io.hops.exception.StorageInitializtionException;
import io.hops.leaderElection.experiments.LightWeightNameNode;
import io.hops.metadata.LEStorageFactory;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.log4j.Level;
import org.apache.log4j.LogManager;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Random;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
public class TestLESpeed {
private static final Log LOG = LogFactory.getLog(TestLESpeed.class);
Configuration conf = null;
List<LightWeightNameNode> nnList;
private final int DFS_LEADER_CHECK_INTERVAL_IN_MS = 2 * 1000;
private final int DFS_LEADER_MISSED_HB_THRESHOLD = 2;
private final String HTTP_ADDRESS = "dummy.address.com:9999";
private final String RPC_ADDRESS = "repc.server.ip:0000";
private final String DRIVER_JAR = "";
private final String DRIVER_CLASS = "io.hops.metadata.ndb.NdbStorageFactory";
private final String DFS_STORAGE_DRIVER_CONFIG_FILE = "ndb-config.properties";
private final long TIME_PERIOD_INCREMENT = 200;
@Before
public void init()
throws StorageInitializtionException, StorageException, IOException,
ClassNotFoundException {
LogManager.getRootLogger().setLevel(Level.ALL);
nnList = new ArrayList<LightWeightNameNode>();
LEStorageFactory.setConfiguration(DRIVER_JAR, DRIVER_CLASS,
DFS_STORAGE_DRIVER_CONFIG_FILE);
LEStorageFactory.formatStorage();
VarsRegister.registerHdfsDefaultValues();
}
@After
public void tearDown() {
//stop all NN
LOG.debug("tearDown");
for (LightWeightNameNode nn : nnList) {
nn.stop();
}
}
/**
*
*/
@Test
public void testLeaderElectionSpeed()
throws IOException, InterruptedException, CloneNotSupportedException {
//create 10 NN
Random rand = new Random(System.currentTimeMillis());
for (int i = 0; i < 50; i++) {
LightWeightNameNode nn = startAProcess();
nnList.add(nn);
Thread.sleep(rand.nextInt(DFS_LEADER_CHECK_INTERVAL_IN_MS *
(DFS_LEADER_MISSED_HB_THRESHOLD + 1)));
}
// kill leader and start new nodes
// see how long does it take to elect a new leader
LOG.debug("I am test. started all nodes");
final long timeToRun = 3 * 60 * 1000;
long startTime = System.currentTimeMillis();
long lastLeaderKillTime = 0;
final long killLeaderAfter = 10 * 1000;
LightWeightNameNode leader = null;
while ((System.currentTimeMillis() - startTime) < timeToRun) {
//verify that there is one and only one leader.
int leaderCount = 0;
for (int i = 0; i < nnList.size(); i++) {
if (nnList.get(i).isLeader()) {
leaderCount++;
assertTrue("Wrong number of leaders. Found " + leaderCount,
leaderCount <= 1);
if (leader != null && lastLeaderKillTime > 0 &&
leader.getLeCurrentId() != nnList.get(i).getLeCurrentId()) {
Long failOverTime = System.currentTimeMillis() - lastLeaderKillTime;
long failOverLowerBound = DFS_LEADER_CHECK_INTERVAL_IN_MS;
long failOverUpperBound = DFS_LEADER_CHECK_INTERVAL_IN_MS *
(DFS_LEADER_MISSED_HB_THRESHOLD + 1);
//assertTrue("Wrong failover time "+failOverTime, failOverTime>failOverLowerBound && failOverTime < failOverUpperBound); no longer applicable
LOG.debug("I am Test. Old Leader Id " + leader.getLeCurrentId() +
" new Leader Id " + nnList.get(i).getLeCurrentId() +
" New leader elected in " + (failOverTime) + " ms.");
}
leader = nnList.get(i);
}
}
assertTrue("Wrong number of leaders. Found " + leaderCount,
leaderCount <= 1);
if ((System.currentTimeMillis() - lastLeaderKillTime) > killLeaderAfter &&
leaderCount == 1) {
LOG.debug("I am test. Starting a new Process ");
LightWeightNameNode nn = startAProcess();
nnList.add(nn);
if (leader.isLeader()) {
LOG.debug("I am test. Stopping the leader process ");
leader.getLeaderElectionInstance().pauseFor(Long.MAX_VALUE);
lastLeaderKillTime = System.currentTimeMillis();
} else {
LOG.debug("I am test. Leader has already been evicted id : " +
leader.getLeCurrentId());
continue;
}
}
}
}
public LightWeightNameNode startAProcess() {
int tries = 10;
while (tries >= 0) {
tries--;
try {
return new LightWeightNameNode(new HdfsLeDescriptorFactory(),
DFS_LEADER_CHECK_INTERVAL_IN_MS, DFS_LEADER_MISSED_HB_THRESHOLD,
TIME_PERIOD_INCREMENT, HTTP_ADDRESS, RPC_ADDRESS);
} catch (Exception e) {
LOG.debug(
"Could not create a process. Retrying ... Exception was " + e);
e.printStackTrace();
}
}
fail("Failed to spawn a new process.");
return null;
}
}