/*
* SonarQube
* Copyright (C) 2009-2017 SonarSource SA
* mailto:info AT sonarsource DOT com
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.sonar.application;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.EnumMap;
import java.util.List;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import org.junit.After;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.DisableOnDebug;
import org.junit.rules.ExpectedException;
import org.junit.rules.TestRule;
import org.junit.rules.Timeout;
import org.mockito.Mockito;
import org.sonar.application.config.TestAppSettings;
import org.sonar.application.process.JavaCommand;
import org.sonar.application.process.JavaCommandFactory;
import org.sonar.application.process.JavaProcessLauncher;
import org.sonar.application.process.ProcessMonitor;
import org.sonar.process.ProcessId;
import org.sonar.process.ProcessProperties;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.timeout;
import static org.mockito.Mockito.verify;
import static org.sonar.process.ProcessId.COMPUTE_ENGINE;
import static org.sonar.process.ProcessId.ELASTICSEARCH;
import static org.sonar.process.ProcessId.WEB_SERVER;
public class SchedulerImplTest {
private static final JavaCommand ES_COMMAND = new JavaCommand(ELASTICSEARCH);
private static final JavaCommand WEB_LEADER_COMMAND = new JavaCommand(WEB_SERVER);
private static final JavaCommand WEB_FOLLOWER_COMMAND = new JavaCommand(WEB_SERVER);
private static final JavaCommand CE_COMMAND = new JavaCommand(COMPUTE_ENGINE);
@Rule
public TestRule safeguardTimeout = new DisableOnDebug(Timeout.seconds(60));
@Rule
public ExpectedException expectedException = ExpectedException.none();
private AppReloader appReloader = mock(AppReloader.class);
private TestAppSettings settings = new TestAppSettings();
private TestJavaCommandFactory javaCommandFactory = new TestJavaCommandFactory();
private TestJavaProcessLauncher processLauncher = new TestJavaProcessLauncher();
private TestAppState appState = new TestAppState();
private List<ProcessId> orderedStops = new ArrayList<>();
@After
public void tearDown() throws Exception {
processLauncher.close();
}
@Test
public void start_and_stop_sequence_of_ES_WEB_CE_in_order() throws Exception {
enableAllProcesses();
SchedulerImpl underTest = newScheduler();
underTest.schedule();
// elasticsearch does not have preconditions to start
TestProcess es = processLauncher.waitForProcess(ELASTICSEARCH);
assertThat(es.isAlive()).isTrue();
assertThat(processLauncher.processes).hasSize(1);
// elasticsearch becomes operational -> web leader is starting
es.operational = true;
waitForAppStateOperational(ELASTICSEARCH);
TestProcess web = processLauncher.waitForProcess(WEB_SERVER);
assertThat(web.isAlive()).isTrue();
assertThat(processLauncher.processes).hasSize(2);
assertThat(processLauncher.commands).containsExactly(ES_COMMAND, WEB_LEADER_COMMAND);
// web becomes operational -> CE is starting
web.operational = true;
waitForAppStateOperational(WEB_SERVER);
TestProcess ce = processLauncher.waitForProcess(COMPUTE_ENGINE);
assertThat(ce.isAlive()).isTrue();
assertThat(processLauncher.processes).hasSize(3);
assertThat(processLauncher.commands).containsExactly(ES_COMMAND, WEB_LEADER_COMMAND, CE_COMMAND);
// all processes are up
processLauncher.processes.values().forEach(p -> assertThat(p.isAlive()).isTrue());
// processes are stopped in reverse order of startup
underTest.terminate();
assertThat(orderedStops).containsExactly(COMPUTE_ENGINE, WEB_SERVER, ELASTICSEARCH);
processLauncher.processes.values().forEach(p -> assertThat(p.isAlive()).isFalse());
// does nothing because scheduler is already terminated
underTest.awaitTermination();
}
private void enableAllProcesses() {
settings.set(ProcessProperties.CLUSTER_ENABLED, "true");
}
@Test
public void all_processes_are_stopped_if_one_process_goes_down() throws Exception {
Scheduler underTest = startAll();
processLauncher.waitForProcess(WEB_SERVER).destroyForcibly();
underTest.awaitTermination();
assertThat(orderedStops).containsExactly(WEB_SERVER, COMPUTE_ENGINE, ELASTICSEARCH);
processLauncher.processes.values().forEach(p -> assertThat(p.isAlive()).isFalse());
// following does nothing
underTest.terminate();
underTest.awaitTermination();
}
@Test
public void all_processes_are_stopped_if_one_process_fails_to_start() throws Exception {
enableAllProcesses();
SchedulerImpl underTest = newScheduler();
processLauncher.makeStartupFail = COMPUTE_ENGINE;
underTest.schedule();
processLauncher.waitForProcess(ELASTICSEARCH).operational = true;
processLauncher.waitForProcess(WEB_SERVER).operational = true;
underTest.awaitTermination();
assertThat(orderedStops).containsExactly(WEB_SERVER, ELASTICSEARCH);
processLauncher.processes.values().forEach(p -> assertThat(p.isAlive()).isFalse());
}
@Test
public void terminate_can_be_called_multiple_times() throws Exception {
Scheduler underTest = startAll();
underTest.terminate();
processLauncher.processes.values().forEach(p -> assertThat(p.isAlive()).isFalse());
// does nothing
underTest.terminate();
}
@Test
public void awaitTermination_blocks_until_all_processes_are_stopped() throws Exception {
Scheduler underTest = startAll();
Thread awaitingTermination = new Thread(() -> underTest.awaitTermination());
awaitingTermination.start();
assertThat(awaitingTermination.isAlive()).isTrue();
underTest.terminate();
// the thread is being stopped
awaitingTermination.join();
assertThat(awaitingTermination.isAlive()).isFalse();
}
@Test
public void restart_reloads_java_commands_and_restarts_all_processes() throws Exception {
Scheduler underTest = startAll();
processLauncher.waitForProcess(WEB_SERVER).askedForRestart = true;
// waiting for all processes to be stopped
boolean stopped = false;
while (!stopped) {
stopped = orderedStops.size() == 3;
Thread.sleep(1L);
}
// restarting
verify(appReloader, timeout(60_000)).reload(settings);
processLauncher.waitForProcessAlive(ELASTICSEARCH);
processLauncher.waitForProcessAlive(COMPUTE_ENGINE);
processLauncher.waitForProcessAlive(WEB_SERVER);
underTest.terminate();
// 3+3 processes have been stopped
assertThat(orderedStops).hasSize(6);
assertThat(processLauncher.waitForProcess(ELASTICSEARCH).isAlive()).isFalse();
assertThat(processLauncher.waitForProcess(COMPUTE_ENGINE).isAlive()).isFalse();
assertThat(processLauncher.waitForProcess(WEB_SERVER).isAlive()).isFalse();
// verify that awaitTermination() does not block
underTest.awaitTermination();
}
@Test
public void restart_stops_all_if_new_settings_are_not_allowed() throws Exception {
Scheduler underTest = startAll();
doThrow(new IllegalStateException("reload error")).when(appReloader).reload(settings);
processLauncher.waitForProcess(WEB_SERVER).askedForRestart = true;
// waiting for all processes to be stopped
processLauncher.waitForProcessDown(ELASTICSEARCH);
processLauncher.waitForProcessDown(COMPUTE_ENGINE);
processLauncher.waitForProcessDown(WEB_SERVER);
// verify that awaitTermination() does not block
underTest.awaitTermination();
}
@Test
public void web_follower_starts_only_when_web_leader_is_operational() throws Exception {
// leader takes the lock, so underTest won't get it
assertThat(appState.tryToLockWebLeader()).isTrue();
appState.setOperational(ProcessId.ELASTICSEARCH);
enableAllProcesses();
SchedulerImpl underTest = newScheduler();
underTest.schedule();
processLauncher.waitForProcessAlive(ProcessId.ELASTICSEARCH);
assertThat(processLauncher.processes).hasSize(1);
// leader becomes operational -> follower can start
appState.setOperational(ProcessId.WEB_SERVER);
processLauncher.waitForProcessAlive(WEB_SERVER);
underTest.terminate();
}
@Test
public void web_server_waits_for_remote_elasticsearch_to_be_started_if_local_es_is_disabled() throws Exception {
settings.set(ProcessProperties.CLUSTER_ENABLED, "true");
settings.set(ProcessProperties.CLUSTER_SEARCH_DISABLED, "true");
SchedulerImpl underTest = newScheduler();
underTest.schedule();
// WEB and CE wait for ES to be up
assertThat(processLauncher.processes).isEmpty();
// ES becomes operational on another node -> web leader can start
appState.setRemoteOperational(ProcessId.ELASTICSEARCH);
processLauncher.waitForProcessAlive(WEB_SERVER);
assertThat(processLauncher.processes).hasSize(1);
underTest.terminate();
}
@Test
public void compute_engine_waits_for_remote_elasticsearch_and_web_leader_to_be_started_if_local_es_is_disabled() throws Exception {
settings.set(ProcessProperties.CLUSTER_ENABLED, "true");
settings.set(ProcessProperties.CLUSTER_SEARCH_DISABLED, "true");
settings.set(ProcessProperties.CLUSTER_WEB_DISABLED, "true");
SchedulerImpl underTest = newScheduler();
underTest.schedule();
// CE waits for ES and WEB leader to be up
assertThat(processLauncher.processes).isEmpty();
// ES and WEB leader become operational on another nodes -> CE can start
appState.setRemoteOperational(ProcessId.ELASTICSEARCH);
appState.setRemoteOperational(ProcessId.WEB_SERVER);
processLauncher.waitForProcessAlive(COMPUTE_ENGINE);
assertThat(processLauncher.processes).hasSize(1);
underTest.terminate();
}
private SchedulerImpl newScheduler() {
return new SchedulerImpl(settings, appReloader, javaCommandFactory, processLauncher, appState)
.setProcessWatcherDelayMs(1L);
}
private Scheduler startAll() throws InterruptedException {
enableAllProcesses();
SchedulerImpl scheduler = newScheduler();
scheduler.schedule();
processLauncher.waitForProcess(ELASTICSEARCH).operational = true;
processLauncher.waitForProcess(WEB_SERVER).operational = true;
processLauncher.waitForProcess(COMPUTE_ENGINE).operational = true;
return scheduler;
}
private void waitForAppStateOperational(ProcessId id) throws InterruptedException {
while (true) {
if (appState.isOperational(id, true)) {
return;
}
Thread.sleep(1L);
}
}
private static class TestJavaCommandFactory implements JavaCommandFactory {
@Override
public JavaCommand createEsCommand() {
return ES_COMMAND;
}
@Override
public JavaCommand createWebCommand(boolean leader) {
return leader ? WEB_LEADER_COMMAND : WEB_FOLLOWER_COMMAND;
}
@Override
public JavaCommand createCeCommand() {
return CE_COMMAND;
}
}
private class TestJavaProcessLauncher implements JavaProcessLauncher {
private final EnumMap<ProcessId, TestProcess> processes = new EnumMap<>(ProcessId.class);
private final List<JavaCommand> commands = new ArrayList<>();
private ProcessId makeStartupFail = null;
@Override
public ProcessMonitor launch(JavaCommand javaCommand) {
commands.add(javaCommand);
if (makeStartupFail == javaCommand.getProcessId()) {
throw new IllegalStateException("cannot start " + javaCommand.getProcessId());
}
TestProcess process = new TestProcess(javaCommand.getProcessId());
processes.put(javaCommand.getProcessId(), process);
return process;
}
private TestProcess waitForProcess(ProcessId id) throws InterruptedException {
while (true) {
TestProcess p = processes.get(id);
if (p != null) {
return p;
}
Thread.sleep(1L);
}
}
private TestProcess waitForProcessAlive(ProcessId id) throws InterruptedException {
while (true) {
TestProcess p = processes.get(id);
if (p != null && p.isAlive()) {
return p;
}
Thread.sleep(1L);
}
}
private TestProcess waitForProcessDown(ProcessId id) throws InterruptedException {
while (true) {
TestProcess p = processes.get(id);
if (p != null && !p.isAlive()) {
return p;
}
Thread.sleep(1L);
}
}
@Override
public void close() {
for (TestProcess process : processes.values()) {
process.destroyForcibly();
}
}
}
private class TestProcess implements ProcessMonitor, AutoCloseable {
private final ProcessId processId;
private final CountDownLatch alive = new CountDownLatch(1);
private boolean operational = false;
private boolean askedForRestart = false;
private TestProcess(ProcessId processId) {
this.processId = processId;
}
@Override
public InputStream getInputStream() {
return mock(InputStream.class, Mockito.RETURNS_MOCKS);
}
@Override
public void closeStreams() {
}
@Override
public boolean isAlive() {
return alive.getCount() == 1;
}
@Override
public void askForStop() {
destroyForcibly();
}
@Override
public void destroyForcibly() {
if (isAlive()) {
orderedStops.add(processId);
}
alive.countDown();
}
@Override
public void waitFor() throws InterruptedException {
alive.await();
}
@Override
public void waitFor(long timeout, TimeUnit timeoutUnit) throws InterruptedException {
alive.await(timeout, timeoutUnit);
}
@Override
public boolean isOperational() {
return operational;
}
@Override
public boolean askedForRestart() {
return askedForRestart;
}
@Override
public void acknowledgeAskForRestart() {
this.askedForRestart = false;
}
@Override
public void close() {
alive.countDown();
}
}
}