/* * SonarQube * Copyright (C) 2009-2017 SonarSource SA * mailto:info AT sonarsource DOT com * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 3 of the License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public License * along with this program; if not, write to the Free Software Foundation, * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ package org.sonar.application; import java.io.InputStream; import java.util.ArrayList; import java.util.EnumMap; import java.util.List; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import org.junit.After; import org.junit.Rule; import org.junit.Test; import org.junit.rules.DisableOnDebug; import org.junit.rules.ExpectedException; import org.junit.rules.TestRule; import org.junit.rules.Timeout; import org.mockito.Mockito; import org.sonar.application.config.TestAppSettings; import org.sonar.application.process.JavaCommand; import org.sonar.application.process.JavaCommandFactory; import org.sonar.application.process.JavaProcessLauncher; import org.sonar.application.process.ProcessMonitor; import org.sonar.process.ProcessId; import org.sonar.process.ProcessProperties; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.timeout; import static org.mockito.Mockito.verify; import static org.sonar.process.ProcessId.COMPUTE_ENGINE; import static org.sonar.process.ProcessId.ELASTICSEARCH; import static org.sonar.process.ProcessId.WEB_SERVER; public class SchedulerImplTest { private static final JavaCommand ES_COMMAND = new JavaCommand(ELASTICSEARCH); private static final JavaCommand WEB_LEADER_COMMAND = new JavaCommand(WEB_SERVER); private static final JavaCommand WEB_FOLLOWER_COMMAND = new JavaCommand(WEB_SERVER); private static final JavaCommand CE_COMMAND = new JavaCommand(COMPUTE_ENGINE); @Rule public TestRule safeguardTimeout = new DisableOnDebug(Timeout.seconds(60)); @Rule public ExpectedException expectedException = ExpectedException.none(); private AppReloader appReloader = mock(AppReloader.class); private TestAppSettings settings = new TestAppSettings(); private TestJavaCommandFactory javaCommandFactory = new TestJavaCommandFactory(); private TestJavaProcessLauncher processLauncher = new TestJavaProcessLauncher(); private TestAppState appState = new TestAppState(); private List<ProcessId> orderedStops = new ArrayList<>(); @After public void tearDown() throws Exception { processLauncher.close(); } @Test public void start_and_stop_sequence_of_ES_WEB_CE_in_order() throws Exception { enableAllProcesses(); SchedulerImpl underTest = newScheduler(); underTest.schedule(); // elasticsearch does not have preconditions to start TestProcess es = processLauncher.waitForProcess(ELASTICSEARCH); assertThat(es.isAlive()).isTrue(); assertThat(processLauncher.processes).hasSize(1); // elasticsearch becomes operational -> web leader is starting es.operational = true; waitForAppStateOperational(ELASTICSEARCH); TestProcess web = processLauncher.waitForProcess(WEB_SERVER); assertThat(web.isAlive()).isTrue(); assertThat(processLauncher.processes).hasSize(2); assertThat(processLauncher.commands).containsExactly(ES_COMMAND, WEB_LEADER_COMMAND); // web becomes operational -> CE is starting web.operational = true; waitForAppStateOperational(WEB_SERVER); TestProcess ce = processLauncher.waitForProcess(COMPUTE_ENGINE); assertThat(ce.isAlive()).isTrue(); assertThat(processLauncher.processes).hasSize(3); assertThat(processLauncher.commands).containsExactly(ES_COMMAND, WEB_LEADER_COMMAND, CE_COMMAND); // all processes are up processLauncher.processes.values().forEach(p -> assertThat(p.isAlive()).isTrue()); // processes are stopped in reverse order of startup underTest.terminate(); assertThat(orderedStops).containsExactly(COMPUTE_ENGINE, WEB_SERVER, ELASTICSEARCH); processLauncher.processes.values().forEach(p -> assertThat(p.isAlive()).isFalse()); // does nothing because scheduler is already terminated underTest.awaitTermination(); } private void enableAllProcesses() { settings.set(ProcessProperties.CLUSTER_ENABLED, "true"); } @Test public void all_processes_are_stopped_if_one_process_goes_down() throws Exception { Scheduler underTest = startAll(); processLauncher.waitForProcess(WEB_SERVER).destroyForcibly(); underTest.awaitTermination(); assertThat(orderedStops).containsExactly(WEB_SERVER, COMPUTE_ENGINE, ELASTICSEARCH); processLauncher.processes.values().forEach(p -> assertThat(p.isAlive()).isFalse()); // following does nothing underTest.terminate(); underTest.awaitTermination(); } @Test public void all_processes_are_stopped_if_one_process_fails_to_start() throws Exception { enableAllProcesses(); SchedulerImpl underTest = newScheduler(); processLauncher.makeStartupFail = COMPUTE_ENGINE; underTest.schedule(); processLauncher.waitForProcess(ELASTICSEARCH).operational = true; processLauncher.waitForProcess(WEB_SERVER).operational = true; underTest.awaitTermination(); assertThat(orderedStops).containsExactly(WEB_SERVER, ELASTICSEARCH); processLauncher.processes.values().forEach(p -> assertThat(p.isAlive()).isFalse()); } @Test public void terminate_can_be_called_multiple_times() throws Exception { Scheduler underTest = startAll(); underTest.terminate(); processLauncher.processes.values().forEach(p -> assertThat(p.isAlive()).isFalse()); // does nothing underTest.terminate(); } @Test public void awaitTermination_blocks_until_all_processes_are_stopped() throws Exception { Scheduler underTest = startAll(); Thread awaitingTermination = new Thread(() -> underTest.awaitTermination()); awaitingTermination.start(); assertThat(awaitingTermination.isAlive()).isTrue(); underTest.terminate(); // the thread is being stopped awaitingTermination.join(); assertThat(awaitingTermination.isAlive()).isFalse(); } @Test public void restart_reloads_java_commands_and_restarts_all_processes() throws Exception { Scheduler underTest = startAll(); processLauncher.waitForProcess(WEB_SERVER).askedForRestart = true; // waiting for all processes to be stopped boolean stopped = false; while (!stopped) { stopped = orderedStops.size() == 3; Thread.sleep(1L); } // restarting verify(appReloader, timeout(60_000)).reload(settings); processLauncher.waitForProcessAlive(ELASTICSEARCH); processLauncher.waitForProcessAlive(COMPUTE_ENGINE); processLauncher.waitForProcessAlive(WEB_SERVER); underTest.terminate(); // 3+3 processes have been stopped assertThat(orderedStops).hasSize(6); assertThat(processLauncher.waitForProcess(ELASTICSEARCH).isAlive()).isFalse(); assertThat(processLauncher.waitForProcess(COMPUTE_ENGINE).isAlive()).isFalse(); assertThat(processLauncher.waitForProcess(WEB_SERVER).isAlive()).isFalse(); // verify that awaitTermination() does not block underTest.awaitTermination(); } @Test public void restart_stops_all_if_new_settings_are_not_allowed() throws Exception { Scheduler underTest = startAll(); doThrow(new IllegalStateException("reload error")).when(appReloader).reload(settings); processLauncher.waitForProcess(WEB_SERVER).askedForRestart = true; // waiting for all processes to be stopped processLauncher.waitForProcessDown(ELASTICSEARCH); processLauncher.waitForProcessDown(COMPUTE_ENGINE); processLauncher.waitForProcessDown(WEB_SERVER); // verify that awaitTermination() does not block underTest.awaitTermination(); } @Test public void web_follower_starts_only_when_web_leader_is_operational() throws Exception { // leader takes the lock, so underTest won't get it assertThat(appState.tryToLockWebLeader()).isTrue(); appState.setOperational(ProcessId.ELASTICSEARCH); enableAllProcesses(); SchedulerImpl underTest = newScheduler(); underTest.schedule(); processLauncher.waitForProcessAlive(ProcessId.ELASTICSEARCH); assertThat(processLauncher.processes).hasSize(1); // leader becomes operational -> follower can start appState.setOperational(ProcessId.WEB_SERVER); processLauncher.waitForProcessAlive(WEB_SERVER); underTest.terminate(); } @Test public void web_server_waits_for_remote_elasticsearch_to_be_started_if_local_es_is_disabled() throws Exception { settings.set(ProcessProperties.CLUSTER_ENABLED, "true"); settings.set(ProcessProperties.CLUSTER_SEARCH_DISABLED, "true"); SchedulerImpl underTest = newScheduler(); underTest.schedule(); // WEB and CE wait for ES to be up assertThat(processLauncher.processes).isEmpty(); // ES becomes operational on another node -> web leader can start appState.setRemoteOperational(ProcessId.ELASTICSEARCH); processLauncher.waitForProcessAlive(WEB_SERVER); assertThat(processLauncher.processes).hasSize(1); underTest.terminate(); } @Test public void compute_engine_waits_for_remote_elasticsearch_and_web_leader_to_be_started_if_local_es_is_disabled() throws Exception { settings.set(ProcessProperties.CLUSTER_ENABLED, "true"); settings.set(ProcessProperties.CLUSTER_SEARCH_DISABLED, "true"); settings.set(ProcessProperties.CLUSTER_WEB_DISABLED, "true"); SchedulerImpl underTest = newScheduler(); underTest.schedule(); // CE waits for ES and WEB leader to be up assertThat(processLauncher.processes).isEmpty(); // ES and WEB leader become operational on another nodes -> CE can start appState.setRemoteOperational(ProcessId.ELASTICSEARCH); appState.setRemoteOperational(ProcessId.WEB_SERVER); processLauncher.waitForProcessAlive(COMPUTE_ENGINE); assertThat(processLauncher.processes).hasSize(1); underTest.terminate(); } private SchedulerImpl newScheduler() { return new SchedulerImpl(settings, appReloader, javaCommandFactory, processLauncher, appState) .setProcessWatcherDelayMs(1L); } private Scheduler startAll() throws InterruptedException { enableAllProcesses(); SchedulerImpl scheduler = newScheduler(); scheduler.schedule(); processLauncher.waitForProcess(ELASTICSEARCH).operational = true; processLauncher.waitForProcess(WEB_SERVER).operational = true; processLauncher.waitForProcess(COMPUTE_ENGINE).operational = true; return scheduler; } private void waitForAppStateOperational(ProcessId id) throws InterruptedException { while (true) { if (appState.isOperational(id, true)) { return; } Thread.sleep(1L); } } private static class TestJavaCommandFactory implements JavaCommandFactory { @Override public JavaCommand createEsCommand() { return ES_COMMAND; } @Override public JavaCommand createWebCommand(boolean leader) { return leader ? WEB_LEADER_COMMAND : WEB_FOLLOWER_COMMAND; } @Override public JavaCommand createCeCommand() { return CE_COMMAND; } } private class TestJavaProcessLauncher implements JavaProcessLauncher { private final EnumMap<ProcessId, TestProcess> processes = new EnumMap<>(ProcessId.class); private final List<JavaCommand> commands = new ArrayList<>(); private ProcessId makeStartupFail = null; @Override public ProcessMonitor launch(JavaCommand javaCommand) { commands.add(javaCommand); if (makeStartupFail == javaCommand.getProcessId()) { throw new IllegalStateException("cannot start " + javaCommand.getProcessId()); } TestProcess process = new TestProcess(javaCommand.getProcessId()); processes.put(javaCommand.getProcessId(), process); return process; } private TestProcess waitForProcess(ProcessId id) throws InterruptedException { while (true) { TestProcess p = processes.get(id); if (p != null) { return p; } Thread.sleep(1L); } } private TestProcess waitForProcessAlive(ProcessId id) throws InterruptedException { while (true) { TestProcess p = processes.get(id); if (p != null && p.isAlive()) { return p; } Thread.sleep(1L); } } private TestProcess waitForProcessDown(ProcessId id) throws InterruptedException { while (true) { TestProcess p = processes.get(id); if (p != null && !p.isAlive()) { return p; } Thread.sleep(1L); } } @Override public void close() { for (TestProcess process : processes.values()) { process.destroyForcibly(); } } } private class TestProcess implements ProcessMonitor, AutoCloseable { private final ProcessId processId; private final CountDownLatch alive = new CountDownLatch(1); private boolean operational = false; private boolean askedForRestart = false; private TestProcess(ProcessId processId) { this.processId = processId; } @Override public InputStream getInputStream() { return mock(InputStream.class, Mockito.RETURNS_MOCKS); } @Override public void closeStreams() { } @Override public boolean isAlive() { return alive.getCount() == 1; } @Override public void askForStop() { destroyForcibly(); } @Override public void destroyForcibly() { if (isAlive()) { orderedStops.add(processId); } alive.countDown(); } @Override public void waitFor() throws InterruptedException { alive.await(); } @Override public void waitFor(long timeout, TimeUnit timeoutUnit) throws InterruptedException { alive.await(timeout, timeoutUnit); } @Override public boolean isOperational() { return operational; } @Override public boolean askedForRestart() { return askedForRestart; } @Override public void acknowledgeAskForRestart() { this.askedForRestart = false; } @Override public void close() { alive.countDown(); } } }