/* * Copyright 2015-2017 Red Hat, Inc. and/or its affiliates * and other contributors as indicated by the @author tags. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.hawkular.alerts.engine.impl; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.io.StringWriter; import java.net.URI; import java.net.URISyntaxException; import java.net.URL; import java.security.NoSuchAlgorithmException; import java.util.Enumeration; import java.util.Map; import java.util.concurrent.TimeUnit; import java.util.jar.Manifest; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; import javax.annotation.Resource; import javax.ejb.AccessTimeout; import javax.ejb.Singleton; import javax.ejb.Startup; import javax.enterprise.inject.Produces; import javax.net.ssl.SSLContext; import org.cassalog.core.Cassalog; import org.cassalog.core.CassalogBuilder; import org.hawkular.alerts.engine.util.TokenReplacingReader; import org.infinispan.Cache; import org.infinispan.manager.EmbeddedCacheManager; import org.jboss.logging.Logger; import com.datastax.driver.core.Cluster; import com.datastax.driver.core.Host; import com.datastax.driver.core.JdkSSLOptions; import com.datastax.driver.core.PoolingOptions; import com.datastax.driver.core.ProtocolVersion; import com.datastax.driver.core.QueryOptions; import com.datastax.driver.core.ResultSet; import com.datastax.driver.core.SSLOptions; import com.datastax.driver.core.Session; import com.datastax.driver.core.SocketOptions; import com.google.common.collect.ImmutableMap; import com.google.common.io.CharStreams; /** * Cassandra cluster representation and session factory. * * @author Lucas Ponce */ @Startup @Singleton public class CassCluster { private static final Logger log = Logger.getLogger(CassCluster.class); /* PORT used by the Cassandra cluster */ private static final String ALERTS_CASSANDRA_PORT = "hawkular-alerts.cassandra-cql-port"; private static final String ALERTS_CASSANDRA_PORT_ENV = "CASSANDRA_CQL_PORT"; private static final String ALERTS_CASSANDRA_PORT_ENV_DEFAULT = "9042"; /* List of nodes defined on the Cassandra cluster */ private static final String ALERTS_CASSANDRA_NODES = "hawkular-alerts.cassandra-nodes"; private static final String ALERTS_CASSANDRA_NODES_ENV = "CASSANDRA_NODES"; private static final String ALERTS_CASSANDRA_NODES_ENV_DEFAULT = "127.0.0.1"; /* Hawkular Alerts keyspace name used on Cassandra cluster */ private static final String ALERTS_CASSANDRA_KEYSPACE = "hawkular-alerts.cassandra-keyspace"; private static final String ALERTS_CASSANDRA_KEYSPACE_DEFAULT = "hawkular_alerts"; /* Number of attempts when Hawkular Alerts cannot connect with Cassandra cluster to retry */ private static final String ALERTS_CASSANDRA_RETRY_ATTEMPTS = "hawkular-alerts.cassandra-retry-attempts"; private static final String ALERTS_CASSANDRA_RETRY_ATTEMPTS_DEFAULT = "5"; /* ALERTS_CASSANDRA_RETRY_TIMEOUT defined in milliseconds */ private static final String ALERTS_CASSANDRA_RETRY_TIMEOUT = "hawkular-alerts.cassandra-retry-timeout"; private static final String ALERTS_CASSANDRA_RETRY_TIMEOUT_DEFAULT = "2000"; /* ALERTS_CASSANDRA_CONNECT_TIMEOUT and ALERTS_CASSANDRA_CONNECT_TIMEOUT_ENV defined in milliseconds */ private static final String ALERTS_CASSANDRA_CONNECT_TIMEOUT = "hawkular-alerts.cassandra-connect-timeout"; private static final String ALERTS_CASSANDRA_CONNECT_TIMEOUT_ENV = "CASSANDRA_CONNECT_TIMEOUT"; /* ALERTS_CASSANDRA_READ_TIMEOUT and ALERTS_CASSANDRA_READ_TIMEOUT_ENV defined in milliseconds */ private static final String ALERTS_CASSANDRA_READ_TIMEOUT = "hawkular-alerts.cassandra-read-timeout"; private static final String ALERTS_CASSANDRA_READ_TIMEOUT_ENV = "CASSANDRA_READ_TIMEOUT"; /* GLOBAL OVERWRITE true/false flag to recreate an existing schema */ private static final String ALERTS_CASSANDRA_OVERWRITE = "hawkular-alerts.cassandra-overwrite"; private static final String ALERTS_CASSANDRA_OVERWRITE_ENV = "CASSANDRA_OVERWRITE"; private static final String ALERTS_CASSANDRA_OVERWRITE_ENV_DEFAULT = "false"; /* True/false flag to use SSL communication with Cassandra cluster */ private static final String ALERTS_CASSANDRA_USESSL = "hawkular-alerts.cassandra-use-ssl"; private static final String ALERTS_CASSANDRA_USESSL_ENV = "CASSANDRA_USESSL"; private static final String ALERTS_CASSANDRA_USESSL_ENV_DEFAULT = "false"; private static final String ALERTS_CASSANDRA_MAX_QUEUE = "hawkular-alerts.cassandra-max-queue"; private static final String ALERTS_CASSANDRA_MAX_QUEUE_ENV = "CASSANDRA_MAX_QUEUE"; private static final String ALERTS_CASSANDRA_MAX_QUEUE_ENV_DEFAULT = "9182"; private int attempts; private int timeout; private String cqlPort; private String nodes; private int connTimeout; private int readTimeout; private boolean overwrite = false; private String keyspace; private boolean cassandraUseSSL; private int maxQueue; private Cluster cluster = null; private Session session = null; private boolean initialized = false; private boolean distributed = false; /** * Access to the manager of the caches used for the partition services. */ @Resource(lookup = "java:jboss/infinispan/container/hawkular-alerts") private EmbeddedCacheManager cacheManager; private static final String SCHEMA = "schema"; /** * This cache will be used to coordinate schema creation across a cluster of nodes. */ @Resource(lookup = "java:jboss/infinispan/cache/hawkular-alerts/schema") private Cache schemaCache; private void readProperties() { attempts = Integer.parseInt(AlertProperties.getProperty(ALERTS_CASSANDRA_RETRY_ATTEMPTS, ALERTS_CASSANDRA_RETRY_ATTEMPTS_DEFAULT)); timeout = Integer.parseInt(AlertProperties.getProperty(ALERTS_CASSANDRA_RETRY_TIMEOUT, ALERTS_CASSANDRA_RETRY_TIMEOUT_DEFAULT)); cqlPort = AlertProperties.getProperty(ALERTS_CASSANDRA_PORT, ALERTS_CASSANDRA_PORT_ENV, ALERTS_CASSANDRA_PORT_ENV_DEFAULT); nodes = AlertProperties.getProperty(ALERTS_CASSANDRA_NODES, ALERTS_CASSANDRA_NODES_ENV, ALERTS_CASSANDRA_NODES_ENV_DEFAULT); connTimeout = Integer.parseInt(AlertProperties.getProperty(ALERTS_CASSANDRA_CONNECT_TIMEOUT, ALERTS_CASSANDRA_CONNECT_TIMEOUT_ENV, String.valueOf(SocketOptions.DEFAULT_CONNECT_TIMEOUT_MILLIS))); readTimeout = Integer.parseInt(AlertProperties.getProperty(ALERTS_CASSANDRA_READ_TIMEOUT, ALERTS_CASSANDRA_READ_TIMEOUT_ENV, String.valueOf(SocketOptions.DEFAULT_READ_TIMEOUT_MILLIS))); overwrite = Boolean.parseBoolean(AlertProperties.getProperty(ALERTS_CASSANDRA_OVERWRITE, ALERTS_CASSANDRA_OVERWRITE_ENV, ALERTS_CASSANDRA_OVERWRITE_ENV_DEFAULT)); keyspace = AlertProperties.getProperty(ALERTS_CASSANDRA_KEYSPACE, ALERTS_CASSANDRA_KEYSPACE_DEFAULT); cassandraUseSSL = Boolean.parseBoolean(AlertProperties.getProperty(ALERTS_CASSANDRA_USESSL, ALERTS_CASSANDRA_USESSL_ENV, ALERTS_CASSANDRA_USESSL_ENV_DEFAULT)); maxQueue = Integer.parseInt(AlertProperties.getProperty(ALERTS_CASSANDRA_MAX_QUEUE, ALERTS_CASSANDRA_MAX_QUEUE_ENV, ALERTS_CASSANDRA_MAX_QUEUE_ENV_DEFAULT)); } @PostConstruct public void initCassCluster() { readProperties(); if (cacheManager != null) { distributed = cacheManager.getTransport() != null; } int currentAttempts = attempts; SocketOptions socketOptions = null; if (connTimeout != SocketOptions.DEFAULT_CONNECT_TIMEOUT_MILLIS || readTimeout != SocketOptions.DEFAULT_READ_TIMEOUT_MILLIS) { socketOptions = new SocketOptions(); if (connTimeout != SocketOptions.DEFAULT_CONNECT_TIMEOUT_MILLIS) { socketOptions.setConnectTimeoutMillis(connTimeout); } if (readTimeout != SocketOptions.DEFAULT_READ_TIMEOUT_MILLIS) { socketOptions.setReadTimeoutMillis(readTimeout); } } Cluster.Builder clusterBuilder = new Cluster.Builder() .addContactPoints(nodes.split(",")) .withPort(new Integer(cqlPort)) .withPoolingOptions(new PoolingOptions().setMaxQueueSize(maxQueue)) .withProtocolVersion(ProtocolVersion.V3) .withQueryOptions(new QueryOptions().setRefreshSchemaIntervalMillis(0)); if (socketOptions != null) { clusterBuilder.withSocketOptions(socketOptions); } if (cassandraUseSSL) { SSLOptions sslOptions = null; try { String[] defaultCipherSuites = { "TLS_RSA_WITH_AES_128_CBC_SHA", "TLS_RSA_WITH_AES_256_CBC_SHA" }; sslOptions = JdkSSLOptions.builder().withSSLContext(SSLContext.getDefault()) .withCipherSuites(defaultCipherSuites).build(); clusterBuilder.withSSL(sslOptions); } catch (NoSuchAlgorithmException e) { throw new RuntimeException("SSL support is required but is not available in the JVM.", e); } } /* It might happen that alerts component is faster than embedded cassandra deployed in hawkular. We will provide a simple attempt/retry loop to avoid issues at initialization. */ while(session == null && !Thread.currentThread().isInterrupted() && currentAttempts >= 0) { try { cluster = clusterBuilder.build(); session = cluster.connect(); } catch (Exception e) { log.warn("Could not connect to Cassandra cluster - assuming is not up yet. Cause: " + ((e.getCause() == null) ? e : e.getCause())); if (attempts == 0) { throw e; } } if (session == null) { log.warn("[" + currentAttempts + "] Retrying connecting to Cassandra cluster " + "in [" + timeout + "]ms..."); currentAttempts--; try { Thread.sleep(timeout); } catch(InterruptedException e) { Thread.currentThread().interrupt(); } } } if (session != null) { try { waitForAllNodesToBeUp(); if (distributed) { initSchemeDistributed(); } else { initScheme(); } } catch (IOException e) { log.error("Error on initialization of Alerts scheme", e); } } if (session == null) { throw new RuntimeException("Cassandra session is null"); } if (session != null && !initialized) { throw new RuntimeException("Cassandra alerts keyspace is not initialized"); } } private void waitForAllNodesToBeUp() { boolean isReady = false; int attempts = this.attempts; while (!isReady && !Thread.currentThread().isInterrupted() && attempts-- >= 0) { isReady = true; for (Host host : cluster.getMetadata().getAllHosts()) { if (!host.isUp()) { isReady = false; log.warnf("Cassandra node %s may not be up yet. Waiting %s ms for node to come up", host, timeout); try { Thread.sleep(timeout); } catch(InterruptedException e) { Thread.currentThread().interrupt(); } break; } } } if (!isReady) { throw new RuntimeException("It appears that not all nodes in the Cassandra cluster are up after " + this.attempts + " checks. Schema updates cannot proceed without all nodes being up."); } } private void initSchemeDistributed() throws IOException { schemaCache.getAdvancedCache().lock(SCHEMA); initScheme(); } private void initScheme() throws IOException { log.infof("Checking Schema existence for keyspace: %s", keyspace); createSchema(session, keyspace, overwrite); waitForSchemaCheck(); if (!checkSchema()) { log.errorf("Schema %s not created correctly", keyspace); initialized = false; } else { initialized = true; log.infof("Done creating Schema for keyspace: %s", keyspace); } } private void waitForSchemaCheck() { int currentAttempts = attempts; while(!checkSchema() && !Thread.currentThread().isInterrupted() && currentAttempts >= 0) { log.warnf("[%s] Keyspace detected but schema not fully created. " + "Retrying in [%s] ms...", currentAttempts, timeout); currentAttempts--; try { Thread.sleep(timeout); } catch(InterruptedException e) { Thread.currentThread().interrupt(); } } } private boolean checkSchema() { ImmutableMap<String, String> schemaVars = ImmutableMap.of("keyspace", keyspace); String updatedCQL = null; try (InputStream isChecker = CassCluster.class.getResourceAsStream("/org/hawkular/alerts/schema/checker.cql"); InputStreamReader readerChecker = new InputStreamReader(isChecker);) { String content = CharStreams.toString(readerChecker); for (String cql : content.split("(?m)^-- #.*$")) { if (!cql.startsWith("--")) { updatedCQL = substituteVars(cql.trim(), schemaVars); if (log.isDebugEnabled()) { log.debugf("Checking CQL:\n %s \n",updatedCQL); } ResultSet rs = session.execute(updatedCQL); if (rs.isExhausted()) { log.warnf("Table not created.\nEXECUTING CQL: \n%s", updatedCQL); return false; } } } return true; } catch (Exception e) { log.errorf("Failed schema check: %s\nEXECUTING CQL:\n%s", e, updatedCQL); return false; } } private String substituteVars(String cql, Map<String, String> vars) { try (TokenReplacingReader reader = new TokenReplacingReader(cql, vars); StringWriter writer = new StringWriter()) { char[] buffer = new char[32768]; int cnt; while ((cnt = reader.read(buffer)) != -1) { writer.write(buffer, 0, cnt); } return writer.toString(); } catch (IOException e) { throw new RuntimeException("Failed to perform variable substition on CQL", e); } } private URI getCassalogScript() { try { return getClass().getResource("/org/hawkular/alerts/schema/cassalog.groovy").toURI(); } catch (URISyntaxException e) { throw new RuntimeException("Failed to load schema change script", e); } } private String getNewHawkularAlertingVersion() { try { Enumeration<URL> resources = getClass().getClassLoader().getResources("META-INF/MANIFEST.MF"); while (resources.hasMoreElements()) { URL resource = resources.nextElement(); Manifest manifest = new Manifest(resource.openStream()); String vendorId = manifest.getMainAttributes().getValue("Implementation-Vendor-Id"); if (vendorId != null && vendorId.equals("org.hawkular.alerts")) { return manifest.getMainAttributes().getValue("Implementation-Version"); } } throw new RuntimeException("Unable to determine implementation version for Hawkular Alerting"); } catch (IOException e) { throw new RuntimeException(e); } } private void createSchema(Session session, String keyspace, boolean resetDB) { CassalogBuilder builder = new CassalogBuilder(); Cassalog cassalog = builder.withKeyspace(keyspace).withSession(session).build(); Map<String, ?> vars = ImmutableMap.of( "keyspace", keyspace, "reset", resetDB, "session", session, "logger", log ); // List of versions of alerting URI script = getCassalogScript(); cassalog.execute(script, vars); session.execute("INSERT INTO " + keyspace + ".sys_config (config_id, name, value) VALUES " + "('org.hawkular.alerts', 'version', '" + getNewHawkularAlertingVersion() + "')"); } @Produces @CassClusterSession /* This timeout value should be adjusted to the worst case on a Cassandra scheme initialization. Normally it takes an order of < 1 minute a full schema generation. Taking into consideration that there are CI systems very slow we will increase this threshold before to throw an exception. */ @AccessTimeout(value = 300, unit = TimeUnit.SECONDS) public Session getSession() { return session; } public boolean isInitialized() { return initialized; } @PreDestroy public void shutdown() { log.info("Closing Cassandra cluster session"); if (session != null && !session.isClosed()) { session.close(); } if (!cluster.isClosed()) { cluster.close(); } } }