/* * Copyright 2015-2017 Red Hat, Inc. and/or its affiliates * and other contributors as indicated by the @author tags. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.hawkular.alerter.elasticsearch; import static java.util.Collections.EMPTY_LIST; import static org.hawkular.alerter.elasticsearch.ElasticsearchAlerter.getIntervalUnit; import static org.hawkular.alerter.elasticsearch.ElasticsearchAlerter.getIntervalValue; import static org.hawkular.alerter.elasticsearch.ElasticsearchQuery.EventField.DATAID; import java.time.Instant; import java.time.ZoneId; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Arrays; import java.util.Calendar; import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.UUID; import org.apache.http.Header; import org.apache.http.HttpEntity; import org.apache.http.HttpHost; import org.apache.http.auth.AuthScope; import org.apache.http.auth.UsernamePasswordCredentials; import org.apache.http.client.CredentialsProvider; import org.apache.http.entity.ContentType; import org.apache.http.impl.client.BasicCredentialsProvider; import org.apache.http.message.BasicHeader; import org.apache.http.nio.entity.NStringEntity; import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; import org.elasticsearch.client.RestClient; import org.hawkular.alerts.api.json.JsonUtil; import org.hawkular.alerts.api.model.event.Event; import org.hawkular.alerts.api.model.trigger.Trigger; import org.hawkular.alerts.api.services.AlertsService; import org.jboss.logging.Logger; /** * This class performs a query into Elasticsearch system and parses results documents based on Trigger tags/context. * @see {@link ElasticsearchAlerter} * * @author Jay Shaughnessy * @author Lucas Ponce */ public class ElasticsearchQuery implements Runnable { private static final Logger log = Logger.getLogger(ElasticsearchQuery.class); private static final int SIZE_DEFAULT = 10; private static final String _ID = "_id"; private static final String _INDEX = "_index"; private static final String _SOURCE = "_source"; private static final String AUTHORIZATION = "Authorization"; private static final String BEARER = "Bearer"; private static final String ERROR = "error"; private static final String FILTER = "filter"; private static final String FORWARDED_FOR = "forwarded-for"; private static final String GET = "GET"; private static final String HITS = "hits"; private static final String ID = "id"; private static final String INDEX = "index"; private static final String INDEX_NOT_FOUND = "index_not_found_exception"; private static final String INTERVAL = "interval"; private static final String INTERVAL_DEFAULT = "2m"; private static final String MAPPING = "mapping"; private static final String PASS = "pass"; private static final String PREFERENCE = "preference"; private static final String PROXY_REMOTE_USER = "proxy-remote-user"; private static final String USER = "user"; private static final String TIMESTAMP = "timestamp"; private static final String TIMESTAMP_PATTERN = "timestamp_pattern"; private static final String TOKEN = "token"; private static final String TOTAL = "total"; private static final String TYPE = "type"; private static final String RESOURCE_ID = "resource.id"; private static final String SOURCE = "source"; private static final String URL = "url"; private static final String X_FORWARDED = "X-Forwarded-For"; private static final String X_PROXY_REMOTE_USER = "X-Proxy-Remote-User"; private static final DateTimeFormatter[] DEFAULT_DATE_FORMATS = { DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSSSSZ"), DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ssZ") }; private static final ZoneId UTC = ZoneId.of("UTC"); /* Event fields */ enum EventField { ID ("id"), CTIME ("ctime"), DATASOURCE ("dataSource"), DATAID ("dataId"), CATEGORY ("category"), TEXT ("text"), CONTEXT ("context"), TAGS ("tags"); private final String name; EventField(String name) { this.name = name; } public boolean equalsName(String name) { return this.name.equals(name); } public static EventField fromString(String name) { for (EventField field : EventField.values()) { if (field.equalsName(name)) { return field; } } return null; } public String toString() { return this.name; } } private Trigger trigger; private Map<String, String> properties; private AlertsService alerts; private Map<String, EventField> mappings = new HashMap<>(); private RestClient client; private Header[] headers = null; public ElasticsearchQuery(Trigger trigger, Map<String, String> properties, AlertsService alerts) { this.trigger = trigger; this.properties = properties == null ? new HashMap<>() : new HashMap<>(properties); this.alerts = alerts; } public void parseProperties() throws Exception { if (trigger == null) { throw new IllegalStateException("trigger must be not null"); } checkContext(TIMESTAMP, true); checkContext(MAPPING, true); checkContext(INTERVAL, INTERVAL_DEFAULT); checkContext(URL, false); checkContext(INDEX, false); checkContext(FILTER, false); checkContext(TIMESTAMP_PATTERN, false); checkContext(USER, false); checkContext(PASS, false); checkContext(TOKEN, false); } public void parseMap() throws Exception { String rawMap = properties.get(MAPPING); if (rawMap == null) { throw new IllegalStateException("mapping must be not null"); } String[] rawMappings = rawMap.split(","); for (String rawMapping : rawMappings) { String[] fields = rawMapping.trim().split(":"); if (fields.length == 2) { EventField eventField = EventField.fromString(fields[1].trim()); if (eventField == null) { log.warnf("Skipping invalid mapping [%s]", rawMapping); } else { mappings.put(fields[0].trim(), eventField); } } else { log.warnf("Skipping invalid mapping [%s]", rawMapping); } } if (!mappings.values().contains(DATAID)) { throw new IllegalStateException("Mapping [" + rawMap + "] does not include dataId"); } } private void checkContext(String property, boolean required) { checkMap(property, required, null); } private void checkContext(String property, String defaultValue) { checkMap(property, true, defaultValue); } private void checkMap(String property, boolean required, String defaultValue) { Map<String, String> map = trigger.getContext(); if (map.containsKey(property)) { properties.put(property, map.get(property)); } else { if (required && defaultValue == null) { throw new IllegalStateException(property + " is not present and it has not a default value"); } if (defaultValue != null) { properties.put(property, defaultValue); } } } public void connect(String url) throws Exception { String[] urls = url.split(","); HttpHost[] hosts = new HttpHost[urls.length]; for (int i=0; i<urls.length; i++) { hosts[i] = HttpHost.create(urls[0]); } client = RestClient.builder(hosts) .setHttpClientConfigCallback(httpClientBuilder -> { httpClientBuilder.useSystemProperties(); CredentialsProvider credentialsProvider = checkBasicCredentials(); if (credentialsProvider != null) { httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider); } return httpClientBuilder; }) .build(); int nHeaders = 0; String token = properties.get(TOKEN); Header bearer = null; if (!isEmpty(token)) { bearer = new BasicHeader(AUTHORIZATION, BEARER + " " + token); nHeaders++; } String forwarded = properties.get(FORWARDED_FOR); Header xforwarded = null; if (!isEmpty(forwarded)) { xforwarded = new BasicHeader(X_FORWARDED, forwarded); nHeaders++; } String proxyRemoteUser = properties.get(PROXY_REMOTE_USER); Header xProxyRemoteUser = null; if (!isEmpty(proxyRemoteUser)) { xProxyRemoteUser = new BasicHeader(X_PROXY_REMOTE_USER, proxyRemoteUser); nHeaders++; } if (nHeaders > 0) { headers = new Header[nHeaders]; int i = 0; if (bearer != null) { headers[i] = bearer; i++; } if (xforwarded != null) { headers[i] = xforwarded; i++; } if (xProxyRemoteUser != null) { headers[i] = xProxyRemoteUser; } } } private CredentialsProvider checkBasicCredentials() { String user = properties.get(USER); String password = properties.get(PASS); if (!isEmpty(user)){ if (!isEmpty(password)) { CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(user, password)); return credentialsProvider; } else { log.warnf("User [%s] without password ", user); } } return null; } public List<Map<String, Object>> query(String filter, String indices) throws Exception { if (filter == null || filter.isEmpty()) { throw new IllegalArgumentException("filter must be not null"); } List<Map<String, Object>> results = new ArrayList<>(); String json = rawQuery(filter); List<String> index = indices == null ? EMPTY_LIST : new ArrayList<>(Arrays.asList(indices.split(","))); Response response = null; // Using a loop in case results are greater than default results size Map<String, String> params = new HashMap<>(); params.put(PREFERENCE, UUID.randomUUID().toString()); String jsonQuery = "{" + "\"from\":0," + "\"size\":" + SIZE_DEFAULT + "," + "\"query\":" + json + "}"; HttpEntity entity = new NStringEntity(jsonQuery, ContentType.APPLICATION_JSON); boolean retry = false; String endpoint = null; do { try { endpoint = "/" + String.join(",", index) + "/_search"; response = headers == null ? client.performRequest(GET, endpoint, params, entity) : client.performRequest(GET, endpoint, params, entity, headers); retry = false; } catch (ResponseException e) { log.warn(e.toString()); Map<String, Object> exception = JsonUtil.getMapper() .readValue(e.getResponse().getEntity().getContent(), Map.class); Map<String, Object> error = (Map<String, Object>) exception.get(ERROR); if (error != null) { String type = (String) error.get(TYPE); String badIndex = (String) error.get(RESOURCE_ID); if (type != null && type.equals(INDEX_NOT_FOUND)) { retry = true; index.remove(badIndex); if (index.isEmpty()) { return results; } } } } } while(retry); Map<String, Object> responseMap = JsonUtil.getMapper().readValue(response.getEntity().getContent(), Map.class); Map<String, Object> allHits = (Map<String, Object>) responseMap.get(HITS); List<Map<String, Object>> hits = (List<Map<String, Object>>) allHits.get(HITS); results.addAll(hits); long totalHits = new Long((Integer) allHits.get(TOTAL)); long currentHits = hits.size(); while (currentHits < totalHits) { long pageSize = SIZE_DEFAULT; if ((currentHits + SIZE_DEFAULT) > totalHits) { pageSize = totalHits - currentHits; } jsonQuery = "{" + "\"from\":" + currentHits + "," + "\"size\":" + pageSize + "," + "\"query\":" + json + "}"; entity = new NStringEntity(jsonQuery, ContentType.APPLICATION_JSON); response = headers == null ? client.performRequest(GET, endpoint, params, entity) : client.performRequest(GET, endpoint, params, entity, headers); responseMap = JsonUtil.getMapper().readValue(response.getEntity().getContent(), Map.class); allHits = (Map<String, Object>) responseMap.get(HITS); hits = (List<Map<String, Object>>) allHits.get(HITS); currentHits += hits.size(); log.debugf("currentHits [%s] totalHits [%s]", currentHits, totalHits); results.addAll(hits); } log.debugf("Results %s", results.size()); return results; } public List<Event> parseEvents(List<Map<String, Object>> hits) { List<Event> parsed = new ArrayList<>(); if (hits != null && !hits.isEmpty()) { for (Map<String, Object> hit : hits) { Event newEvent = new Event(); newEvent.getContext().put(SOURCE, JsonUtil.toJson(hit)); Map<String, Object> source = (Map<String, Object>) hit.get(_SOURCE); for (Map.Entry<String, EventField> entry : mappings.entrySet()) { boolean isIndex = entry.getKey().equals(INDEX); boolean isId = entry.getKey().equals(ID); switch (entry.getValue()) { case ID: newEvent.setId(isId ? (String) hit.get(_ID) : getField(source, entry.getKey())); break; case CTIME: newEvent.setCtime(parseTimestamp(getField(source, entry.getKey()))); break; case DATAID: newEvent.setDataId(isIndex ? (String) hit.get(_INDEX) : getField(source, entry.getKey())); break; case DATASOURCE: newEvent.setDataSource(getField(source, entry.getKey())); break; case CATEGORY: newEvent.setCategory(getField(source, entry.getKey())); break; case TEXT: newEvent.setText(getField(source, entry.getKey())); break; case CONTEXT: newEvent.getContext().put(entry.getKey(), getField(source, entry.getKey())); break; case TAGS: if (isIndex) { newEvent.getTags().put(INDEX, (String) hit.get(_INDEX)); } else { newEvent.getTags().put(entry.getKey(), getField(source, entry.getKey())); } break; } } if (newEvent.getId() == null) { newEvent.setId(UUID.randomUUID().toString()); } parsed.add(newEvent); } } return parsed; } protected String getField(Map<String, Object> source, String name) { if (source == null || name == null) { return null; } if (name.charAt(0) == '\'' && name.charAt(name.length() - 1) == '\'') { return name.substring(1, name.length() - 1); } String[] names = name.split("\\|"); String defaultValue = ""; if (names.length > 1) { if (names[1].charAt(0) == '\'' && names[1].charAt(names[1].length() - 1) == '\'') { defaultValue = names[1].substring(1, names[1].length() - 1); } name = names[0]; } String[] fields = name.split("\\."); for (int i=0; i < fields.length; i++) { Object value = source.get(fields[i]); if (value instanceof String) { return (String) value; } if (value instanceof Map) { source = (Map<String, Object>) value; } } return defaultValue; } public long parseTimestamp(String timestamp) { String definedPattern = properties.get(TIMESTAMP_PATTERN); if (definedPattern != null) { DateTimeFormatter formatter = null; try { formatter = DateTimeFormatter.ofPattern(definedPattern); return ZonedDateTime.parse(timestamp, formatter).toInstant().toEpochMilli(); } catch (Exception e) { log.debugf("Not able to parse [%s] with format [%s]", timestamp, formatter); } } for (DateTimeFormatter formatter : DEFAULT_DATE_FORMATS) { try { return ZonedDateTime.parse(timestamp, formatter).toInstant().toEpochMilli(); } catch (Exception e) { log.debugf("Not able to parse [%s] with format [%s]", timestamp, formatter); } } try { return new Long(timestamp).longValue(); } catch (Exception e) { log.debugf("Not able to parse [%s] as plain timestamp", timestamp); } return System.currentTimeMillis(); } public String formatTimestamp(Date date) { String definedPattern = properties.get(TIMESTAMP_PATTERN); if (definedPattern != null) { DateTimeFormatter formatter = null; try { formatter = DateTimeFormatter.ofPattern(definedPattern); return formatter.format(ZonedDateTime.ofInstant(Instant.ofEpochMilli(date.getTime()), UTC)); } catch (Exception e) { log.debugf("Not able to format [%s] with pattern [%s]", date, formatter); } } return DEFAULT_DATE_FORMATS[0].format(ZonedDateTime.ofInstant(Instant.ofEpochMilli(date.getTime()), UTC)); } private String rawQuery(String filter) { return "{" + "\"constant_score\":{" + "\"filter\":{" + "\"bool\":{" + "\"must\":" + filter + "}" + "}" + "}" + "}"; } private Date intervalDate() { String interval = properties.get(INTERVAL); int value = getIntervalValue(interval); switch (getIntervalUnit(interval)) { case HOURS: value = value * 3600 * 1000; break; case MINUTES: value = value * 60 * 1000; break; case SECONDS: value = value * 1000; break; } Calendar cal = Calendar.getInstance(); cal.setTimeInMillis(System.currentTimeMillis() - value); return cal.getTime(); } private String prepareQuery() { String range = new StringBuilder("{\"range\":{\"").append(properties.get(TIMESTAMP)) .append("\":{\"gt\":\"").append(formatTimestamp(intervalDate())).append("\"}}}").toString(); String filter = properties.get(FILTER); String filters; if (filter != null) { filters =new StringBuilder("[").append(filter).append(",").append(range).append("]").toString(); } else { filters = new StringBuilder("[").append(range).append("]").toString(); } return rawQuery(filters); } public void disconnect() throws Exception { if (client != null) { client.close(); } } @Override public void run() { try { parseProperties(); parseMap(); connect(properties.get(URL)); String preparedQuery = prepareQuery(); log.debugf("Fetching documents from Elasticsearch [%s] %s", preparedQuery, trigger.getContext()); List<Event> events = parseEvents(query(preparedQuery, (String) properties.get(INDEX))); log.debugf("Found [%s]", events.size()); disconnect(); events.stream().forEach(e -> e.setTenantId(trigger.getTenantId())); alerts.sendEvents(events); } catch (Exception e) { log.error("Error querying Elasticsearch.", e); } } private boolean isEmpty(String s) { return s == null || s.isEmpty(); } }