/*
* Copyright 2016 ANI Technologies Pvt. Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.olacabs.fabric.processors.kafkawriter;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.Lists;
import com.olacabs.fabric.compute.ProcessingContext;
import com.olacabs.fabric.compute.processor.InitializationException;
import com.olacabs.fabric.compute.processor.ProcessingException;
import com.olacabs.fabric.compute.processor.StreamingProcessor;
import com.olacabs.fabric.compute.util.ComponentPropertyReader;
import com.olacabs.fabric.model.common.ComponentMetadata;
import com.olacabs.fabric.model.event.Event;
import com.olacabs.fabric.model.event.EventSet;
import com.olacabs.fabric.model.processor.Processor;
import com.olacabs.fabric.model.processor.ProcessorType;
import kafka.javaapi.producer.Producer;
import kafka.producer.DefaultPartitioner;
import kafka.producer.KeyedMessage;
import kafka.producer.ProducerConfig;
import lombok.Getter;
import lombok.Setter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.List;
import java.util.Properties;
/**
* TODO Javadoc.
*/
@Processor(
namespace = "global",
name = "kafka-writer",
version = "2.0",
description = "A processor that write data into kafka",
cpu = 0.1,
memory = 1,
processorType = ProcessorType.EVENT_DRIVEN,
requiredProperties = {
"brokerList",
"ingestionPoolSize",
"kafkaKeyJsonPath"
},
optionalProperties = {
"isTopicOnJsonPath",
"topic",
"topicJsonPath",
"ignoreError",
"kafkaSerializerClass",
"ackCount"
})
public class KafkaWriter extends StreamingProcessor {
private static final Logger LOGGER = LoggerFactory.getLogger(KafkaWriter.class.getSimpleName());
private static final boolean DEFAULT_IGNORE_SERIALIZATION_ERROR = false;
private static final boolean DEFAULT_TOPIC_ON_JSON_PATH = false;
private static final String DEFAULT_SERIALIZER_CLASS = "kafka.serializer.StringEncoder";
private static final String DEFAULT_KAFKA_KEY_JSON_PATH = "/metadata/partitionKey/value";
private static final int DEFAULT_ACK_COUNT = 1;
private static final int DEFAULT_BATCH_SIZE = 10;
private static final String ACK_COUNT = "-1";
private String kafkaKeyJsonPath;
private boolean ignoreError;
private ObjectMapper mapper;
@Getter
@Setter
private String kafkaTopic;
@Getter
@Setter
private String kafkaTopicJsonPath;
@Getter
@Setter
private int ingestionPoolSize;
@Getter
@Setter
private Producer<String, String> producer;
@Getter
@Setter
private boolean isTopicOnJsonPath = false;
@Override
protected EventSet consume(ProcessingContext processingContext, EventSet eventSet) throws ProcessingException {
final List<KeyedMessage<String, String>> messages = Lists.newArrayList();
try {
eventSet.getEvents().forEach(event -> {
KeyedMessage<String, String> convertedMessage = null;
try {
convertedMessage = convertEvent(event);
} catch (ProcessingException e) {
LOGGER.error("Error converting byte stream to event: ", e);
throw new RuntimeException(e);
}
if (null != convertedMessage) {
messages.add(convertedMessage);
}
});
} catch (final Exception e) {
LOGGER.error("Error converting byte stream to event: ", e);
throw new ProcessingException(e);
}
Lists.partition(messages, ingestionPoolSize).forEach(messageList -> getProducer().send(messageList));
return eventSet;
}
/**
* convert the event into Kafka keyed messages.
*
* @param event to convert
* @return KeyedMessage
*/
protected KeyedMessage<String, String> convertEvent(Event event) throws ProcessingException {
JsonNode eventData = event.getJsonNode();
if (null == eventData) {
if (event.getData() instanceof byte[]) {
try {
eventData = mapper.readTree((byte[]) event.getData());
} catch (IOException e) {
LOGGER.error("Error converting byte stream to event: ", e);
if (!ignoreError) {
LOGGER.error("Error converting byte stream to event");
throw new ProcessingException("Error converting byte stream to event", e);
}
return null;
}
} else {
if (!ignoreError) {
LOGGER.error("Error converting byte stream to event: Event is not byte stream");
throw new ProcessingException("Error converting byte stream to event: Event is not byte stream");
}
return null;
}
}
final String kafkaKey = kafkaKeyJsonPath != null
? eventData.at(kafkaKeyJsonPath).asText().replace("\"", "")
: eventData.at(DEFAULT_KAFKA_KEY_JSON_PATH).asText().replace("\"", "");
final String topic = isTopicOnJsonPath()
? eventData.at(getKafkaTopicJsonPath()).toString().replace("\"", "")
: getKafkaTopic().replace("\"", "");
return new KeyedMessage<>(topic, kafkaKey, eventData.toString());
}
@Override
public void initialize(String instanceId, Properties globalProperties, Properties properties,
ComponentMetadata componentMetadata) throws InitializationException {
final String kafkaBrokerList = ComponentPropertyReader
.readString(properties, globalProperties, "brokerList", instanceId, componentMetadata);
isTopicOnJsonPath = ComponentPropertyReader
.readBoolean(properties, globalProperties, "isTopicOnJsonPath", instanceId, componentMetadata,
DEFAULT_TOPIC_ON_JSON_PATH);
if (!isTopicOnJsonPath) {
kafkaTopic = ComponentPropertyReader
.readString(properties, globalProperties, "topic", instanceId, componentMetadata);
if (kafkaTopic == null) {
LOGGER.error("Kafka topic in properties not found");
throw new RuntimeException("Kafka topic in properties not found");
}
setKafkaTopic(kafkaTopic);
} else {
kafkaTopicJsonPath = ComponentPropertyReader
.readString(properties, globalProperties, "topicJsonPath", instanceId, componentMetadata);
if (kafkaTopicJsonPath == null) {
LOGGER.error("Kafka topic json path not found");
throw new RuntimeException("Kafka topic json path not found");
}
setKafkaTopicJsonPath(kafkaTopicJsonPath);
}
kafkaKeyJsonPath = ComponentPropertyReader
.readString(properties, globalProperties, "kafkaKeyJsonPath", instanceId, componentMetadata,
DEFAULT_KAFKA_KEY_JSON_PATH);
final String kafkaSerializerClass = ComponentPropertyReader
.readString(properties, globalProperties, "kafkaSerializerClass", instanceId, componentMetadata,
DEFAULT_SERIALIZER_CLASS);
ingestionPoolSize = ComponentPropertyReader
.readInteger(properties, globalProperties, "ingestionPoolSize", instanceId, componentMetadata,
DEFAULT_BATCH_SIZE);
final Integer ackCount = ComponentPropertyReader
.readInteger(properties, globalProperties, "ackCount", instanceId, componentMetadata,
DEFAULT_ACK_COUNT);
ignoreError = ComponentPropertyReader
.readBoolean(properties, globalProperties, "ignoreError", instanceId, componentMetadata,
DEFAULT_IGNORE_SERIALIZATION_ERROR);
final Properties props = new Properties();
props.put("metadata.broker.list", kafkaBrokerList);
props.put("serializer.class", kafkaSerializerClass);
props.put("partitioner.class", DefaultPartitioner.class.getName());
props.put("request.required.acks", ACK_COUNT);
props.put("min.isr", Integer.toString(ackCount));
producer = new Producer<>(new ProducerConfig(props));
mapper = new ObjectMapper();
LOGGER.info("Initialized kafka writer...");
}
@Override
public void destroy() {
producer.close();
LOGGER.info("Closed kafka writer...");
}
}