/*
* Copyright 2013-2014 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.xd.dirt.stream.dsl;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import org.springframework.data.repository.CrudRepository;
import org.springframework.util.Assert;
import org.springframework.xd.dirt.core.BaseDefinition;
/**
* @author Andy Clement
*/
public class StreamConfigParser implements StreamLookupEnvironment {
private String expressionString;
private List<Token> tokenStream;
private int tokenStreamLength;
private int tokenStreamPointer; // Current location in the token stream when
// processing tokens
private int lastGoodPoint;
/** The repository (if supplied) is used to chase down substream/label references */
private CrudRepository<? extends BaseDefinition, String> repository;
public StreamConfigParser(CrudRepository<? extends BaseDefinition, String> repository) {
this.repository = repository;
}
/**
* Parse a stream definition without supplying the stream name up front. The stream name may be embedded in the
* definition. For example: <code>mystream = http | file</code>
*
* @return the AST for the parsed stream
*/
public StreamNode parse(String stream) {
return parse(null, stream);
}
/**
* Parse a stream definition.
*
* @return the AST for the parsed stream
* @throws StreamDefinitionException
*/
public StreamNode parse(String name, String stream) {
this.expressionString = stream;
Tokenizer tokenizer = new Tokenizer(expressionString);
tokenStream = tokenizer.getTokens();
tokenStreamLength = tokenStream.size();
tokenStreamPointer = 0;
StreamNode ast = eatStream();
// Check the stream name, however it was specified
if (ast.getName() != null && !isValidStreamName(ast.getName())) {
throw new StreamDefinitionException(ast.getName(), 0, XDDSLMessages.ILLEGAL_STREAM_NAME, ast.getName());
}
if (name != null && !isValidStreamName(name)) {
throw new StreamDefinitionException(name, 0, XDDSLMessages.ILLEGAL_STREAM_NAME, name);
}
// Check that each module has a unique label (either explicit or implicit)
Map<String, ModuleNode> alreadySeen = new LinkedHashMap<String, ModuleNode>();
for (int m = 0; m < ast.getModuleNodes().size(); m++) {
ModuleNode node = ast.getModuleNodes().get(m);
ModuleNode previous = alreadySeen.put(node.getLabelName(), node);
if (previous != null) {
String duplicate = node.getLabelName();
int previousIndex = new ArrayList<String>(alreadySeen.keySet()).indexOf(duplicate);
throw new StreamDefinitionException(stream, node.startpos, XDDSLMessages.DUPLICATE_LABEL,
duplicate, previous.getName(), previousIndex, node.getName(), m);
}
}
// Check if the stream name is same as that of any of its modules' names
// Can lead to infinite recursion during resolution, when parsing a composite module.
if (ast.getModule(name) != null) {
throw new StreamDefinitionException(stream, stream.indexOf(name),
XDDSLMessages.STREAM_NAME_MATCHING_MODULE_NAME,
name);
}
if (moreTokens()) {
throw new StreamDefinitionException(this.expressionString, peekToken().startpos, XDDSLMessages.MORE_INPUT,
toString(nextToken()));
}
ast.resolve(this, this.expressionString);
return ast;
}
// (name =)
private String maybeEatStreamName() {
String streamName = null;
if (lookAhead(1, TokenKind.EQUALS)) {
if (peekToken(TokenKind.IDENTIFIER)) {
streamName = eatToken(TokenKind.IDENTIFIER).data;
nextToken(); // skip '='
}
else {
raiseException(peekToken().startpos, XDDSLMessages.ILLEGAL_STREAM_NAME, toString(peekToken()));
}
}
return streamName;
}
// stream: (streamName) (sourceChannel) moduleList (sinkChannel)
private StreamNode eatStream() {
String streamName = maybeEatStreamName();
SourceChannelNode sourceChannelNode = maybeEatSourceChannel();
// This construct: queue:foo > topic:bar is a source then a sink channel
// with no module. Special handling for that is right here:
boolean bridge = false;
if (sourceChannelNode != null) { // so if we are just after a '>'
if (looksLikeChannel() && noMorePipes()) {
bridge = true;
}
}
List<ModuleNode> moduleNodes = null;
if (bridge) {
// Create a bridge module to hang the source/sink channels off
tokenStreamPointer--; // Rewind so we can nicely eat the sink channel
moduleNodes = new ArrayList<ModuleNode>();
moduleNodes.add(new ModuleNode(null, "bridge", peekToken().startpos, peekToken().endpos, null));
}
else {
moduleNodes = eatModuleList();
}
SinkChannelNode sinkChannelNode = maybeEatSinkChannel();
// Further data is an error
if (moreTokens()) {
Token t = peekToken();
raiseException(t.startpos, XDDSLMessages.UNEXPECTED_DATA_AFTER_STREAMDEF, toString(t));
}
StreamNode streamNode = new StreamNode(expressionString, streamName, moduleNodes, sourceChannelNode,
sinkChannelNode);
return streamNode;
}
private boolean noMorePipes() {
return noMorePipes(tokenStreamPointer);
}
private boolean noMorePipes(int tp) {
while (tp < tokenStreamLength) {
if (tokenStream.get(tp++).getKind() == TokenKind.PIPE) {
return false;
}
}
return true;
}
private boolean looksLikeChannel() {
return looksLikeChannel(tokenStreamPointer);
}
enum ChannelPrefix {
queue, tap, topic;
}
// return true if the specified tokenpointer appears to be pointing at a channel
private boolean looksLikeChannel(int tp) {
if (moreTokens() && tokenStream.get(tp).getKind() == TokenKind.IDENTIFIER) {
String prefix = tokenStream.get(tp).data;
if (isLegalChannelPrefix(prefix)) {
if (tokenStreamPointer + 1 < tokenStream.size() && tokenStream.get(tp + 1).getKind() == TokenKind.COLON) {
// if (isNextTokenAdjacent(tp) && isNextTokenAdjacent(tp + 1)) {
return true;
// }
}
}
}
return false;
}
// identifier ':' identifier >
// tap ':' identifier ':' identifier '.' identifier >
private SourceChannelNode maybeEatSourceChannel() {
boolean gtBeforePipe = false;
// Seek for a GT(>) before a PIPE(|)
for (int tp = tokenStreamPointer; tp < tokenStreamLength; tp++) {
Token t = tokenStream.get(tp);
if (t.getKind() == TokenKind.GT) {
gtBeforePipe = true;
break;
}
else if (t.getKind() == TokenKind.PIPE) {
break;
}
}
if (!gtBeforePipe || !looksLikeChannel(tokenStreamPointer)) {
return null;
}
ChannelNode channel = eatChannelReference(true);
Token gt = eatToken(TokenKind.GT);
return new SourceChannelNode(channel, gt.endpos);
}
// '>' identifier ':' identifier
private SinkChannelNode maybeEatSinkChannel() {
SinkChannelNode sinkChannelNode = null;
if (peekToken(TokenKind.GT)) {
Token gt = eatToken(TokenKind.GT);
ChannelNode channelNode = eatChannelReference(false);
sinkChannelNode = new SinkChannelNode(channelNode, gt.startpos);
}
return sinkChannelNode;
}
private boolean isLegalChannelPrefix(String string) {
return string.equals(ChannelPrefix.queue.toString()) ||
string.equals(ChannelPrefix.topic.toString()) ||
string.equals(ChannelPrefix.tap.toString());
}
// A channel reference is a colon separated list of identifiers that determine
// the appropriate scope then a sequence of dot separated identifiers that
// reference something within that scope.
// Only three types of top level prefix are supported for channels:queue, topic, tap
// identifier [ ':' identifier ]* [ '.' identifier ]*
// If the first identifier is a tap (and tapping is allowed) then
// the dereferencing is allowed
private ChannelNode eatChannelReference(boolean tapAllowed) {
Token firstToken = nextToken();
if (!firstToken.isIdentifier() || !isLegalChannelPrefix(firstToken.data)) {
raiseException(firstToken.startpos,
tapAllowed ? XDDSLMessages.EXPECTED_CHANNEL_PREFIX_QUEUE_TOPIC_TAP
: XDDSLMessages.EXPECTED_CHANNEL_PREFIX_QUEUE_TOPIC,
toString(firstToken));
}
List<Token> channelScopeComponents = new ArrayList<Token>();
channelScopeComponents.add(firstToken);
while (peekToken(TokenKind.COLON)) {
if (!isNextTokenAdjacent()) {
raiseException(peekToken().startpos, XDDSLMessages.NO_WHITESPACE_IN_CHANNEL_DEFINITION);
}
nextToken(); // skip colon
if (!isNextTokenAdjacent()) {
raiseException(peekToken().startpos, XDDSLMessages.NO_WHITESPACE_IN_CHANNEL_DEFINITION);
}
channelScopeComponents.add(eatToken(TokenKind.IDENTIFIER));
}
List<Token> channelReferenceComponents = new ArrayList<Token>();
if (tapAllowed && firstToken.data.equalsIgnoreCase("tap")) {
if (peekToken(TokenKind.DOT)) {
if (channelScopeComponents.size() < 3) {
raiseException(firstToken.startpos, XDDSLMessages.TAP_NEEDS_THREE_COMPONENTS);
}
String tokenData = channelScopeComponents.get(1).data;
// for Stream, tap:stream:XXX - the channel name is always indexed
// for Job, tap:job:XXX - the channel name can have "." in case of job notification channels
if (!tokenData.equalsIgnoreCase("stream") && !tokenData.equalsIgnoreCase("job")) {
raiseException(peekToken().startpos, XDDSLMessages.ONLY_A_TAP_ON_A_STREAM_OR_JOB_CAN_BE_INDEXED);
}
}
while (peekToken(TokenKind.DOT)) {
if (!isNextTokenAdjacent()) {
raiseException(peekToken().startpos, XDDSLMessages.NO_WHITESPACE_IN_CHANNEL_DEFINITION);
}
nextToken(); // skip dot
if (!isNextTokenAdjacent()) {
raiseException(peekToken().startpos, XDDSLMessages.NO_WHITESPACE_IN_CHANNEL_DEFINITION);
}
channelReferenceComponents.add(eatToken(TokenKind.IDENTIFIER));
}
}
else if (peekToken(TokenKind.DOT)) {
if (tapAllowed) {
raiseException(peekToken().startpos, XDDSLMessages.ONLY_A_TAP_ON_A_STREAM_OR_JOB_CAN_BE_INDEXED);
}
else {
raiseException(peekToken().startpos, XDDSLMessages.CHANNEL_INDEXING_NOT_ALLOWED);
}
}
// Verify the structure:
ChannelType channelType = null;
if (firstToken.data.equalsIgnoreCase("tap")) {
// tap:stream:XXX.YYY
// tap:job:XXX
// tap:queue:XXX
// tap:topic:XXX
if (channelScopeComponents.size() < 3) {
raiseException(firstToken.startpos, XDDSLMessages.TAP_NEEDS_THREE_COMPONENTS);
}
Token tappingToken = channelScopeComponents.get(1);
String tapping = tappingToken.data.toLowerCase();
channelScopeComponents.remove(0); // remove 'tap'
if (tapping.equals("stream")) {
channelType = ChannelType.TAP_STREAM;
}
else if (tapping.equals("job")) {
channelType = ChannelType.TAP_JOB;
}
else if (tapping.equals("queue")) {
channelType = ChannelType.TAP_QUEUE;
}
else if (tapping.equals("topic")) {
channelType = ChannelType.TAP_TOPIC;
}
else {
raiseException(tappingToken.startpos, XDDSLMessages.NOT_ALLOWED_TO_TAP_THAT, tappingToken.data);
}
}
else {
// queue:XXX
// topic:XXX
if (firstToken.data.equalsIgnoreCase("queue")) {
channelType = ChannelType.QUEUE;
}
else if (firstToken.data.equalsIgnoreCase("topic")) {
channelType = ChannelType.TOPIC;
}
// TODO: DT not sure if this is the best way to handle
// StreamConfigParserTests.substreamsWithSourceChannels()
if (channelScopeComponents.size() >= 3) {
channelScopeComponents.remove(0);
}
}
int endpos = channelScopeComponents.get(channelScopeComponents.size() - 1).endpos;
if (!channelReferenceComponents.isEmpty()) {
endpos = channelReferenceComponents.get(channelReferenceComponents.size() - 1).endpos;
}
return new ChannelNode(channelType, firstToken.startpos, endpos, tokenListToStringList(channelScopeComponents),
tokenListToStringList(channelReferenceComponents));
}
private List<String> tokenListToStringList(List<Token> tokens) {
if (tokens.isEmpty()) {
return Collections.<String> emptyList();
}
List<String> data = new ArrayList<String>();
for (Token token : tokens) {
data.add(token.data);
}
return data;
}
// moduleList: module (| module)*
// A stream may end in a module (if it is a sink) or be followed by
// a sink channel.
private List<ModuleNode> eatModuleList() {
List<ModuleNode> moduleNodes = new ArrayList<ModuleNode>();
moduleNodes.add(eatModule());
while (moreTokens()) {
Token t = peekToken();
if (t.kind == TokenKind.PIPE) {
nextToken();
moduleNodes.add(eatModule());
}
else {
// might be followed by sink channel
break;
}
}
return moduleNodes;
}
// module: [label':']? identifier (moduleArguments)*
private ModuleNode eatModule() {
Token label = null;
Token name = nextToken();
if (!name.isKind(TokenKind.IDENTIFIER)) {
raiseException(name.startpos, XDDSLMessages.EXPECTED_MODULENAME, name.data != null ? name.data
: new String(name.getKind().tokenChars));
}
if (peekToken(TokenKind.COLON)) {
if (!isNextTokenAdjacent()) {
raiseException(peekToken().startpos, XDDSLMessages.NO_WHITESPACE_BETWEEN_LABEL_NAME_AND_COLON);
}
nextToken(); // swallow colon
label = name;
name = eatToken(TokenKind.IDENTIFIER);
}
Token moduleName = name;
checkpoint();
ArgumentNode[] args = maybeEatModuleArgs();
int startpos = label != null ? label.startpos : moduleName.startpos;
return new ModuleNode(toLabelNode(label), moduleName.data, startpos, moduleName.endpos, args);
}
private LabelNode toLabelNode(Token label) {
if (label == null) {
return null;
}
return new LabelNode(label.data, label.startpos, label.endpos);
}
// moduleArguments : DOUBLE_MINUS identifier(name) EQUALS identifier(value)
private ArgumentNode[] maybeEatModuleArgs() {
List<ArgumentNode> args = null;
if (peekToken(TokenKind.DOUBLE_MINUS) && isNextTokenAdjacent()) {
raiseException(peekToken().startpos, XDDSLMessages.EXPECTED_WHITESPACE_AFTER_MODULE_BEFORE_ARGUMENT);
}
while (peekToken(TokenKind.DOUBLE_MINUS)) {
Token dashDash = nextToken(); // skip the '--'
if (peekToken(TokenKind.IDENTIFIER) && !isNextTokenAdjacent()) {
raiseException(peekToken().startpos, XDDSLMessages.NO_WHITESPACE_BEFORE_ARG_NAME);
}
List<Token> argNameComponents = eatDottedName();
if (peekToken(TokenKind.EQUALS) && !isNextTokenAdjacent()) {
raiseException(peekToken().startpos, XDDSLMessages.NO_WHITESPACE_BEFORE_ARG_EQUALS);
}
eatToken(TokenKind.EQUALS);
if (peekToken(TokenKind.IDENTIFIER) && !isNextTokenAdjacent()) {
raiseException(peekToken().startpos, XDDSLMessages.NO_WHITESPACE_BEFORE_ARG_VALUE);
}
// Process argument value:
Token t = peekToken();
String argValue = eatArgValue();
checkpoint();
if (args == null) {
args = new ArrayList<ArgumentNode>();
}
args.add(new ArgumentNode(data(argNameComponents), argValue, dashDash.startpos, t.endpos));
}
return args == null ? null : args.toArray(new ArgumentNode[args.size()]);
}
// argValue: identifier | literal_string
private String eatArgValue() {
Token t = nextToken();
String argValue = null;
if (t.getKind() == TokenKind.IDENTIFIER) {
argValue = t.data;
}
else if (t.getKind() == TokenKind.LITERAL_STRING) {
argValue = t.data.substring(1, t.data.length() - 1).replaceAll("''", "'").replaceAll("\"\"", "\"");
}
else {
raiseException(t.startpos, XDDSLMessages.EXPECTED_ARGUMENT_VALUE, t.data);
}
return argValue;
}
private Token eatToken(TokenKind expectedKind) {
Token t = nextToken();
if (t == null) {
raiseException(expressionString.length(), XDDSLMessages.OOD);
}
if (t.kind != expectedKind) {
raiseException(t.startpos, XDDSLMessages.NOT_EXPECTED_TOKEN, expectedKind.toString().toLowerCase(),
t.getKind().toString().toLowerCase() + (t.data == null ? "" : "(" + t.data + ")"));
}
return t;
}
private boolean peekToken(TokenKind desiredTokenKind) {
return peekToken(desiredTokenKind, false);
}
private boolean lookAhead(int distance, TokenKind desiredTokenKind) {
if ((tokenStreamPointer + distance) >= tokenStream.size()) {
return false;
}
Token t = tokenStream.get(tokenStreamPointer + distance);
if (t.kind == desiredTokenKind) {
return true;
}
return false;
}
private List<Token> eatDottedName() {
return eatDottedName(XDDSLMessages.NOT_EXPECTED_TOKEN);
}
/**
* Consumes and returns (identifier [DOT identifier]*) as long as they're adjacent.
*
* @param error the kind of error to report if input is ill-formed
*/
private List<Token> eatDottedName(XDDSLMessages error) {
List<Token> result = new ArrayList<Token>(3);
Token name = nextToken();
if (!name.isKind(TokenKind.IDENTIFIER)) {
raiseException(name.startpos, error, name.data != null ? name.data
: new String(name.getKind().tokenChars));
}
result.add(name);
while (peekToken(TokenKind.DOT)) {
if (!isNextTokenAdjacent()) {
raiseException(peekToken().startpos, XDDSLMessages.NO_WHITESPACE_IN_DOTTED_NAME);
}
result.add(nextToken()); // consume dot
if (peekToken(TokenKind.IDENTIFIER) && !isNextTokenAdjacent()) {
raiseException(peekToken().startpos, XDDSLMessages.NO_WHITESPACE_IN_DOTTED_NAME);
}
result.add(eatToken(TokenKind.IDENTIFIER));
}
return result;
}
/**
* Verify the supplied name is a valid stream name. Valid stream names must follow the same rules as java
* identifiers, with the additional option to use a hyphen ('-') after the first character.
*
* @param streamname the name to validate
* @return true if name is valid
*/
public static boolean isValidStreamName(String streamname) {
if (streamname.length() == 0) {
return false;
}
if (!Character.isJavaIdentifierStart(streamname.charAt(0))) {
return false;
}
for (int i = 1, max = streamname.length(); i < max; i++) {
char ch = streamname.charAt(i);
if (!(Character.isJavaIdentifierPart(ch) || ch == '-')) {
return false;
}
}
return true;
}
/**
* Return the startPos of the first token in the list (must be non empty).
*/
private int startPos(Iterable<Token> many) {
Iterator<Token> iterator = many.iterator();
Assert.isTrue(iterator.hasNext(), "list of tokens must not be empty");
return iterator.next().startpos;
}
/**
* Return the endPos of the end token in the list (must be non empty).
*/
private int endPos(Iterable<Token> many) {
int result = -1;
for (Token t : many) {
result = t.endpos;
}
Assert.isTrue(result != -1, "list of tokens must not be empty");
return result;
}
/**
* Return the concatenation of the data of many tokens.
*/
private String data(Iterable<Token> many) {
StringBuilder result = new StringBuilder();
for (Token t : many) {
if (t.getKind().hasPayload()) {
result.append(t.data);
}
else {
result.append(t.getKind().tokenChars);
}
}
return result.toString();
}
private boolean peekToken(TokenKind desiredTokenKind, boolean consumeIfMatched) {
if (!moreTokens()) {
return false;
}
Token t = peekToken();
if (t.kind == desiredTokenKind) {
if (consumeIfMatched) {
tokenStreamPointer++;
}
return true;
}
else {
return false;
}
}
private boolean moreTokens() {
return tokenStreamPointer < tokenStream.size();
}
private Token nextToken() {
if (tokenStreamPointer >= tokenStreamLength) {
raiseException(expressionString.length(), XDDSLMessages.OOD);
}
return tokenStream.get(tokenStreamPointer++);
}
private boolean isNextTokenAdjacent() {
if (tokenStreamPointer >= tokenStreamLength) {
return false;
}
Token last = tokenStream.get(tokenStreamPointer - 1);
Token next = tokenStream.get(tokenStreamPointer);
return next.startpos == last.endpos;
}
private Token peekToken() {
if (tokenStreamPointer >= tokenStreamLength) {
return null;
}
return tokenStream.get(tokenStreamPointer);
}
private void raiseException(int pos, XDDSLMessages message, Object... inserts) {
throw new CheckpointedStreamDefinitionException(expressionString, pos, tokenStreamPointer, lastGoodPoint,
tokenStream, message, inserts);
}
private void checkpoint() {
lastGoodPoint = tokenStreamPointer;
}
private String toString(Token t) {
if (t.getKind().hasPayload()) {
return t.stringValue();
}
else {
return new String(t.kind.getTokenChars());
}
}
@Override
public String toString() {
StringBuilder s = new StringBuilder();
s.append(tokenStream).append("\n");
s.append("tokenStreamPointer=" + tokenStreamPointer).append("\n");
return s.toString();
}
// LookupEnvironment implementation
@Override
public StreamNode lookupStream(String name) {
if (this.repository != null) {
BaseDefinition baseDefinition = repository.findOne(name);
if (baseDefinition != null) {
StreamNode streamNode = new StreamConfigParser(repository).parse(baseDefinition.getDefinition());
return streamNode;
}
}
return null;
}
}