/*
* SonarQube
* Copyright (C) 2009-2017 SonarSource SA
* mailto:info AT sonarsource DOT com
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.sonar.duplications.token;
import java.io.Reader;
import java.io.StringReader;
import org.sonar.channel.ChannelDispatcher;
import org.sonar.channel.CodeReader;
import org.sonar.duplications.DuplicationsException;
public final class TokenChunker {
private final ChannelDispatcher<TokenQueue> channelDispatcher;
public static Builder builder() {
return new Builder();
}
private TokenChunker(Builder builder) {
this.channelDispatcher = builder.getChannelDispatcher();
}
public TokenQueue chunk(String sourceCode) {
return chunk(new StringReader(sourceCode));
}
public TokenQueue chunk(Reader reader) {
CodeReader code = new CodeReader(reader);
TokenQueue queue = new TokenQueue();
try {
channelDispatcher.consume(code, queue);
return queue;
} catch (Exception e) {
throw new DuplicationsException("Unable to lex source code at line : " + code.getLinePosition() + " and column : " + code.getColumnPosition(), e);
}
}
/**
* Note that order is important, e.g.
* <code>token("A").ignore("A")</code> for the input string "A" will produce token, whereas
* <code>ignore("A").token("A")</code> will not.
*/
public static final class Builder {
private ChannelDispatcher.Builder channelDispatcherBuilder = ChannelDispatcher.builder();
private Builder() {
}
public TokenChunker build() {
return new TokenChunker(this);
}
/**
* Defines that sequence of characters must be ignored, if it matches specified regular expression.
*/
public Builder ignore(String regularExpression) {
channelDispatcherBuilder.addChannel(new BlackHoleTokenChannel(regularExpression));
return this;
}
/**
* Defines that sequence of characters, which is matched specified regular expression, is a token.
*/
public Builder token(String regularExpression) {
channelDispatcherBuilder.addChannel(new TokenChannel(regularExpression));
return this;
}
/**
* Defines that sequence of characters, which is matched specified regular expression, is a token with specified value.
*/
public Builder token(String regularExpression, String normalizationValue) {
channelDispatcherBuilder.addChannel(new TokenChannel(regularExpression, normalizationValue));
return this;
}
private ChannelDispatcher<TokenQueue> getChannelDispatcher() {
return channelDispatcherBuilder.build();
}
}
}