LogEventParsingIterator.java

/*
 * Copyright (C) 2015
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

package org.cleverbus.admin.services.log;

import java.io.Closeable;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Queue;

import org.cleverbus.common.log.Log;

import org.apache.commons.io.FileUtils;
import org.apache.commons.io.LineIterator;

/**
 * Iterates over provided log files, parsing and returning {@link LogEvent}s on demand.
 */
class LogEventParsingIterator implements Iterator<LogEvent>, Closeable {
    private final LogParser parser;
    private final LogParserConfig config;
    private final Queue<File> logFiles;

    private LineIterator lineIterator;

    private LogEvent parsedEvent; // event to be returned by this iterator on next()
    private LogEvent preParsedEvent; // event that was preParsed from a single line, but might have more lines

    private List<String> groupKey = null;

    private int totalCount = 0; // events found total
    private int fileEventsFound = -1; // events per file, resets to 0 on new file
    private int groupCount = 0; // events in current group, resets to 0 on new group

    LogEventParsingIterator(LogParser parser, LogParserConfig config, Collection<File> logFiles) throws IOException {
        this.parser = parser;
        this.config = config;
        this.logFiles = new LinkedList<File>(logFiles);
        nextEvent();
    }

    /**
     * Ensures that either {@link #parsedEvent} is not null, or the end is reached.
     *
     * @throws IOException if there's a problem opening a new file, while advancing to the next event
     */
    private void nextEvent() throws IOException {
        if (totalCount >= config.getLimit()) {
            Log.debug("Reached {} events limit - stopping", config.getLimit());
            close();
            return;
        }

        if (parsedEvent != null) {
            return;
        }

        boolean haveMore = !reachedEnd();
        while (parsedEvent == null && haveMore) {
            haveMore = seekToNextEvent();
            if (preParsedEvent != null && config.getGroupBy() != null && config.getGroupLimit() != null) {
                // grouping enabled - check group:
                List<String> nextGroupKey = getGroupKey(preParsedEvent, config);
                if (!nextGroupKey.equals(groupKey)) {
                    groupKey = nextGroupKey; // this event starts a new group
                    groupCount = 1; // reset group counter
                } else if (groupCount >= config.getGroupLimit()) {
                    preParsedEvent = null; // discard the pre-parsed event, as its group is full
                } else {
                    groupCount++;
                }
            }
        }

        if (parsedEvent != null) {
            fileEventsFound++;
            totalCount++;
        }
    }

    /**
     * Advance to the position where next line is available,
     * regardless of whether it passes filter requirements or not.
     * On the way to this position any lines that are not log events
     * will be appended to the previous log event.
     * <p/>
     * If this call is successful, either {@link #parsedEvent} will be a new event,
     * or {@link #preParsedEvent} will not be null; or both.
     *
     * @return true, if there might be more events; false otherwise
     * @throws IOException if there's a problem opening a new file
     */
    private boolean seekToNextEvent() throws IOException {
        LogEvent nextEvent = config.createLogEvent();

        LineIterator iterator;
        while ((iterator = getLineIterator()) != null) {
            while (iterator.hasNext()) {
                // process the line:
                LogEvent event = parser.parseLine(iterator.next(), nextEvent, preParsedEvent, config);
                // check what to do next:
                if (preParsedEvent != null && event != preParsedEvent) {
                    // there is pre-parsed event, but the line was NOT appended to it
                    // => pre-parsed event will not get any more lines, it can be considered fully parsed
                    parsedEvent = preParsedEvent; // graduate pre-parsed event - it'll be the next event
                    preParsedEvent = event; // next event that was found (if any) is now pre-parsed
                    return true; // found full pre-parsed event => success
                } else if (event == nextEvent) {
                    // nextEvent is now pre-parsed, but previous pre-parsed event is null, so no event graduated
                    preParsedEvent = nextEvent;
                    return true;
                }
                // otherwise line was ignored or added to the pre-parsed event, nothing really changed
            }
        }
        return false; // failure
    }

    private boolean reachedEnd() throws IOException {
        return getLineIterator() == null;
    }

    /**
     * Returns a line iterator to process next/current file, opening the next file, if necessary.
     *
     * @return line iterator for the current/next file; or null if no files left
     */
    private LineIterator getLineIterator() throws IOException {
        if (lineIterator != null && lineIterator.hasNext()) {
            return lineIterator;
        }

        // discard current line iterator
        LineIterator.closeQuietly(lineIterator);
        lineIterator = null;

        if (fileEventsFound == 0) {
            Log.debug("No events in the last file, closing prematurely");
            close(); // optimize: last file had no events, no point continuing
        } else if (!logFiles.isEmpty()) {
            File file = logFiles.poll();
            Log.debug("Opening {}", file);
            lineIterator = FileUtils.lineIterator(file);
            fileEventsFound = 0; // restart per-file counter
        }

        return lineIterator;
    }


    private List<String> getGroupKey(LogEvent event, LogParserConfig config) {
        List<String> groupKey = new ArrayList<String>(config.getGroupBy().size());
        for (int propertyIndex = 0; propertyIndex < event.getPropertyCount(); propertyIndex++) {
            if (config.getGroupBy().contains(event.getPropertyNames().get(propertyIndex))) {
                groupKey.add(String.valueOf(event.getProperties()[propertyIndex]));
            }
        }
        return groupKey;
    }

    @Override
    public boolean hasNext() {
        return parsedEvent != null;
    }

    @Override
    public LogEvent next() {
        LogEvent result = parsedEvent;
        parsedEvent = null;

        try {
            nextEvent();
        } catch (IOException exc) {
            throw new IllegalStateException("Error prefetching next event", exc);
        }
        return result;
    }

    @Override
    public void remove() {
        throw new UnsupportedOperationException();
    }

    @Override
    public void close() {
        LineIterator.closeQuietly(lineIterator);
        lineIterator = null;
        logFiles.clear();
        Log.debug("Closed");
    }
}