C24ItemReaderParallelTests.java

/*
 * Copyright 2012 C24 Technologies
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *          http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package biz.c24.io.spring.batch.reader;

import static org.hamcrest.CoreMatchers.is;
import static org.junit.Assert.assertThat;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;

import java.io.IOException;
import java.util.Collection;
import java.util.Collections;
import java.util.LinkedList;

import org.junit.Test;
import org.springframework.batch.core.JobParameters;
import org.springframework.batch.core.StepExecution;
import org.springframework.batch.item.ItemReader;
import org.springframework.batch.item.NonTransientResourceException;
import org.springframework.batch.item.ParseException;
import org.springframework.batch.item.UnexpectedInputException;
import org.springframework.core.io.ClassPathResource;

import biz.c24.io.api.data.ComplexDataObject;
import biz.c24.io.api.data.ValidationException;
import biz.c24.io.examples.models.basic.EmployeeElement;
import biz.c24.io.spring.batch.reader.source.SplittingReaderSource;
import biz.c24.io.spring.batch.reader.source.ZipFileSource;
import biz.c24.io.spring.core.C24Model;
import biz.c24.io.spring.source.SourceFactory;

/**
 * Validate the C24ItemReader works correctly in parallel
 * 
 * @author Andrew Elmore
 *
 */
public class C24ItemReaderParallelTests {
    private C24Model employeeModel = new C24Model(EmployeeElement.getInstance());
    private C24Model employeeXmlModel = new C24Model(biz.c24.io.examples.models.xml.EmployeeElement.getInstance());

    
    @Test
    public void testValidZipSingleRead() throws UnexpectedInputException, ParseException, NonTransientResourceException, IOException, ValidationException {
        
        ZipFileSource source = new ZipFileSource();
        source.setResource(new ClassPathResource("employees-100-valid-individual-noparent.xml.zip"));
        
        // No validation, no splitting
        // Unsupported due to XMLSource not detecting it has no data before it starts parsing
        // As it caches data internally we can't make the decision for it
        
        Collection<ComplexDataObject> objs = readFile(employeeXmlModel, null, null, false, source);
        assertThat(objs.size(), is(100));
        
        // Validation but no splitting
        objs = readFile(employeeXmlModel, null, null, true, source);
        assertThat(objs.size(), is(100));
        assertThat(source.useMultipleThreadsPerReader(), is(false));
        
        
        // Validation & splitting - start pattern only
        objs = readFile(employeeXmlModel, ".*<employee.*", null, true, source);
        assertThat(objs.size(), is(100));
        assertThat(source.useMultipleThreadsPerReader(), is(false));

        // Validation & splitting - start & stop patterns
        objs = readFile(employeeXmlModel, ".*<employee.*", ".*/>.*", true, source);
        assertThat(objs.size(), is(100));
        assertThat(source.useMultipleThreadsPerReader(), is(false));
    }
    
    @Test
    public void testValidZipCombinedRead() throws UnexpectedInputException, ParseException, NonTransientResourceException, IOException, ValidationException {
        
        ZipFileSource source = new ZipFileSource();
        source.setResource(new ClassPathResource("employees-1500-valid-combined.xml.zip"));

        // Validation & splitting - start pattern only
        Collection<ComplexDataObject> objs = readFile(employeeXmlModel, ".*<employee .*", ".*/>.*", true, source);
        assertThat(objs.size(), is(1500));
        assertThat(source.useMultipleThreadsPerReader(), is(true));
    }
    
    @Test
    public void testValidZipCombinedNoParentRead() throws UnexpectedInputException, ParseException, NonTransientResourceException, IOException, ValidationException {
        
        ZipFileSource source = new ZipFileSource();
        source.setResource(new ClassPathResource("employees-1500-valid-combined-noparent.xml.zip"));
        
        // Validation & splitting - start pattern only
        Collection<ComplexDataObject> objs = readFile(employeeXmlModel, ".*<employee .*", null, true, source);
        assertThat(objs.size(), is(1500));
        assertThat(source.useMultipleThreadsPerReader(), is(true));

        // Validation & splitting - start & stop patterns
        objs = readFile(employeeXmlModel, ".*<employee .*", ".*/>.*", true, source);
        assertThat(objs.size(), is(1500));
        assertThat(source.useMultipleThreadsPerReader(), is(true));
    }
    
    
    private Collection<ComplexDataObject> readFile(C24Model model, String optionalElementStartRegEx, String optionalElementStopRegEx, boolean validate, SplittingReaderSource source) throws IOException, UnexpectedInputException, ParseException, NonTransientResourceException, ValidationException {
        return readFile(model, optionalElementStartRegEx, optionalElementStopRegEx, validate, source, null);
    }

    private Collection<ComplexDataObject> readFile(C24Model model, String optionalElementStartRegEx, String optionalElementStopRegEx, boolean validate, SplittingReaderSource source, SourceFactory factory) throws IOException, UnexpectedInputException, ParseException, NonTransientResourceException, ValidationException { 
        C24ItemReader<ComplexDataObject> reader = new C24ItemReader<ComplexDataObject>();
        reader.setModel(model);
        if(optionalElementStartRegEx != null) {
            reader.setElementStartPattern(optionalElementStartRegEx);
        }
        if(optionalElementStopRegEx != null) {
            reader.setElementStopPattern(optionalElementStopRegEx);
        }
        if(factory != null) {
            reader.setSourceFactory(factory);
        }
        
        reader.setSource(source);
        reader.setValidate(validate);
        
        StepExecution stepExecution = getStepExecution();
        
        reader.setup(stepExecution);

        Collection<ComplexDataObject> objs = Collections.synchronizedList(new LinkedList<ComplexDataObject>());
        
        Thread[] threads = new Thread[8];
        for(int i=0; i < threads.length; i++) {
            threads[i] = new Thread(new Worker<ComplexDataObject>(model, objs, reader));
            threads[i].start();
        }
        
        for(int i=0; i < threads.length; i++) {
            try {
                threads[i].join(20000);
            } catch (InterruptedException e) {
                throw new RuntimeException("Interrupted waiting on thread");
            }
            if(threads[i].isAlive()) {
                throw new RuntimeException("Timed out waiting for thread to complete its work");
            }
        }
        
        reader.cleanup();
        
        return objs;
    }
    
    private static class Worker<T extends ComplexDataObject> implements Runnable {
        private C24Model model;
        private Collection<T> collection;
        private ItemReader<T> reader;
        
        public Worker(C24Model model, Collection<T> collection, ItemReader<T> itemReader) {
            this.model = model;
            this.collection = collection;
            this.reader = itemReader;
        }
        
        public void run() {
            T obj = null;
            try {
                while((obj = reader.read()) != null) {
                    assertThat(obj.getDefiningElementDecl(), is(model.getRootElement()));
                    collection.add(obj);                  
                }
            } catch (Exception e) {
                throw new RuntimeException(e);
            }
        }
    }
        
    private StepExecution getStepExecution() throws IOException {
        
        JobParameters jobParams = mock(JobParameters.class);

        StepExecution stepExecution = mock(StepExecution.class);
        when(stepExecution.getJobParameters()).thenReturn(jobParams);
        
        return stepExecution;
        
    }
    
}