MemBuffered.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.hadoop.chukwa.datacollection.adaptor;

import static org.apache.hadoop.chukwa.datacollection.adaptor.AdaptorShutdownPolicy.RESTARTING;
import java.util.*;
import org.apache.hadoop.chukwa.Chunk;
import org.apache.hadoop.chukwa.datacollection.ChunkReceiver;
import org.apache.hadoop.chukwa.datacollection.agent.AdaptorManager;

public class MemBuffered extends AbstractWrapper {
  
  static final String BUF_SIZE_OPT = "adaptor.memBufWrapper.size";
  static final int DEFAULT_BUF_SIZE = 1024*1024; //1 MB
  
  //true by default. If you were willing to discard data, you didn't need Mem Buffers
  static boolean BLOCK_WHEN_FULL = true;
  
  static class MemBuf {
    long dataSizeBytes;
    final long maxDataSize;
    final ArrayDeque<Chunk> chunks;
    
    public MemBuf(long maxDataSize) {
      dataSizeBytes = 0;
      this.maxDataSize = maxDataSize;
      chunks = new ArrayDeque<Chunk>();
    }
    
    synchronized void add(Chunk c) throws InterruptedException{
      int len = c.getData().length;
      if(BLOCK_WHEN_FULL)
        while(len + dataSizeBytes > maxDataSize)
          wait();
      else
        chunks.remove();
      dataSizeBytes += len;
      chunks.add(c);
    }
    
    synchronized void removeUpTo(long l) {

      long bytesFreed = 0;
      while(!chunks.isEmpty()) {
        Chunk c = chunks.getFirst();
        if(c.getSeqID() > l)
          chunks.addFirst(c);
        else
          bytesFreed += c.getData().length;
      }
      
      if(bytesFreed > 0) {
        dataSizeBytes -= bytesFreed;
        notifyAll();
      }
    }
    
  }

  static Map<String, MemBuf> buffers;
  static {
    buffers = new HashMap<String, MemBuf>();
  }
  
  MemBuf myBuffer;
  
  @Override
  public void add(Chunk event) throws InterruptedException {
    myBuffer.add(event);
    dest.add(event);
  }
  
  @Override
  public void start(String adaptorID, String type, long offset,
      ChunkReceiver dest) throws AdaptorException {
    try {
      String dummyAdaptorID = adaptorID;
      this.adaptorID = adaptorID;
      this.dest = dest;
      
      long bufSize = manager.getConfiguration().getInt(BUF_SIZE_OPT, DEFAULT_BUF_SIZE);
      synchronized(buffers) {
        myBuffer = buffers.get(adaptorID);
        if(myBuffer == null) {
          myBuffer = new MemBuf(bufSize);
          buffers.put(adaptorID, myBuffer);
        }
      }

      //Drain buffer into output queue
      long offsetToStartAt = offset;
      for(Chunk c:myBuffer.chunks) {
        dest.add(c);
        long seq = c.getSeqID();
        if(seq > offsetToStartAt)
          offsetToStartAt = seq;
      }
      
      inner.start(dummyAdaptorID, innerType, offsetToStartAt, this);
    } catch(InterruptedException e) {
     throw new AdaptorException(e);
    }
  }
  
  @Override
  public void committed(long l) {
    myBuffer.removeUpTo(l);
  }
  
  @Override
  public long shutdown(AdaptorShutdownPolicy p) throws AdaptorException {
    if(p != RESTARTING)
      buffers.remove(adaptorID);    
    return inner.shutdown(p);
  }


}