DataOperations.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.hadoop.chukwa.validationframework.util;


import java.io.DataInputStream;
import java.io.EOFException;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.net.URI;
import org.apache.hadoop.chukwa.ChukwaArchiveKey;
import org.apache.hadoop.chukwa.Chunk;
import org.apache.hadoop.chukwa.ChunkImpl;
import org.apache.hadoop.chukwa.conf.ChukwaConfiguration;
import org.apache.hadoop.chukwa.extraction.engine.ChukwaRecord;
import org.apache.hadoop.chukwa.extraction.engine.ChukwaRecordKey;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.SequenceFile;
import org.apache.log4j.Logger;

public class DataOperations {
  static Logger log = Logger.getLogger(DataOperations.class);

  public static void copyFile(String fromFileName, String toFileName)
      throws IOException {
    File fromFile = new File(fromFileName);
    File toFile = new File(toFileName);

    FileInputStream from = null;
    FileOutputStream to = null;
    try {
      from = new FileInputStream(fromFile);
      to = new FileOutputStream(toFile);
      byte[] buffer = new byte[4096];
      int bytesRead;

      while ((bytesRead = from.read(buffer)) != -1)
        to.write(buffer, 0, bytesRead); // write
    } finally {
      if (from != null)
        try {
          from.close();
        } catch (IOException e) {
          ;
        }
      if (to != null)
        try {
          to.close();
        } catch (IOException e) {
          // ;
        }
    }
  }

  public static boolean validateMD5(String inputFile, String testFile) {
    // System.out.println("validateMD5 [" + inputFile + "] [" + testFile
    // + "]");
    String md5_1 = MD5.checksum(new File(inputFile));
    String md5_2 = MD5.checksum(new File(testFile));
    // System.out.println("MD5 [" + md5_1 + "] [" + md5_2 + "]");
    return md5_1.intern() == md5_2.intern();
  }

  public static boolean validateMD5(FileSystem fs, Path inputFile, Path testFile) {
    // System.out.println("validateMD5 [" + inputFile + "] [" + testFile
    // + "]");
    String md5_1 = MD5.checksum(fs, inputFile);
    String md5_2 = MD5.checksum(fs, testFile);
    // System.out.println("MD5 [" + md5_1 + "] [" + md5_2 + "]");
    return md5_1.intern() == md5_2.intern();
  }

  public static boolean validateChukwaRecords(FileSystem fs,
      Configuration conf, Path inputFile, Path testFile) {
    SequenceFile.Reader goldReader = null;
    SequenceFile.Reader testReader = null;
    try {
      // log.info(">>>>>>>>>>>>>> Openning records [" + inputFile.getName()
      // +"][" + testFile.getName() +"]");
      goldReader = new SequenceFile.Reader(fs, inputFile, conf);
      testReader = new SequenceFile.Reader(fs, testFile, conf);

      ChukwaRecordKey goldKey = new ChukwaRecordKey();
      ChukwaRecord goldRecord = new ChukwaRecord();

      ChukwaRecordKey testKey = new ChukwaRecordKey();
      ChukwaRecord testRecord = new ChukwaRecord();

      // log.info(">>>>>>>>>>>>>> Start reading");
      while (goldReader.next(goldKey, goldRecord)) {
        testReader.next(testKey, testRecord);

        if (goldKey.compareTo(testKey) != 0) {
          log.info(">>>>>>>>>>>>>> Not the same Key");
          log.info(">>>>>>>>>>>>>> Record [" + goldKey.getKey() + "] ["
              + goldKey.getReduceType() + "]");
          log.info(">>>>>>>>>>>>>> Record [" + testKey.getKey() + "] ["
              + testKey.getReduceType() + "]");
          return false;
        }

        if (goldRecord.compareTo(testRecord) != 0) {
          log.info(">>>>>>>>>>>>>> Not the same Value");
          log.info(">>>>>>>>>>>>>> Record [" + goldKey.getKey() + "] ["
              + goldKey.getReduceType() + "]");
          log.info(">>>>>>>>>>>>>> Record [" + testKey.getKey() + "] ["
              + testKey.getReduceType() + "]");
          log.info(">>>>>>>>>>>>>> Gold Value [" + goldRecord.toString() + "]");
          log.info(">>>>>>>>>>>>>> Test value [" + testRecord.toString() + "]");

          return false;
        }
      }
      // log.info(">>>>>>>>>>>>>> Same File");
      return true;
    } catch (IOException e) {
      e.printStackTrace();
      return false;
    } finally {
      try {
        goldReader.close();
        testReader.close();
      } catch (IOException e) {
      }

    }
  }

  public static void extractRawLogFromdataSink(String directory, String fileName)
      throws Exception {
    ChukwaConfiguration conf = new ChukwaConfiguration();
    String fsName = conf.get("writer.hdfs.filesystem");
    FileSystem fs = FileSystem.get(new URI(fsName), conf);

    SequenceFile.Reader r = new SequenceFile.Reader(fs, new Path(directory
        + fileName + ".done"), conf);

    File outputFile = new File(directory + fileName + ".raw");

    ChukwaArchiveKey key = new ChukwaArchiveKey();
    ChunkImpl chunk = ChunkImpl.getBlankChunk();
    FileWriter out = new FileWriter(outputFile);
    try {
      while (r.next(key, chunk)) {
        out.write(new String(chunk.getData()));
      }
    } finally {
      out.close();
      r.close();
    }

  }

  public static void extractRawLogFromDump(String directory, String fileName)
      throws Exception {
    File inputFile = new File(directory + fileName + ".bin");
    File outputFile = new File(directory + fileName + ".raw");
    DataInputStream dis = new DataInputStream(new FileInputStream(inputFile));
    Chunk chunk = null;
    FileWriter out = new FileWriter(outputFile);
    boolean eof = false;
    do {
      try {
        chunk = ChunkImpl.read(dis);
        out.write(new String(chunk.getData()));
      } catch (EOFException e) {
        eof = true;
      }

    } while (!eof);

    dis.close();
    out.close();
  }
}