package com.zillabyte.motherbrain.utils;
import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.net.Inet4Address;
import java.net.InetAddress;
import java.net.NetworkInterface;
import java.net.SocketException;
import java.nio.channels.ClosedByInterruptException;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.Arrays;
import java.util.Collections;
import java.util.Enumeration;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicLong;
import java.util.regex.Pattern;
import javax.annotation.RegEx;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.lang.NotImplementedException;
import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.commons.lang.mutable.MutableObject;
import org.apache.log4j.Logger;
import org.apache.tools.ant.types.Commandline;
import org.eclipse.jdt.annotation.NonNull;
import com.github.rholder.retry.RetryException;
import com.github.rholder.retry.Retryer;
import com.github.rholder.retry.RetryerBuilder;
import com.github.rholder.retry.StopStrategies;
import com.github.rholder.retry.WaitStrategies;
import com.google.common.collect.Lists;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import com.google.monitoring.runtime.instrumentation.common.com.google.common.base.Throwables;
import com.zillabyte.motherbrain.shell.MachineType;
import com.zillabyte.motherbrain.universe.Universe;
public final class Utils {
private static final int SCHEDULE_POOL_SIZE = 16;
private static Logger execLog = Utils.getLogger(Utils.class);
private static ExecutorService _executorPool = Executors.newCachedThreadPool(new ThreadFactoryBuilder().setNameFormat("util-pool-%s").build());
private static ScheduledExecutorService _scheduleExecutorPool = Executors.newScheduledThreadPool(SCHEDULE_POOL_SIZE, new ThreadFactoryBuilder().setNameFormat("util-schedule-pool-%s").build());
private static String _cachedUserName = null;
public static ExecutorService createPrefixedExecutorPool(String prefix) {
return Executors.newCachedThreadPool(new ThreadFactoryBuilder().setNameFormat(prefix + "-%s").build());
}
/***
* If dest only ever monotonically increases, this function is guaranteed to return with
* dest having a value of at least tryValue and preserves monotonicity.
* @param dest
* @param tryValue
* @return
*/
public static final boolean compareAndSetIfGreater(final AtomicLong dest, final long tryValue) {
long destValue;
do {
destValue = dest.get();
if (tryValue <= destValue) {
return false;
}
} while (!dest.compareAndSet(destValue, tryValue));
return true;
}
/***
*
* @param clazz
* @return
*/
public static final Logger getLogger(final @NonNull Class<?> clazz) {
return Logger.getLogger(clazz);
}
/***
*
* @return
*/
public static String getSystemUserName() {
if (_cachedUserName == null) {
_cachedUserName = System.getProperty("user.name");
}
return _cachedUserName;
}
/***
*
* @return
*/
public static final MachineType getMachineType() {
switch (getSystemUserName().toLowerCase()) {
case "ubuntu":
return MachineType.UBUNTU_EC2;
case "vagrant":
return MachineType.UBUNTU_VAGRANT;
case "root":
return MachineType.UBUNTU_TEAMCITY;
default:
return MachineType.OSX_LOCAL;
}
}
/***
*
* @param runnable
* @return
*/
public static Future<?> run(Runnable runnable) {
return _executorPool.submit(runnable);
}
public static <T> Future<T> run(Callable<T> callable) {
return _executorPool.submit(callable);
}
private static List<Future> _futures = Lists.newLinkedList();
private static <T extends Future> T trackFuture(T future) {
synchronized(_futures) {
_futures.add(future);
return future;
}
}
public static void killAllFutures() {
synchronized(_futures) {
for(Future f : _futures) {
if ( !f.isDone() && !f.isCancelled()) {
execLog.warn("killing future: " + f);
f.cancel(true);
}
}
_futures.clear();
_scheduleExecutorPool.shutdown();
_executorPool.shutdown();
_executorPool = Executors.newCachedThreadPool();
_scheduleExecutorPool = Executors.newScheduledThreadPool(SCHEDULE_POOL_SIZE);
}
}
/***
*
* @param delay
* @param unit
* @param command
* @return
*/
public static ScheduledFuture<?> schedule(long delay, TimeUnit unit, Runnable command) {
return trackFuture(_scheduleExecutorPool.schedule(command, delay, unit));
}
public static ScheduledFuture<?> schedule(long delay, Runnable command) {
return schedule(delay, TimeUnit.MILLISECONDS, command);
}
public static <T> ScheduledFuture<T> schedule(long delay, TimeUnit unit, Callable<T> command) {
return trackFuture(_scheduleExecutorPool.schedule(command, delay, unit));
}
public static <T> ScheduledFuture<T> schedule(long delay, Callable<T> command) {
return schedule(delay, TimeUnit.MILLISECONDS, command);
}
public static ScheduledFuture<?> scheduleDedicated(long delay, TimeUnit unit, Runnable command) {
return trackFuture(Executors.newSingleThreadScheduledExecutor().schedule(command, delay, unit));
}
public static ScheduledFuture<?> scheduleDedicated(long delay, Runnable command) {
return schedule(delay, TimeUnit.MILLISECONDS, command);
}
public static <T> ScheduledFuture<T> scheduleDedicated(long delay, TimeUnit unit, Callable<T> command) {
return trackFuture(Executors.newSingleThreadScheduledExecutor().schedule(command, delay, unit));
}
public static <T> ScheduledFuture<T> scheduleDedicated(long delay, Callable<T> command) {
return schedule(delay, TimeUnit.MILLISECONDS, command);
}
/***
*
* @param ms
* @param runnable
* @throws InterruptedException
* @throws ExecutionException
* @throws TimeoutException
*/
public static void executeWithin(long ms, Runnable runnable) throws InterruptedException, ExecutionException, TimeoutException {
executeWithin(ms, TimeUnit.MILLISECONDS, runnable);
}
public static void executeWithin(long time, TimeUnit unit, Runnable runnable) throws InterruptedException, ExecutionException, TimeoutException {
Future<?> future = Utils.run(runnable);
future.get(time, unit);
}
public static <T> T executeWithin(long ms, Callable<T> runnable) throws InterruptedException, ExecutionException, TimeoutException {
return executeWithin(ms, TimeUnit.MILLISECONDS, runnable);
}
public static <T> T executeWithin(long time, TimeUnit unit, Callable<T> runnable) throws InterruptedException, ExecutionException, TimeoutException {
Future<T> future = Utils.run(runnable);
return future.get(time, unit);
}
/***
*
* @param initialDelay
* @param delay
* @param unit
* @param command
* @return
*/
public static ScheduledFuture<?> timerFromPool(long initialDelay, long delay, TimeUnit unit, Runnable command) {
return _scheduleExecutorPool.scheduleWithFixedDelay(command, initialDelay, delay, unit);
}
public static ScheduledFuture<?> timerFromPool(long initialDelay, long delay, Runnable command) {
return timerFromPool(initialDelay, delay, TimeUnit.MILLISECONDS, command);
}
public static ScheduledFuture<?> timerFromPool(long delay, Runnable command) {
return timerFromPool(0, delay, TimeUnit.MILLISECONDS, command);
}
public static ScheduledFuture<?> timerFromPool(long delay, TimeUnit unit, Runnable command) {
return timerFromPool(0, delay, unit, command);
}
/****
*
* @param initialDelay
* @param delay
* @param unit
* @param command
* @return
*/
public static ScheduledFuture<?> timerDedicated(long initialDelay, long delay, TimeUnit unit, Runnable command) {
return trackFuture(Executors.newSingleThreadScheduledExecutor().scheduleWithFixedDelay(command, initialDelay, delay, unit));
}
public static ScheduledFuture<?> timerDedicated(long initialDelay, long delay, Runnable command) {
return timerDedicated(initialDelay, delay, TimeUnit.MILLISECONDS, command);
}
public static ScheduledFuture<?> timerDedicated(long delay, Runnable command) {
return timerDedicated(0, delay, TimeUnit.MILLISECONDS, command);
}
public static ScheduledFuture<?> timerDedicated(long delay, TimeUnit unit, Runnable command) {
return timerDedicated(0, delay, unit, command);
}
/**
*
* @param path
* @return
*/
public static String expandPath(String path) {
if (path.startsWith("~" + File.separator)) {
return System.getProperty("user.home") + path.substring(1);
}
return path;
}
/***
*
* @param e
* @param klass
* @return
*/
public static <T extends Throwable> T getRootException(final Throwable e, final Class<T> klass) {
// We can infinitely recurse examining getCause here, so we use a set to keep
// track of what we've seen. In theory someone devious could probably rewrite
// getCause() to return a different Throwable each time AND recurse infinitely,
// but hopefully that will be rare.
Set<Throwable> seenSet = new HashSet<>();
Throwable ex = e;
T root = null;
while (ex != null) {
if (seenSet.contains(ex)) {
// infinite loop, abort
return null;
}
seenSet.add(ex);
if (klass.isInstance(ex)) {
root = klass.cast(ex);
}
ex = ex.getCause();
}
return root;
}
/***
*
* @param e
* @param klass
* @return
*/
public static <T extends Throwable> T getInitialException(final Throwable e, final Class<T> klass) {
// We can infinitely recurse examining getCause here, so we use a set to keep
// track of what we've seen. In theory someone devious could probably rewrite
// getCause() to return a different Throwable each time AND recurse infinitely,
// but hopefully that will be rare.
Set<Throwable> seenSet = new HashSet<>();
Throwable ex = e;
while (ex != null) {
if (seenSet.contains(ex)) {
// infinite loop, abort
return null;
}
seenSet.add(ex);
if (klass.isInstance(ex)) {
return klass.cast(ex);
}
ex = ex.getCause();
}
return null;
}
/***
*
* @param ex
* @return
* @throws InterruptedException
*/
public static <T extends Throwable> T handleInterruptible(final T ex) throws InterruptedException {
if (getInitialException(ex, ClosedByInterruptException.class) != null) {
// This is how you detect InterruptedException from within BufferedReader.
// I figured this out by looking at the source code for the GNU nio Channels
// interface but it's backed up by
// http://java.sun.com/j2se/1.5.0/docs/api/java/nio/channels/ClosedByInterruptException.html
throw (InterruptedException) new InterruptedException().initCause(ex);
}
if (getInitialException(ex, InterruptedException.class) != null) {
// There was an underlying InterruptedException that was chained below another error.
throw (InterruptedException) new InterruptedException().initCause(ex);
}
// The exception probably wasn't an InterruptedException.
return ex;
}
/***
*
* @param command
* @param stdout
* @param stderr
* @return
* @throws InterruptedException
* @throws IOException
*/
public static int shell(Map<String,String> env, String[] command, final StringBuilder stdout, final StringBuilder stderr, MutableObject processCapture) throws InterruptedException, IOException {
// Init
ProcessBuilder pb = new ProcessBuilder(command);
pb.environment().putAll(Universe.instance().shellFactory().getEnvironment());
pb.environment().putAll(env);
execLog.info("shell: " + Arrays.toString( command ) + " " + (env.size() > 0 ? "with env:" + env : ""));
Process proc;
try {
proc = pb.start();
if (processCapture != null) processCapture.setValue(proc);
} catch (IOException e) {
throw handleInterruptible(e);
}
final BufferedReader stderrBuffer = new BufferedReader(new InputStreamReader(proc.getErrorStream()));
final BufferedReader stdoutBuffer = new BufferedReader(new InputStreamReader(proc.getInputStream()));
Future<?> stderrThread = run(new Callable<Void>() {
@Override
public Void call() throws IOException {
while (!Thread.currentThread().isInterrupted()) {
final String line;
try {
line = stderrBuffer.readLine();
} catch(IOException e) {
try {
throw handleInterruptible(e);
} catch (InterruptedException e1) {
return null;
}
}
if (line == null) {
return null;
}
execLog.info("STDERR: " + line);
if (stderr != null) {
stderr.append(line);
stderr.append("\n");
}
}
return null;
}
});
Future<?> stdoutThread = run(new Callable<Void>() {
@Override
public Void call() throws IOException {
while (!Thread.currentThread().isInterrupted()) {
final String line;
try {
line = stdoutBuffer.readLine();
} catch(IOException e) {
try {
throw handleInterruptible(e);
} catch (InterruptedException e1) {
return null;
}
}
if (line == null) {
return null;
}
execLog.info("stdout: " + line);
if (stdout != null) {
stdout.append(line);
stdout.append("\n");
}
}
return null;
}
});
final int exitCode;
try {
exitCode = proc.waitFor();
} finally {
try {
stdoutThread.get();
} catch (ExecutionException e) {
execLog.error(e, e.getCause());
}
try {
stderrThread.get();
} catch (ExecutionException e) {
execLog.error(e, e.getCause());
}
}
return exitCode;
}
public static int shell(String[] command, final StringBuilder stdout, final StringBuilder stderr, MutableObject processCapture) throws InterruptedException, IOException {
return shell(Collections.EMPTY_MAP, command, stdout, stderr, processCapture);
}
public static int shell(String[] command, final StringBuilder stdout, final StringBuilder stderr) throws InterruptedException, IOException {
return shell(command, stdout, stderr, null);
}
public static int shell(String command) throws InterruptedException, IOException {
return shell(Commandline.translateCommandline(command), null, null);
}
public static int shell(String command, final StringBuilder stdout, final StringBuilder stderr, MutableObject processCapture) throws InterruptedException, IOException {
return shell(Commandline.translateCommandline(command), stdout, stderr, processCapture);
}
public static int shell(String command, final StringBuilder stdout, final StringBuilder stderr) throws InterruptedException, IOException {
return shell(Commandline.translateCommandline(command), stdout, stderr);
}
public static int shell(String command, final StringBuilder stdout) throws InterruptedException, IOException {
return shell(Commandline.translateCommandline(command), stdout, null);
}
public static int shell(String... command) throws InterruptedException, IOException {
return shell(command, null, null);
}
public static int shell(Map<String,String> env, String... command) throws InterruptedException, IOException {
return shell(env, command, null, null, null);
}
public static int shell(Map<String, String> env, String command, StringBuilder stdout) throws InterruptedException, IOException {
return shell(env, Commandline.translateCommandline(command), stdout, null, null);
}
public static int shell(Map<String, String> env, String command, StringBuilder stdout, StringBuilder stderr) throws InterruptedException, IOException {
return shell(env, Commandline.translateCommandline(command), stdout, stderr, null);
}
/***
*
* @param obj
* @return
*/
public static byte[] serialize(Object obj) {
final byte[] objBytes;
// Stolen from storm
try {
ByteArrayOutputStream bos = new ByteArrayOutputStream();
ObjectOutputStream oos = new ObjectOutputStream(bos);
oos.writeObject(obj);
oos.close();
objBytes = bos.toByteArray();
} catch(IOException ioe) {
throw new RuntimeException(ioe);
}
assert (objBytes != null);
return objBytes;
}
/***
*
* @param b
* @return
*/
@SuppressWarnings("unchecked")
public static <T> T deserialize(byte[] b) {
if (b == null) return null;
final Object o;
// Stolen from storm
try {
ByteArrayInputStream bis = new ByteArrayInputStream(b);
ObjectInputStream ois = new ObjectInputStream(bis);
Object ret = ois.readObject();
ois.close();
o = ret;
} catch(IOException ioe) {
throw new RuntimeException(ioe);
} catch(ClassNotFoundException e) {
throw new RuntimeException(e);
}
return (T)o;
}
/***
*
* @param bw
* @return
*/
@SuppressWarnings("unchecked")
public static <T> T deserialize(ByteArrayWrapper bw) {
byte[] b = ByteArrayWrapper.unwrap(bw);
return (T)deserialize(b);
}
/***
*
* @param obj
* @return
*/
public static String serializeBase64(Object obj) {
return Base64.encodeBase64String(serialize(obj));
}
/***
*
* @param b
* @return
*/
public static Object deserializeBase64(String b) {
return deserialize(Base64.decodeBase64(b));
}
/***
*
* @param glob
* @return
*/
public static final String createRegexFromGlob(final String glob) {
@RegEx
final StringBuilder re = new StringBuilder(glob.length() * 2);
re.append('^');
for (final char c : glob.toCharArray()) {
switch (c) {
case '?':
re.append('.');
// fall through
case '*':
re.append('*');
break;
default:
re.append(Pattern.quote(Character.toString(c)));
}
}
re.append('$');
return re.toString();
}
/**
* Performs a wildcard matching for the text and pattern
* provided.
*
* Source: http://www.adarshr.com/papers/wildcard
*
* @param text the text to be tested for matches.
*
* @param pattern the pattern to be matched for.
* This can contain the wildcard character '*' (asterisk).
*
* @return <tt>true</tt> if a match is found, <tt>false</tt>
* otherwise.
*/
public static boolean matchGlob(final StringBuilder text, final String pattern) {
return Pattern.matches(createRegexFromGlob(pattern), text);
}
/***
*
* @param text
* @param pattern
*/
public static boolean matchGlob(final String text, final String pattern) {
return matchGlob(new StringBuilder(text), pattern);
}
/***
*
* @return
*/
public static String getHost() {
try {
Enumeration<NetworkInterface> interfaces = NetworkInterface.getNetworkInterfaces();
while (interfaces.hasMoreElements()){
NetworkInterface current = interfaces.nextElement();
if (!current.isUp() || current.isLoopback() || current.isVirtual()) continue;
if (current.getName().contains("lxc")) continue; // <-- do not get LXC interfaces
Enumeration<InetAddress> addresses = current.getInetAddresses();
while (addresses.hasMoreElements()){
InetAddress current_addr = addresses.nextElement();
if (current_addr instanceof Inet4Address) {
return current_addr.getHostAddress();
}
}
}
} catch(SocketException ex){
return null;
}
return null;
}
/***
*
* @param o
* @param max
* @return
*/
public static String truncate(Object o, int max) {
String s = o.toString();
if (s.length() > max) {
return s.substring(0, max) + "...";
}
return s;
}
/***
*
* @param o
* @return
*/
public static String truncate(Object o) {
return truncate(o, 30);
}
/***
*
* @param ms
*/
public static void sleep(long ms) {
try {
Thread.sleep(ms);
} catch(InterruptedException e) {
throw new RuntimeException(e);
}
}
/***
*
* @return
*/
public static boolean isInterrupted() {
return Thread.currentThread().isInterrupted();
}
/***
*
* @param v
* @return
*/
@Deprecated
public static <T> T valueOf(T v) {
return v;
}
/***
*
* @param col
* @param key
* @param val
* @return
*/
public static <K,V> V putIfAbsent(ConcurrentHashMap<K, V> col, K key, V val) {
if (col.containsKey(key) == false) {
col.put(key, val);
}
return col.get(key);
}
public static class CopyFileVisitor extends SimpleFileVisitor<Path> {
private final Path targetPath;
private Path sourcePath = null;
private File lxcRootDir = null;
public CopyFileVisitor(Path targetPath) {
this.targetPath = targetPath;
}
public CopyFileVisitor(Path targetPath, File lxcPrefix) {
this.targetPath = targetPath;
this.lxcRootDir = lxcPrefix;
}
@Override
public FileVisitResult preVisitDirectory(final Path dir, final BasicFileAttributes attrs) throws IOException {
if (sourcePath == null) {
sourcePath = dir;
} else {
Files.createDirectories(targetPath.resolve(sourcePath.relativize(dir)));
}
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult visitFile(final Path file, final BasicFileAttributes attrs) throws IOException {
/***
* Copying from and LXC container is a bit weird, especially when there are symlinks involved. Since we
* run zillabyte prep within the container, all symlinks with absolute paths are "absolute" relative to
* the LXC container's root directory (rootfs). Hence, when we copy them, we need to tack on the lxcRootDir
* path. Relative symlinks are naturally resolved by java.
*/
if(lxcRootDir != null && Files.isSymbolicLink(file)) {
Path linkPath = Files.readSymbolicLink(file);
if(linkPath.isAbsolute()) {
linkPath = (new File(lxcRootDir.toString(), linkPath.toString())).toPath();
Files.copy(linkPath, targetPath.resolve(sourcePath.relativize(file)));
return FileVisitResult.CONTINUE;
}
}
// Standard copy, resolves relative symlinks too
Files.copy(file, targetPath.resolve(sourcePath.relativize(file)));
return FileVisitResult.CONTINUE;
}
}
/***
*
*/
public static <T> T TODO() {
throw new NotImplementedException();
}
public static <T> T TODO(String s) {
throw new NotImplementedException(s);
}
/**
* Attempt to make a call, retrying on timeout
* @param times
* @param callable
* @return
* @throws ExecutionException
* @throws RetryException
*/
public static <T> T retry(int times, Callable<T> callable) throws ExecutionException, RetryException {
// Build the retryer
RetryerBuilder<T> builder = RetryerBuilder.newBuilder();
builder.withStopStrategy(StopStrategies.stopAfterAttempt(times));
builder.withWaitStrategy(WaitStrategies.randomWait(2, TimeUnit.SECONDS, 20, TimeUnit.SECONDS));
Retryer<T> retryer = builder.build();
return retryer.call(callable);
}
public static <T> T retry(Callable<T> callable) throws ExecutionException, RetryException {
return retry(3, callable);
}
/**
* Retry a call multiple times, supressing non runtime exceptions
* @param times
* @param callable
* @return
*/
public static <T> T retryUnchecked(int times, Callable<T> callable) {
try {
return retry(times, callable);
} catch (ExecutionException | RetryException e) {
throw new RuntimeException(e);
}
}
public static <T> T retryUnchecked(Callable<T> callable) {
return retryUnchecked(3, callable);
}
public static boolean exists(String path) {
path = Utils.expandPath(path);
return new File(path).exists();
}
public static void bash(String command) throws InterruptedException, IOException {
Utils.shell("/bin/bash", "-l", "-c", command);
}
public static boolean isCause(Throwable error, Class<? extends Exception> klass) {
return ExceptionUtils.indexOfType(error, klass) != -1;
}
public static String stripLeading(String s, String leading) {
if (s.startsWith(leading)) {
return s.replaceFirst(Pattern.quote(leading), "");
} else {
return s;
}
}
public static String prefixKey(String key) {
String prefix = Universe.instance().env().name();
String f = prefix + key.replaceFirst("$/+", "");
return f.replaceFirst("$/+", ""); // don't start keys with slashes.
}
public static <T> T propagate(Exception e) {
Throwables.propagate(e);
return null;
}
}