/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.accumulo.test.replication; import static org.easymock.EasyMock.createMock; import static org.easymock.EasyMock.expectLastCall; import static org.easymock.EasyMock.replay; import static org.easymock.EasyMock.verify; import java.util.HashMap; import java.util.Map; import org.apache.accumulo.core.client.BatchWriter; import org.apache.accumulo.core.client.Connector; import org.apache.accumulo.core.data.Mutation; import org.apache.accumulo.core.protobuf.ProtobufUtil; import org.apache.accumulo.core.replication.ReplicationSchema.OrderSection; import org.apache.accumulo.core.replication.ReplicationSchema.WorkSection; import org.apache.accumulo.core.replication.ReplicationTable; import org.apache.accumulo.core.replication.ReplicationTarget; import org.apache.accumulo.core.security.TablePermission; import org.apache.accumulo.master.replication.SequentialWorkAssigner; import org.apache.accumulo.server.replication.DistributedWorkQueueWorkAssignerHelper; import org.apache.accumulo.server.replication.proto.Replication.Status; import org.apache.accumulo.server.zookeeper.DistributedWorkQueue; import org.apache.accumulo.server.zookeeper.ZooCache; import org.apache.accumulo.test.functional.ConfigurableMacBase; import org.apache.hadoop.io.Text; import org.junit.Assert; import org.junit.Before; import org.junit.Test; public class SequentialWorkAssignerIT extends ConfigurableMacBase { private Connector conn; private MockSequentialWorkAssigner assigner; private static class MockSequentialWorkAssigner extends SequentialWorkAssigner { public MockSequentialWorkAssigner(Connector conn) { super(null, conn); } @Override public void setConnector(Connector conn) { super.setConnector(conn); } @Override public void setQueuedWork(Map<String,Map<String,String>> queuedWork) { super.setQueuedWork(queuedWork); } @Override public void setWorkQueue(DistributedWorkQueue workQueue) { super.setWorkQueue(workQueue); } @Override public void setMaxQueueSize(int maxQueueSize) { super.setMaxQueueSize(maxQueueSize); } @Override public void createWork() { super.createWork(); } @Override public void setZooCache(ZooCache zooCache) { super.setZooCache(zooCache); } @Override public void cleanupFinishedWork() { super.cleanupFinishedWork(); } } @Before public void init() throws Exception { conn = getConnector(); assigner = new MockSequentialWorkAssigner(conn); // grant ourselves write to the replication table conn.securityOperations().grantTablePermission(conn.whoami(), ReplicationTable.NAME, TablePermission.READ); conn.securityOperations().grantTablePermission(conn.whoami(), ReplicationTable.NAME, TablePermission.WRITE); ReplicationTable.setOnline(conn); } @Test public void createWorkForFilesInCorrectOrder() throws Exception { ReplicationTarget target = new ReplicationTarget("cluster1", "table1", "1"); Text serializedTarget = target.toText(); // Create two mutations, both of which need replication work done BatchWriter bw = ReplicationTable.getBatchWriter(conn); // We want the name of file2 to sort before file1 String filename1 = "z_file1", filename2 = "a_file1"; String file1 = "/accumulo/wal/tserver+port/" + filename1, file2 = "/accumulo/wal/tserver+port/" + filename2; // File1 was closed before file2, however Status stat1 = Status.newBuilder().setBegin(0).setEnd(100).setClosed(true).setInfiniteEnd(false).setCreatedTime(250).build(); Status stat2 = Status.newBuilder().setBegin(0).setEnd(100).setClosed(true).setInfiniteEnd(false).setCreatedTime(500).build(); Mutation m = new Mutation(file1); WorkSection.add(m, serializedTarget, ProtobufUtil.toValue(stat1)); bw.addMutation(m); m = new Mutation(file2); WorkSection.add(m, serializedTarget, ProtobufUtil.toValue(stat2)); bw.addMutation(m); m = OrderSection.createMutation(file1, stat1.getCreatedTime()); OrderSection.add(m, target.getSourceTableId(), ProtobufUtil.toValue(stat1)); bw.addMutation(m); m = OrderSection.createMutation(file2, stat2.getCreatedTime()); OrderSection.add(m, target.getSourceTableId(), ProtobufUtil.toValue(stat2)); bw.addMutation(m); bw.close(); DistributedWorkQueue workQueue = createMock(DistributedWorkQueue.class); Map<String,Map<String,String>> queuedWork = new HashMap<>(); assigner.setQueuedWork(queuedWork); assigner.setWorkQueue(workQueue); assigner.setMaxQueueSize(Integer.MAX_VALUE); // Make sure we expect the invocations in the correct order (accumulo is sorted) workQueue.addWork(DistributedWorkQueueWorkAssignerHelper.getQueueKey(filename1, target), file1); expectLastCall().once(); // file2 is *not* queued because file1 must be replicated first replay(workQueue); assigner.createWork(); verify(workQueue); Assert.assertEquals(1, queuedWork.size()); Assert.assertTrue(queuedWork.containsKey("cluster1")); Map<String,String> cluster1Work = queuedWork.get("cluster1"); Assert.assertEquals(1, cluster1Work.size()); Assert.assertTrue(cluster1Work.containsKey(target.getSourceTableId())); Assert.assertEquals(DistributedWorkQueueWorkAssignerHelper.getQueueKey(filename1, target), cluster1Work.get(target.getSourceTableId())); } @Test public void workAcrossTablesHappensConcurrently() throws Exception { ReplicationTarget target1 = new ReplicationTarget("cluster1", "table1", "1"); Text serializedTarget1 = target1.toText(); ReplicationTarget target2 = new ReplicationTarget("cluster1", "table2", "2"); Text serializedTarget2 = target2.toText(); // Create two mutations, both of which need replication work done BatchWriter bw = ReplicationTable.getBatchWriter(conn); // We want the name of file2 to sort before file1 String filename1 = "z_file1", filename2 = "a_file1"; String file1 = "/accumulo/wal/tserver+port/" + filename1, file2 = "/accumulo/wal/tserver+port/" + filename2; // File1 was closed before file2, however Status stat1 = Status.newBuilder().setBegin(0).setEnd(100).setClosed(true).setInfiniteEnd(false).setCreatedTime(250).build(); Status stat2 = Status.newBuilder().setBegin(0).setEnd(100).setClosed(true).setInfiniteEnd(false).setCreatedTime(500).build(); Mutation m = new Mutation(file1); WorkSection.add(m, serializedTarget1, ProtobufUtil.toValue(stat1)); bw.addMutation(m); m = new Mutation(file2); WorkSection.add(m, serializedTarget2, ProtobufUtil.toValue(stat2)); bw.addMutation(m); m = OrderSection.createMutation(file1, stat1.getCreatedTime()); OrderSection.add(m, target1.getSourceTableId(), ProtobufUtil.toValue(stat1)); bw.addMutation(m); m = OrderSection.createMutation(file2, stat2.getCreatedTime()); OrderSection.add(m, target2.getSourceTableId(), ProtobufUtil.toValue(stat2)); bw.addMutation(m); bw.close(); DistributedWorkQueue workQueue = createMock(DistributedWorkQueue.class); Map<String,Map<String,String>> queuedWork = new HashMap<>(); assigner.setQueuedWork(queuedWork); assigner.setWorkQueue(workQueue); assigner.setMaxQueueSize(Integer.MAX_VALUE); // Make sure we expect the invocations in the correct order (accumulo is sorted) workQueue.addWork(DistributedWorkQueueWorkAssignerHelper.getQueueKey(filename1, target1), file1); expectLastCall().once(); workQueue.addWork(DistributedWorkQueueWorkAssignerHelper.getQueueKey(filename2, target2), file2); expectLastCall().once(); // file2 is *not* queued because file1 must be replicated first replay(workQueue); assigner.createWork(); verify(workQueue); Assert.assertEquals(1, queuedWork.size()); Assert.assertTrue(queuedWork.containsKey("cluster1")); Map<String,String> cluster1Work = queuedWork.get("cluster1"); Assert.assertEquals(2, cluster1Work.size()); Assert.assertTrue(cluster1Work.containsKey(target1.getSourceTableId())); Assert.assertEquals(DistributedWorkQueueWorkAssignerHelper.getQueueKey(filename1, target1), cluster1Work.get(target1.getSourceTableId())); Assert.assertTrue(cluster1Work.containsKey(target2.getSourceTableId())); Assert.assertEquals(DistributedWorkQueueWorkAssignerHelper.getQueueKey(filename2, target2), cluster1Work.get(target2.getSourceTableId())); } @Test public void workAcrossPeersHappensConcurrently() throws Exception { ReplicationTarget target1 = new ReplicationTarget("cluster1", "table1", "1"); Text serializedTarget1 = target1.toText(); ReplicationTarget target2 = new ReplicationTarget("cluster2", "table1", "1"); Text serializedTarget2 = target2.toText(); // Create two mutations, both of which need replication work done BatchWriter bw = ReplicationTable.getBatchWriter(conn); // We want the name of file2 to sort before file1 String filename1 = "z_file1", filename2 = "a_file1"; String file1 = "/accumulo/wal/tserver+port/" + filename1, file2 = "/accumulo/wal/tserver+port/" + filename2; // File1 was closed before file2, however Status stat1 = Status.newBuilder().setBegin(0).setEnd(100).setClosed(true).setInfiniteEnd(false).setCreatedTime(250).build(); Status stat2 = Status.newBuilder().setBegin(0).setEnd(100).setClosed(true).setInfiniteEnd(false).setCreatedTime(500).build(); Mutation m = new Mutation(file1); WorkSection.add(m, serializedTarget1, ProtobufUtil.toValue(stat1)); bw.addMutation(m); m = new Mutation(file2); WorkSection.add(m, serializedTarget2, ProtobufUtil.toValue(stat2)); bw.addMutation(m); m = OrderSection.createMutation(file1, stat1.getCreatedTime()); OrderSection.add(m, target1.getSourceTableId(), ProtobufUtil.toValue(stat1)); bw.addMutation(m); m = OrderSection.createMutation(file2, stat2.getCreatedTime()); OrderSection.add(m, target2.getSourceTableId(), ProtobufUtil.toValue(stat2)); bw.addMutation(m); bw.close(); DistributedWorkQueue workQueue = createMock(DistributedWorkQueue.class); Map<String,Map<String,String>> queuedWork = new HashMap<>(); assigner.setQueuedWork(queuedWork); assigner.setWorkQueue(workQueue); assigner.setMaxQueueSize(Integer.MAX_VALUE); // Make sure we expect the invocations in the correct order (accumulo is sorted) workQueue.addWork(DistributedWorkQueueWorkAssignerHelper.getQueueKey(filename1, target1), file1); expectLastCall().once(); workQueue.addWork(DistributedWorkQueueWorkAssignerHelper.getQueueKey(filename2, target2), file2); expectLastCall().once(); // file2 is *not* queued because file1 must be replicated first replay(workQueue); assigner.createWork(); verify(workQueue); Assert.assertEquals(2, queuedWork.size()); Assert.assertTrue(queuedWork.containsKey("cluster1")); Map<String,String> cluster1Work = queuedWork.get("cluster1"); Assert.assertEquals(1, cluster1Work.size()); Assert.assertTrue(cluster1Work.containsKey(target1.getSourceTableId())); Assert.assertEquals(DistributedWorkQueueWorkAssignerHelper.getQueueKey(filename1, target1), cluster1Work.get(target1.getSourceTableId())); Map<String,String> cluster2Work = queuedWork.get("cluster2"); Assert.assertEquals(1, cluster2Work.size()); Assert.assertTrue(cluster2Work.containsKey(target2.getSourceTableId())); Assert.assertEquals(DistributedWorkQueueWorkAssignerHelper.getQueueKey(filename2, target2), cluster2Work.get(target2.getSourceTableId())); } @Test public void reprocessingOfCompletedWorkRemovesWork() throws Exception { ReplicationTarget target = new ReplicationTarget("cluster1", "table1", "1"); Text serializedTarget = target.toText(); // Create two mutations, both of which need replication work done BatchWriter bw = ReplicationTable.getBatchWriter(conn); // We want the name of file2 to sort before file1 String filename1 = "z_file1", filename2 = "a_file1"; String file1 = "/accumulo/wal/tserver+port/" + filename1, file2 = "/accumulo/wal/tserver+port/" + filename2; // File1 was closed before file2, however Status stat1 = Status.newBuilder().setBegin(100).setEnd(100).setClosed(true).setInfiniteEnd(false).setCreatedTime(250).build(); Status stat2 = Status.newBuilder().setBegin(0).setEnd(100).setClosed(true).setInfiniteEnd(false).setCreatedTime(500).build(); Mutation m = new Mutation(file1); WorkSection.add(m, serializedTarget, ProtobufUtil.toValue(stat1)); bw.addMutation(m); m = new Mutation(file2); WorkSection.add(m, serializedTarget, ProtobufUtil.toValue(stat2)); bw.addMutation(m); m = OrderSection.createMutation(file1, stat1.getCreatedTime()); OrderSection.add(m, target.getSourceTableId(), ProtobufUtil.toValue(stat1)); bw.addMutation(m); m = OrderSection.createMutation(file2, stat2.getCreatedTime()); OrderSection.add(m, target.getSourceTableId(), ProtobufUtil.toValue(stat2)); bw.addMutation(m); bw.close(); DistributedWorkQueue workQueue = createMock(DistributedWorkQueue.class); // Treat filename1 as we have already submitted it for replication Map<String,Map<String,String>> queuedWork = new HashMap<>(); Map<String,String> queuedWorkForCluster = new HashMap<>(); queuedWorkForCluster.put(target.getSourceTableId(), DistributedWorkQueueWorkAssignerHelper.getQueueKey(filename1, target)); queuedWork.put("cluster1", queuedWorkForCluster); assigner.setQueuedWork(queuedWork); assigner.setWorkQueue(workQueue); assigner.setMaxQueueSize(Integer.MAX_VALUE); // Make sure we expect the invocations in the correct order (accumulo is sorted) workQueue.addWork(DistributedWorkQueueWorkAssignerHelper.getQueueKey(filename2, target), file2); expectLastCall().once(); // file2 is queued because we remove file1 because it's fully replicated replay(workQueue); assigner.createWork(); verify(workQueue); Assert.assertEquals(1, queuedWork.size()); Assert.assertTrue(queuedWork.containsKey("cluster1")); Map<String,String> cluster1Work = queuedWork.get("cluster1"); Assert.assertEquals(1, cluster1Work.size()); Assert.assertTrue(cluster1Work.containsKey(target.getSourceTableId())); Assert.assertEquals(DistributedWorkQueueWorkAssignerHelper.getQueueKey(filename2, target), cluster1Work.get(target.getSourceTableId())); } }