/*
* Copyright (c) 2011 LinkedIn, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
* use this file except in compliance with the License. You may obtain a copy of
* the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations under
* the License.
*/
package com.flaptor.indextank.storage;
import static com.flaptor.util.TestInfo.TestType.UNIT;
import java.io.File;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import org.apache.thrift.TException;
import com.flaptor.indextank.rpc.LogRecord;
import com.flaptor.util.TestInfo;
import com.flaptor.util.TestInfo.TestType;
import com.google.common.collect.Lists;
public class LogStorageTest extends BaseLogStorageTest {
/*@TestInfo(testType=UNIT)
public void testThreeSegments() throws IOException, TException {
File parent = new File(dir, "three-segments");
parent.mkdirs();
Segment s1 = Segment.continueLog(root, parent, false);
s1.write(text(addRecord("a", 1), "some text goes here"));
s1.write(text(addRecord("b", 3), "another text goes here"));
Segment s2 = s1.sortAndCloseSegment(100L);
s2.write(text(addRecord("d", 5), "some text goes here 2"));
s2.write(text(addRecord("c", 7), "another text goes here 2"));
Segment s3 = s2.sortAndCloseSegment(200L);
s3.write(text(addRecord("e", 9), "some text goes here 3"));
s3.write(text(addRecord("f", 11), "another text goes here 3"));
s3.release(); // release it but leave it open
List<Segment> segments = Segment.getSegments(root, parent);
assertEquals("Didn't find three segments as expected", 3, segments.size());
Segment ss1 = segments.get(0);
//assertSegmentInfo(ss1, 1L, 2L, false);
List<LogRecord> records1 = Lists.newArrayList(ss1.reader());
assertRecordIds(records1, 1L, 3L);
assertRecordDocIds(records1, "a", "b");
Segment ss2 = segments.get(1);
//assertSegmentInfo(ss2, 3L, 4L, false);
List<LogRecord> records2 = Lists.newArrayList(ss2.reader());
assertRecordIds(records2, 7L, 5L);
assertRecordDocIds(records2, "c", "d");
Segment ss3 = segments.get(2);
//assertSegmentInfo(ss3, 5L, null, false);
List<LogRecord> records3 = Lists.newArrayList(ss3.reader());
assertRecordIds(records3, 9L, 11L);
assertRecordDocIds(records3, "e", "f");
}*/
@TestInfo(testType=UNIT)
public void testLiveLog() throws TException, IOException {
RawLog log = new RawLog(root, 100);
log.write(text(addRecord("a", "idx"), "some text goes here"));
log.write(text(addRecord("a", "idx"), "another text goes here"));
log.write(text(addRecord("c", "idx"), "some text goes here 2"));
log.write(text(addRecord("b", "idx"), "another text goes here 2"));
log.write(text(addRecord("c", "idx"), "some text goes here 3"));
log.write(text(addRecord("b", "idx"), "another text goes here 3"));
log.flush();
List<Segment> segments = log.getLiveSegments();
assertEquals("Didn't find 3 segments as expected", 3, segments.size());
Segment s1 = segments.get(0);
//assertSegmentInfo(s1, 1L, 2L, false);
for (LogRecord r : s1.reader()) {
System.out.println(r);
}
List<LogRecord> records1 = Lists.newArrayList(s1.reader());
//assertRecordIds(records1, 1L, 2L);
assertRecordDocIds(records1, "a", "a");
Segment s2 = segments.get(1);
//assertSegmentInfo(s2, 3L, 4L, false);
List<LogRecord> records2 = Lists.newArrayList(s2.reader());
//assertRecordIds(records2, 3L, 4L);
assertRecordDocIds(records2, "c", "b");
Segment s3 = segments.get(2);
//assertSegmentInfo(s3, 5L, null, false);
List<LogRecord> records3 = Lists.newArrayList(s3.reader());
//assertRecordIds(records3, 5L, 6L);
assertRecordDocIds(records3, "c", "b");
}
@TestInfo(testType=UNIT)
public void testLiveLogReopen() throws TException, IOException {
RawLog log = new RawLog(root, 100);
log.write(text(addRecord("a", "idx"), "some text goes here"));
log.write(text(addRecord("a", "idx"), "another text goes here"));
log.write(text(addRecord("c", "idx"), "some text goes here 2"));
log.write(text(addRecord("b", "idx"), "another text goes here 2"));
log.write(text(addRecord("c", "idx"), "some text goes here 3"));
log.flush();
log.write(text(addRecord("b", "idx"), "another text goes here 3"));
log.flush();
List<Segment> segments = log.getLiveSegments();
assertEquals("Didn't find 3 segments as expected", 3, segments.size());
Segment s1 = segments.get(0);
//assertSegmentInfo(s1, 1L, 2L, false);
List<LogRecord> records1 = Lists.newArrayList(s1.reader());
//assertRecordIds(records1, 1L, 2L);
assertRecordDocIds(records1, "a", "a");
Segment s2 = segments.get(1);
///assertSegmentInfo(s2, 3L, 4L, false);
List<LogRecord> records2 = Lists.newArrayList(s2.reader());
//assertRecordIds(records2, 3L, 4L);
assertRecordDocIds(records2, "c", "b");
Segment s3 = segments.get(2);
//assertSegmentInfo(s3, 5L, null, false);
List<LogRecord> records3 = Lists.newArrayList(s3.reader());
//assertRecordIds(records3, 5L, 6L);
assertRecordDocIds(records3, "c", "b");
}
/*@TestInfo(testType=UNIT)
public void testIndexLogStraight() throws TException, IOException {
IndexLog log = new IndexLog("saraza", manager, 100);
log.write(text(addRecord("a",1), "some text goes here"));
log.write(text(addRecord("b",3), "another text goes here"));
log.write(text(addRecord("d",5), "some text goes here 2"));
log.write(text(addRecord("c",7), "another text goes here 2"));
log.write(text(addRecord("e",9), "some text goes here 3"));
log.write(text(addRecord("f",11), "another text goes here 3"));
log.release();
List<Segment> segments = log.getSegments();
assertEquals("Didn't find 3 segments as expected", 3, segments.size());
Segment s1 = segments.get(0);
assertSegmentInfo(s1, 1L, 3L, true);
List<LogRecord> records1 = Lists.newArrayList(s1.reader());
assertRecordIds(records1, 1L, 3L);
assertRecordDocIds(records1, "a", "b");
Segment s2 = segments.get(1);
assertSegmentInfo(s2, 4L, 7L, true);
List<LogRecord> records2 = Lists.newArrayList(s2.reader());
assertRecordIds(records2, 7L, 5L);
assertRecordDocIds(records2, "c", "d");
Segment s3 = segments.get(2);
assertSegmentInfo(s3, 8L, null, false);
List<LogRecord> records3 = Lists.newArrayList(s3.reader());
assertRecordIds(records3, 9L, 11L);
assertRecordDocIds(records3, "e", "f");
}
@TestInfo(testType=UNIT)
public void testIndexLogSegmentMerge() throws TException, IOException {
IndexLog log = new IndexLog("saraza", manager, 100);
log.write(text(addRecord("a",1), "some text goes here"));
log.write(text(addRecord("a",3), "another text goes here"));
log.write(text(addRecord("b",5), "some text goes here 2"));
log.write(text(addRecord("b",7), "another text goes here 2"));
log.write(text(addRecord("c",9), "some text goes here 3"));
log.write(text(addRecord("c",11), "another text goes here 3"));
log.release();
List<Segment> segments = log.getSegments();
assertEquals("Didn't find 3 segments as expected", 3, segments.size());
Segment s1 = segments.get(0);
assertSegmentInfo(s1, 1L, 3L, true);
List<LogRecord> records1 = Lists.newArrayList(s1.reader());
assertRecordIds(records1, 3L);
assertRecordDocIds(records1, "a");
Segment s2 = segments.get(1);
assertSegmentInfo(s2, 4L, 7L, true);
List<LogRecord> records2 = Lists.newArrayList(s2.reader());
assertRecordIds(records2, 7L);
assertRecordDocIds(records2, "b");
Segment s3 = segments.get(2);
assertSegmentInfo(s3, 8L, null, false);
List<LogRecord> records3 = Lists.newArrayList(s3.reader());
assertRecordIds(records3, 9L, 11L);
assertRecordDocIds(records3, "c", "c");
}
@TestInfo(testType=UNIT)
public void testIndexLogOptimized() throws TException, IOException {
IndexLog log = new IndexLog("saraza", manager, 100);
log.write(text(addRecord("a",1), "some text goes here"));
log.write(text(addRecord("a",3), "another text goes here"));
log.write(text(addRecord("c",5), "some text goes here 2"));
log.write(text(addRecord("b",7), "another text goes here 2"));
log.write(text(addRecord("c",9), "some text goes here 3"));
log.write(text(addRecord("b",11), "another text goes here 3"));
log.release();
log = new IndexLog("saraza", manager, 100);
log.forceClose(12);
log = new IndexLog("saraza", manager, 100);
new LogOptimizer(manager).optimize(log);
Segment segment = log.getLargestOptimizedSegment();
assertSegmentInfo(segment, 1L, 12L, true);
List<LogRecord> records = Lists.newArrayList(segment.reader());
assertRecordIds(records, 3L, 11L, 9L);
assertRecordDocIds(records, "a", "b", "c");
}*/
/*@SuppressWarnings("unchecked")
@TestInfo(testType=UNIT)
public void testLiveLogDealt() throws TException, IOException {
LiveLog log = new LiveLog(root, 1024 * 1024);
Set<String> originalIds[] = new Set[2];
originalIds[0] = Sets.newHashSet();
originalIds[1] = Sets.newHashSet();
for (int i = 0; i < 5000; i++) {
String id = String.valueOf(i);
originalIds[i%2].add(id);
log.write(text(addRecord(id, "idx" + i%2), LONG_TEXT));
}
log.flush();
assertEquals(0, new IndexLog("idx0", root, 256 * 1024).getSegments().size());
assertEquals(0, new IndexLog("idx1", root, 256 * 1024).getSegments().size());
LogDealerServer dealer = new LogDealerServer(root, 1024 * 1024, 256 * 1024);
dealer.runCycle();
for (int i = 0; i < 2; i++) {
String idx = "idx" + i;
IndexLog indexLog = new IndexLog(idx, root, 256 * 1024);
indexLog.forceClose(dealer.getNextTimestamp());
LogOptimizer optimizer = new LogOptimizer(root);
optimizer.checkOptimize(indexLog);
Segment segment = indexLog.getLargestOptimizedSegment();
Segment last = log.getLastSegment();
Set<String> ids = Sets.newHashSet();
for (LogRecord c : segment.reader()) { ids.add(c.get_docid()); }
for (LogRecord c : last.reader()) { if (c.get_index_code().equals(idx)) ids.add(c.get_docid()); }
assertEquals(originalIds[i], ids);
}
}*/
/*@SuppressWarnings("unchecked")
@TestInfo(testType=TestType.INTEGRATION)
public void testIntegration() throws TException, IOException {
LogWriterServer writer = new LogWriterServer(root, 20000, true);
IndexesLogServer reader = new IndexesLogServer(root, 20000, 10000);
LogOptimizer optimizer = new LogOptimizer(reader.dealer, root);
for (int i = 0; i < 300; i++) {
String id = String.valueOf(i);
String code = "code" + i%3;
LogRecord r1 = variable(text(addRecord(id,code), LONG_TEXT), 1, 2.0);
writer.send_batch(batch(r1));
}
Map<String, LogRecord> maps[] = new Map[3];
for (int i = 0; i < 3; i++) {
maps[i] = readFully(reader, "code"+i);
}
for (int i = 0; i < 300; i++) {
String docid = String.valueOf(i);
int idx = i % 3;
assertTrue("Document not found on index " + i, maps[idx].containsKey(docid));
assertEquals("Variable not set properly", 2.0, maps[idx].get(docid).get_variables().get(1));
}
reader.dealer.runCycle();
for (int i = 0; i < 3; i++) {
maps[i] = readFully(reader, "code"+i);
}
for (int i = 0; i < 300; i++) {
String docid = String.valueOf(i);
int idx = i % 3;
assertTrue("Document not found on index " + i, maps[idx].containsKey(docid));
assertEquals("Variable not set properly", 2.0, maps[idx].get(docid).get_variables().get(1));
}
optimizer.optimize("code0");
optimizer.optimize("code1");
for (int i = 0; i < 3; i++) {
maps[i] = readFully(reader, "code"+i);
}
for (int i = 0; i < 300; i++) {
String docid = String.valueOf(i);
int idx = i % 3;
assertTrue("Document not found on index " + i, maps[idx].containsKey(docid));
assertEquals("Variable not set properly", 2.0, maps[idx].get(docid).get_variables().get(1));
}
optimizer.optimize("code1");
optimizer.optimize("code2");
for (int i = 0; i < 3; i++) {
maps[i] = readFully(reader, "code"+i);
}
for (int i = 0; i < 300; i++) {
String docid = String.valueOf(i);
int idx = i % 3;
assertTrue("Document not found on index " + i, maps[idx].containsKey(docid));
assertEquals("Variable not set properly", 2.0, maps[idx].get(docid).get_variables().get(1));
}
}*/
}