Skip to content

Commit

Permalink
TEZ-3609. Improve ATSv15 performance for DAG entities read calls.
Browse files Browse the repository at this point in the history 
Contributed by Harish Jaiprakash.
  • Loading branch information
@sidseth
sidseth committed Feb 8, 2017
1 parent 63ae97d commit 11815a7
Show file tree
Hide file tree
Showing 3 changed files with 23 additions and 21 deletions.
2 changes: 2 additions & 0 deletions CHANGES.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ INCOMPATIBLE CHANGES

ALL CHANGES:

TEZ-3609. Improve ATSv15 performance for DAG entities read calls.
TEZ-3244. Allow overlap of input and output memory when they are not concurrent
TEZ-3581. Add different logger to enable suppressing logs for specific lines.
TEZ-3601. Add another HistoryLogLevel to suppress TaskAttempts at specific levels
Expand Down Expand Up @@ -198,6 +199,7 @@ INCOMPATIBLE CHANGES

ALL CHANGES:

TEZ-3609. Improve ATSv15 performance for DAG entities read calls.
TEZ-3244. Allow overlap of input and output memory when they are not concurrent
TEZ-3601. Add another HistoryLogLevel to suppress TaskAttempts at specific levels
TEZ-3462. Task attempt failure during container shutdown loses useful container diagnostics
Expand Down
1 change: 0 additions & 1 deletion ...-plugin/src/main/java/org/apache/tez/dag/history/logging/ats/TimelineCachePluginImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,6 @@ public TimelineCachePluginImpl() {
private Set<TimelineEntityGroupId> createTimelineEntityGroupIds(TezDAGID dagId) {
ApplicationId appId = dagId.getApplicationId();
HashSet<TimelineEntityGroupId> groupIds = Sets.newHashSet(
TimelineEntityGroupId.newInstance(appId, appId.toString()),
TimelineEntityGroupId.newInstance(appId, dagId.toString()));
for (int numGroupsPerDag : allNumGroupsPerDag) {
groupIds.add(TimelineEntityGroupId.newInstance(appId, dagId.getGroupId(numGroupsPerDag)));
Expand Down
41 changes: 21 additions & 20 deletions ...gin/src/test/java/org/apache/tez/dag/history/logging/ats/TestTimelineCachePluginImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -124,12 +124,12 @@ public void testGetTimelineEntityGroupIdByPrimaryFilter() {
Assert.assertNull(groupIds);
continue;
}
Assert.assertEquals(3, groupIds.size());
Assert.assertEquals(2, groupIds.size());
Iterator<TimelineEntityGroupId> iter = groupIds.iterator();
while (iter.hasNext()) {
TimelineEntityGroupId groupId = iter.next();
Assert.assertEquals(appId1, groupId.getApplicationId());
Assert.assertTrue(getGroupIds(dagID1, appId1, 100).contains(groupId.getTimelineEntityGroupId()));
Assert.assertTrue(getGroupIds(dagID1, 100).contains(groupId.getTimelineEntityGroupId()));
}
}
}
Expand All @@ -143,12 +143,12 @@ public void testGetTimelineEntityGroupIdByIdDefaultConfig() {
Assert.assertNull(groupIds);
continue;
}
Assert.assertEquals(2, groupIds.size());
Assert.assertEquals(1, groupIds.size());
Iterator<TimelineEntityGroupId> iter = groupIds.iterator();
while (iter.hasNext()) {
TimelineEntityGroupId groupId = iter.next();
Assert.assertEquals(appId1, groupId.getApplicationId());
Assert.assertTrue(getGroupIds(dagID1, appId1).contains(groupId.getTimelineEntityGroupId()));
Assert.assertTrue(getGroupIds(dagID1).contains(groupId.getTimelineEntityGroupId()));
}
}
}
Expand All @@ -162,12 +162,12 @@ public void testGetTimelineEntityGroupIdByIdNoGroupingConf() {
Assert.assertNull(groupIds);
continue;
}
Assert.assertEquals(2, groupIds.size());
Assert.assertEquals(1, groupIds.size());
Iterator<TimelineEntityGroupId> iter = groupIds.iterator();
while (iter.hasNext()) {
TimelineEntityGroupId groupId = iter.next();
Assert.assertEquals(appId1, groupId.getApplicationId());
Assert.assertTrue(getGroupIds(dagID1, appId1).contains(groupId.getTimelineEntityGroupId()));
Assert.assertTrue(getGroupIds(dagID1).contains(groupId.getTimelineEntityGroupId()));
}
}
}
Expand All @@ -181,12 +181,12 @@ public void testGetTimelineEntityGroupIdById() {
Assert.assertNull(groupIds);
continue;
}
Assert.assertEquals(3, groupIds.size());
Assert.assertEquals(2, groupIds.size());
Iterator<TimelineEntityGroupId> iter = groupIds.iterator();
while (iter.hasNext()) {
TimelineEntityGroupId groupId = iter.next();
Assert.assertEquals(appId1, groupId.getApplicationId());
Assert.assertTrue(getGroupIds(dagID1, appId1, 100).contains(groupId.getTimelineEntityGroupId()));
Assert.assertTrue(getGroupIds(dagID1, 100).contains(groupId.getTimelineEntityGroupId()));
}
}
}
Expand All @@ -200,12 +200,12 @@ public void testGetTimelineEntityGroupIdByIdWithOldGroupIdsSingle() {
Assert.assertNull(groupIds);
continue;
}
Assert.assertEquals(4, groupIds.size());
Assert.assertEquals(3, groupIds.size());
Iterator<TimelineEntityGroupId> iter = groupIds.iterator();
while (iter.hasNext()) {
TimelineEntityGroupId groupId = iter.next();
Assert.assertEquals(appId2, groupId.getApplicationId());
Assert.assertTrue(getGroupIds(dagID2, appId2, 100, 50).contains(groupId.getTimelineEntityGroupId()));
Assert.assertTrue(getGroupIds(dagID2, 100, 50).contains(groupId.getTimelineEntityGroupId()));
}
}
}
Expand All @@ -219,12 +219,13 @@ public void testGetTimelineEntityGroupIdByIdWithOldGroupIdsMultiple() {
Assert.assertNull(groupIds);
continue;
}
Assert.assertEquals(5, groupIds.size());
Assert.assertEquals(4, groupIds.size());
Iterator<TimelineEntityGroupId> iter = groupIds.iterator();
while (iter.hasNext()) {
TimelineEntityGroupId groupId = iter.next();
Assert.assertEquals(appId2, groupId.getApplicationId());
Assert.assertTrue(getGroupIds(dagID2, appId2, 100, 25, 50).contains(groupId.getTimelineEntityGroupId()));
Assert.assertTrue(
getGroupIds(dagID2, 100, 25, 50).contains(groupId.getTimelineEntityGroupId()));
}
}
}
Expand All @@ -238,12 +239,12 @@ public void testGetTimelineEntityGroupIdByIdWithOldGroupIdsEmpty() {
Assert.assertNull(groupIds);
continue;
}
Assert.assertEquals(3, groupIds.size());
Assert.assertEquals(2, groupIds.size());
Iterator<TimelineEntityGroupId> iter = groupIds.iterator();
while (iter.hasNext()) {
TimelineEntityGroupId groupId = iter.next();
Assert.assertEquals(appId2, groupId.getApplicationId());
Assert.assertTrue(getGroupIds(dagID2, appId2, 100).contains(groupId.getTimelineEntityGroupId()));
Assert.assertTrue(getGroupIds(dagID2, 100).contains(groupId.getTimelineEntityGroupId()));
}
}
}
Expand All @@ -261,21 +262,21 @@ public void testGetTimelineEntityGroupIdByIds() {
Assert.assertNull(groupIds);
continue;
}
Assert.assertEquals(6, groupIds.size());
Assert.assertEquals(4, groupIds.size());
int found = 0;
Iterator<TimelineEntityGroupId> iter = groupIds.iterator();
while (iter.hasNext()) {
TimelineEntityGroupId groupId = iter.next();
if (groupId.getApplicationId().equals(appId1)) {
String entityGroupId = groupId.getTimelineEntityGroupId();
if (getGroupIds(dagID1, appId1, 100).contains(entityGroupId)) {
if (getGroupIds(dagID1, 100).contains(entityGroupId)) {
++found;
} else {
Assert.fail("Unexpected group id: " + entityGroupId);
}
} else if (groupId.getApplicationId().equals(appId2)) {
String entityGroupId = groupId.getTimelineEntityGroupId();
if (getGroupIds(dagID2, appId2, 100).contains(entityGroupId)) {
if (getGroupIds(dagID2, 100).contains(entityGroupId)) {
++found;
} else {
Assert.fail("Unexpected group id: " + entityGroupId);
Expand All @@ -284,7 +285,7 @@ public void testGetTimelineEntityGroupIdByIds() {
Assert.fail("Unexpected appId: " + groupId.getApplicationId());
}
}
Assert.assertEquals("All groupIds not returned", 6, found);
Assert.assertEquals("All groupIds not returned", 4, found);
}
}

Expand Down Expand Up @@ -374,8 +375,8 @@ public void testContainerIdConversion() {
Assert.assertEquals("All groupIds not returned", 1, found);
}

private Set<String> getGroupIds(TezDAGID dagId, ApplicationId appId, int ... allNumDagsPerGroup) {
HashSet<String> groupIds = Sets.newHashSet(dagId.toString(), appId.toString());
private Set<String> getGroupIds(TezDAGID dagId, int ... allNumDagsPerGroup) {
HashSet<String> groupIds = Sets.newHashSet(dagId.toString());
for (int numDagsPerGroup : allNumDagsPerGroup) {
groupIds.add(dagId.getGroupId(numDagsPerGroup));
}
Expand Down

0 comments on commit 11815a7

Please sign in to comment.