Skip to content

Commit

Permalink
HBASE-25891 Remove dependence on storing WAL filenames for backup (ap…
Browse files Browse the repository at this point in the history 
…ache#3359)

Signed-off-by: stack <[email protected]>
Signed-off-by: Duo Zhang <[email protected]>
  • Loading branch information
@rda3mon
rda3mon authored and Duo Zhang committed Sep 13, 2021
1 parent fc5a8fd commit 04c3888
Show file tree
Hide file tree
Showing 14 changed files with 187 additions and 537 deletions.
51 changes: 39 additions & 12 deletions hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupInfo.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,15 +28,13 @@
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;

import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.backup.util.BackupUtils;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.shaded.protobuf.generated.BackupProtos;
import org.apache.hadoop.hbase.shaded.protobuf.generated.BackupProtos.BackupInfo.Builder;
Expand Down Expand Up @@ -136,7 +134,13 @@ public enum BackupPhase {
* New region server log timestamps for table set after distributed log roll key - table name,
* value - map of RegionServer hostname -> last log rolled timestamp
*/
private HashMap<TableName, HashMap<String, Long>> tableSetTimestampMap;
private Map<TableName, Map<String, Long>> tableSetTimestampMap;

/**
* Previous Region server log timestamps for table set after distributed log roll key -
* table name, value - map of RegionServer hostname -> last log rolled timestamp
*/
private Map<TableName, Map<String, Long>> incrTimestampMap;

/**
* Backup progress in %% (0-100)
Expand Down Expand Up @@ -190,12 +194,12 @@ public void setBackupTableInfoMap(Map<TableName, BackupTableInfo> backupTableInf
this.backupTableInfoMap = backupTableInfoMap;
}

public HashMap<TableName, HashMap<String, Long>> getTableSetTimestampMap() {
public Map<TableName, Map<String, Long>> getTableSetTimestampMap() {
return tableSetTimestampMap;
}

public void setTableSetTimestampMap(HashMap<TableName,
HashMap<String, Long>> tableSetTimestampMap) {
public void setTableSetTimestampMap(Map<TableName,
Map<String, Long>> tableSetTimestampMap) {
this.tableSetTimestampMap = tableSetTimestampMap;
}

Expand Down Expand Up @@ -351,19 +355,19 @@ public void setIncrBackupFileList(List<String> incrBackupFileList) {

/**
* Set the new region server log timestamps after distributed log roll
* @param newTableSetTimestampMap table timestamp map
* @param prevTableSetTimestampMap table timestamp map
*/
public void setIncrTimestampMap(HashMap<TableName,
HashMap<String, Long>> newTableSetTimestampMap) {
this.tableSetTimestampMap = newTableSetTimestampMap;
public void setIncrTimestampMap(Map<TableName,
Map<String, Long>> prevTableSetTimestampMap) {
this.incrTimestampMap = prevTableSetTimestampMap;
}

/**
* Get new region server log timestamps after distributed log roll
* @return new region server log timestamps
*/
public HashMap<TableName, HashMap<String, Long>> getIncrTimestampMap() {
return this.tableSetTimestampMap;
public Map<TableName, Map<String, Long>> getIncrTimestampMap() {
return this.incrTimestampMap;
}

public TableName getTableBySnapshot(String snapshotName) {
Expand All @@ -379,6 +383,7 @@ public BackupProtos.BackupInfo toProtosBackupInfo() {
BackupProtos.BackupInfo.Builder builder = BackupProtos.BackupInfo.newBuilder();
builder.setBackupId(getBackupId());
setBackupTableInfoMap(builder);
setTableSetTimestampMap(builder);
builder.setCompleteTs(getCompleteTs());
if (getFailedMsg() != null) {
builder.setFailedMessage(getFailedMsg());
Expand Down Expand Up @@ -446,6 +451,16 @@ private void setBackupTableInfoMap(Builder builder) {
}
}

private void setTableSetTimestampMap(Builder builder) {
if (this.getTableSetTimestampMap() != null) {
for (Entry<TableName, Map<String, Long>> entry : this.getTableSetTimestampMap().entrySet()) {
builder.putTableSetTimestamp(entry.getKey().getNameAsString(),
BackupProtos.BackupInfo.RSTimestampMap.newBuilder().putAllRsTimestamp(entry.getValue())
.build());
}
}
}

public static BackupInfo fromByteArray(byte[] data) throws IOException {
return fromProto(BackupProtos.BackupInfo.parseFrom(data));
}
Expand All @@ -458,6 +473,7 @@ public static BackupInfo fromProto(BackupProtos.BackupInfo proto) {
BackupInfo context = new BackupInfo();
context.setBackupId(proto.getBackupId());
context.setBackupTableInfoMap(toMap(proto.getBackupTableInfoList()));
context.setTableSetTimestampMap(getTableSetTimestampMap(proto.getTableSetTimestampMap()));
context.setCompleteTs(proto.getCompleteTs());
if (proto.hasFailedMessage()) {
context.setFailedMsg(proto.getFailedMessage());
Expand Down Expand Up @@ -491,6 +507,17 @@ private static Map<TableName, BackupTableInfo> toMap(List<BackupProtos.BackupTab
return map;
}

private static Map<TableName, Map<String, Long>> getTableSetTimestampMap(
Map<String, BackupProtos.BackupInfo.RSTimestampMap> map) {
Map<TableName, Map<String, Long>> tableSetTimestampMap = new HashMap<>();
for (Entry<String, BackupProtos.BackupInfo.RSTimestampMap> entry : map.entrySet()) {
tableSetTimestampMap
.put(TableName.valueOf(entry.getKey()), entry.getValue().getRsTimestampMap());
}

return tableSetTimestampMap;
}

public String getShortDescription() {
StringBuilder sb = new StringBuilder();
sb.append("{");
Expand Down
23 changes: 2 additions & 21 deletions hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
Expand Down Expand Up @@ -482,7 +481,7 @@ public ArrayList<BackupInfo> getBackupHistory(boolean completed) throws IOExcept
* @throws IOException exception
*/
public void writeRegionServerLogTimestamp(Set<TableName> tables,
HashMap<String, Long> newTimestamps) throws IOException {
Map<String, Long> newTimestamps) throws IOException {
systemTable.writeRegionServerLogTimestamp(tables, newTimestamps, backupInfo.getBackupRootDir());
}

Expand All @@ -493,7 +492,7 @@ public void writeRegionServerLogTimestamp(Set<TableName> tables,
* RegionServer,PreviousTimeStamp
* @throws IOException exception
*/
public HashMap<TableName, HashMap<String, Long>> readLogTimestampMap() throws IOException {
public Map<TableName, Map<String, Long>> readLogTimestampMap() throws IOException {
return systemTable.readLogTimestampMap(backupInfo.getBackupRootDir());
}

Expand All @@ -515,24 +514,6 @@ public void addIncrementalBackupTableSet(Set<TableName> tables) throws IOExcepti
systemTable.addIncrementalBackupTableSet(tables, backupInfo.getBackupRootDir());
}

/**
* Saves list of WAL files after incremental backup operation. These files will be stored until
* TTL expiration and are used by Backup Log Cleaner plug-in to determine which WAL files can be
* safely purged.
*/
public void recordWALFiles(List<String> files) throws IOException {
systemTable.addWALFiles(files, backupInfo.getBackupId(), backupInfo.getBackupRootDir());
}

/**
* Get WAL files iterator.
* @return WAL files iterator from backup system table
* @throws IOException if getting the WAL files iterator fails
*/
public Iterator<BackupSystemTable.WALItem> getWALFilesFromBackupSystem() throws IOException {
return systemTable.getWALFilesIterator(backupInfo.getBackupRootDir());
}

public Connection getConnection() {
return conn;
}
Expand Down
20 changes: 10 additions & 10 deletions hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupManifest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,7 @@ BackupImage build() {
private long startTs;
private long completeTs;
private ArrayList<BackupImage> ancestors;
private HashMap<TableName, HashMap<String, Long>> incrTimeRanges;
private Map<TableName, Map<String, Long>> incrTimeRanges;

static Builder newBuilder() {
return new Builder();
Expand Down Expand Up @@ -187,19 +187,19 @@ BackupProtos.BackupImage toProto() {
return builder.build();
}

private static HashMap<TableName, HashMap<String, Long>> loadIncrementalTimestampMap(
private static Map<TableName, Map<String, Long>> loadIncrementalTimestampMap(
BackupProtos.BackupImage proto) {
List<BackupProtos.TableServerTimestamp> list = proto.getTstMapList();

HashMap<TableName, HashMap<String, Long>> incrTimeRanges = new HashMap<>();
Map<TableName, Map<String, Long>> incrTimeRanges = new HashMap<>();

if (list == null || list.size() == 0) {
return incrTimeRanges;
}

for (BackupProtos.TableServerTimestamp tst : list) {
TableName tn = ProtobufUtil.toTableName(tst.getTableName());
HashMap<String, Long> map = incrTimeRanges.get(tn);
Map<String, Long> map = incrTimeRanges.get(tn);
if (map == null) {
map = new HashMap<>();
incrTimeRanges.put(tn, map);
Expand All @@ -217,9 +217,9 @@ private void setIncrementalTimestampMap(BackupProtos.BackupImage.Builder builder
if (this.incrTimeRanges == null) {
return;
}
for (Entry<TableName, HashMap<String, Long>> entry : this.incrTimeRanges.entrySet()) {
for (Entry<TableName, Map<String, Long>> entry : this.incrTimeRanges.entrySet()) {
TableName key = entry.getKey();
HashMap<String, Long> value = entry.getValue();
Map<String, Long> value = entry.getValue();
BackupProtos.TableServerTimestamp.Builder tstBuilder =
BackupProtos.TableServerTimestamp.newBuilder();
tstBuilder.setTableName(ProtobufUtil.toProtoTableName(key));
Expand Down Expand Up @@ -359,11 +359,11 @@ public int hashCode() {
return hash;
}

public HashMap<TableName, HashMap<String, Long>> getIncrTimeRanges() {
public Map<TableName, Map<String, Long>> getIncrTimeRanges() {
return incrTimeRanges;
}

private void setIncrTimeRanges(HashMap<TableName, HashMap<String, Long>> incrTimeRanges) {
private void setIncrTimeRanges(Map<TableName, Map<String, Long>> incrTimeRanges) {
this.incrTimeRanges = incrTimeRanges;
}
}
Expand Down Expand Up @@ -512,11 +512,11 @@ public void addDependentImage(BackupImage image) {
* Set the incremental timestamp map directly.
* @param incrTimestampMap timestamp map
*/
public void setIncrTimestampMap(HashMap<TableName, HashMap<String, Long>> incrTimestampMap) {
public void setIncrTimestampMap(Map<TableName, Map<String, Long>> incrTimestampMap) {
this.backupImage.setIncrTimeRanges(incrTimestampMap);
}

public Map<TableName, HashMap<String, Long>> getIncrTimestampMap() {
public Map<TableName, Map<String, Long>> getIncrTimestampMap() {
return backupImage.getIncrTimeRanges();
}

Expand Down
Loading

0 comments on commit 04c3888

Please sign in to comment.