br: add encryption config to streaming backup (pingcap#1255)

* rebase master

* minor

* fix comment

* add one more field and address comments

proto/brpb.proto

@@ -277,6 +277,14 @@ enum CompressionType {
     ZSTD = 3;
 }
 
+message StreamBackupTaskSecurityConfig {
+    oneof encryption {
+        // not recommended in production. configure KMS based or locally managed master key instead in TiKV StreamBackupConfig
+        CipherInfo plaintext_data_key = 1;
+        MasterKeyConfig master_key_config = 2;
+    }
+}
+
 // BackupMpde represents the mode of this whole backup request to the cluster.
 // and we need to store it in `backupmeta`.
 enum BackupMode {
@@ -295,6 +303,11 @@ message CipherInfo {
     bytes cipher_key = 2;
 }
 
+message MasterKeyConfig {
+    encryptionpb.EncryptionMethod encryption_type = 1;
+    repeated encryptionpb.MasterKey master_keys = 2;
+}
+
 message BackupRequest {
     uint64 cluster_id = 1;
 
@@ -356,7 +369,7 @@ message StreamBackupTaskInfo {
     uint64 start_ts = 2;
     uint64 end_ts = 3;
 
-    // Misc meta datas.
+    // Misc meta data.
     // The name of the task, also the ID of the task.
     string name = 4;
     // The table filter of the task.
@@ -366,7 +379,10 @@ message StreamBackupTaskInfo {
     // compression type
     CompressionType compression_type = 6;
 
-    // The last timestamp of the task has been updated. 
+    // security config for backup files
+    StreamBackupTaskSecurityConfig security_config = 7;
+
+    // The last timestamp of the task has been updated.
     // This is a simple solution for unfrequent config changing: 
     // When we watched a config change(via polling or etcd watching),
     // We perform a incremental scan between [last_update_ts, now), 
@@ -665,7 +681,7 @@ message DataFileGroup {
 }
 
 message DataFileInfo {
-    // SHA256 of the file.
+    // Checksum of the plaintext file, i.e., pre-compression, pre-encryption.
     bytes sha256 = 1;
     // Path of the file.
     string path = 2;
@@ -721,12 +737,12 @@ message DataFileInfo {
     bytes region_end_key = 21;
     // The region epoch that the log file belongs to.
     // In older versions, this might be empty.
-    // If a region get split or merged during observing, 
+    // If a region get split or merged during observing,
     // the file may contain multi epoches.
     repeated metapb.RegionEpoch region_epoch = 22;
 
-    // It may support encrypting at future.
-    reserved "iv";
+    // Encryption information of this data file, not set if plaintext.
+    encryptionpb.FileEncryptionInfo file_encryption_info = 23;
 }
 
 message StreamBackupError {
@@ -812,12 +828,12 @@ message LogFileCompaction {
 }
 
 message MetaEdit {
-    // Path to the meta file. 
+    // Path to the meta file.
     string path = 1;
     // Delete the physical files (MetaFileGroup) in the meta file.
     repeated string delete_physical_files = 2;
     // Delete the logical files (MetaFileInfo) in the meta file.
-    // Note: Even the operation have been performed in the meta, 
+    // Note: Even the operation have been performed in the meta,
     // this modification should be kept as long as the corresponding physical
     // file not deleted. Or we may cannot know when to delete the physical file.
     // Then the file will be leak until truncated.
@@ -826,7 +842,7 @@ message MetaEdit {
     bool destruct_self = 4;
 }
 
-// An extended version of `SpansOfFile`, added more metadata for the 
+// An extended version of `SpansOfFile`, added more metadata for the
 // execution of delayed deletion.
 message DeleteSpansOfFile {
     string path = 1;

proto/encryptionpb.proto

@@ -96,6 +96,30 @@ message MasterKeyKms {
     string region = 3;
     // KMS endpoint. Normally not needed.
     string endpoint = 4;
+    // optional, used to set up azure master key backend
+    AzureKms azure_kms = 5;
+    // optional, used to set up gcp master key backend
+    GcpKms gcp_kms = 6;
+
+}
+
+message AzureKms {
+    string tenant_id = 1;
+    string client_id = 2;
+    string client_secret = 3;
+    // Key vault to encrypt/decrypt data key.
+    string key_vault_url = 4;
+    // optional hsm used to generate data key
+    string hsm_name = 5;
+    string hsm_url = 6;
+    string client_certificate = 7;
+    string client_certificate_path = 8;
+    string client_certificate_password = 9;
+
+}
+
+message GcpKms {
+    string credential = 1;
 }
 
 message EncryptedContent {
@@ -113,3 +137,25 @@ message EncryptedContent {
     // Valid only when KMS is used.
     bytes ciphertext_key = 5;
 }
+
+message FileEncryptionInfo {
+    oneof mode {
+        PlainTextDataKey plain_text_data_key = 1;
+        MasterKeyBased master_key_based = 2;
+    }
+    // file encryption method
+    encryptionpb.EncryptionMethod encryption_method = 3;
+    // iv to encrypt the file by data key
+    bytes file_iv = 4;
+    // file checksum after encryption, optional if using GCM
+    bytes checksum = 5;
+}
+
+// not recommended in production.
+// user needs to pass back the same data key for restore.
+message PlainTextDataKey {}
+
+message MasterKeyBased {
+    // encrypted data key with metadata
+    repeated encryptionpb.EncryptedContent data_key_encrypted_content = 1;
+}

proto/import_sstpb.proto

@@ -8,6 +8,7 @@ import "kvrpcpb.proto";
 import "gogoproto/gogo.proto";
 import "rustproto.proto";
 import "brpb.proto";
+import "encryptionpb.proto";
 
 option (gogoproto.sizer_all) = true;
 option (gogoproto.marshaler_all) = true;
@@ -394,6 +395,9 @@ message KVMeta {
 
     // the compression type for the file.
     backup.CompressionType compression_type = 13;
+
+    // encryption information of the kv file, not set if encryption is not enabled.
+    encryptionpb.FileEncryptionInfo file_encryption_info = 14;
 }
 
 
@@ -425,8 +429,11 @@ message ApplyRequest {
     // context represents region info and it used to build raft commands.
     kvrpcpb.Context context = 4;
 
-    // cipher_info is used to decrypt kv file when download file.
+    // plaintext data key to decrypt kv file if configured during log backup.
     backup.CipherInfo cipher_info = 11;
+
+    // master keys config used to decrypt data keys in restore if configured during log backup.
+    repeated encryptionpb.MasterKey master_keys = 14;
 }
 
 message ApplyResponse {

scripts/check.sh

@@ -1,12 +1,17 @@
 #!/usr/bin/env bash
 
 check_protoc_version() {
-    version=$(protoc --version)
-    major=$(echo ${version} | sed -n -e 's/.*\([0-9]\{1,\}\)\.[0-9]\{1,\}\.[0-9]\{1,\}.*/\1/p')
-    minor=$(echo ${version} | sed -n -e 's/.*[0-9]\{1,\}\.\([0-9]\{1,\}\)\.[0-9]\{1,\}.*/\1/p')
+    version=$(protoc --version | awk '{print $NF}')
+    major=$(echo ${version} | cut -d '.' -f 1)
+    minor=$(echo ${version} | cut -d '.' -f 2)
     if [ "$major" -eq 3 ] && [ "$minor" -ge 8 ]; then
         return 0
     fi
+    # protobuf bumps the major version to 21 after 3.
+    # https://github.com/protocolbuffers/protobuf/releases/tag/v21.7
+    if [ "$major" -ge 21 ]; then
+        return 0
+    fi
     echo "protoc version not match, version 3.8.x+ is needed, current version: ${version}"
     return 1
 }