Azure Backend cherry-pick from hashicorp#28211 hashicorp#28216

CHANGELOG.md

@@ -1,5 +1,9 @@
 ## 0.15.0-rc1 (Unreleased)
 
+ENHANCEMENTS:
+
+* backend/azurerm: Dependency Update and Fixes [GH-28181]
+
 BUG FIXES:
 
 * core: Fix crash when referencing resources with sensitive fields that may be unknown [GH-28180]

backend/remote-state/azure/arm_client.go

@@ -11,7 +11,7 @@ import (
 	"github.com/tombuildsstuff/giovanni/storage/2018-11-09/blob/containers"
 
 	"github.com/Azure/azure-sdk-for-go/profiles/2017-03-09/resources/mgmt/resources"
-	armStorage "github.com/Azure/azure-sdk-for-go/profiles/2017-03-09/storage/mgmt/storage"
+	armStorage "github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2021-01-01/storage"
 	"github.com/Azure/go-autorest/autorest"
 	"github.com/Azure/go-autorest/autorest/azure"
 	"github.com/hashicorp/go-azure-helpers/authentication"
@@ -26,6 +26,9 @@ type ArmClient struct {
 	containersClient      *containers.Client
 	blobsClient           *blobs.Client
 
+	// azureAdStorageAuth is only here if we're using AzureAD Authentication but is an Authorizer for Storage
+	azureAdStorageAuth *autorest.Authorizer
+
 	accessKey          string
 	environment        azure.Environment
 	resourceGroupName  string
@@ -92,11 +95,20 @@ func buildArmClient(ctx context.Context, config BackendConfig) (*ArmClient, erro
 		return nil, err
 	}
 
-	auth, err := armConfig.GetAuthorizationToken(sender.BuildSender("backend/remote-state/azure"), oauthConfig, env.TokenAudience)
+	sender := sender.BuildSender("backend/remote-state/azure")
+	auth, err := armConfig.GetAuthorizationToken(sender, oauthConfig, env.TokenAudience)
 	if err != nil {
 		return nil, err
 	}
 
+	if config.UseAzureADAuthentication {
+		storageAuth, err := armConfig.GetAuthorizationToken(sender, oauthConfig, env.ResourceIdentifiers.Storage)
+		if err != nil {
+			return nil, err
+		}
+		client.azureAdStorageAuth = &storageAuth
+	}
+
 	accountsClient := armStorage.NewAccountsClientWithBaseURI(env.ResourceManagerEndpoint, armConfig.SubscriptionID)
 	client.configureClient(&accountsClient.Client, auth)
 	client.storageAccountsClient = &accountsClient
@@ -109,6 +121,8 @@ func buildArmClient(ctx context.Context, config BackendConfig) (*ArmClient, erro
 }
 
 func buildArmEnvironment(config BackendConfig) (*azure.Environment, error) {
+	// TODO: can we remove this?
+	// https://github.com/hashicorp/terraform/issues/27156
 	if config.CustomResourceManagerEndpoint != "" {
 		log.Printf("[DEBUG] Loading Environment from Endpoint %q", config.CustomResourceManagerEndpoint)
 		return authentication.LoadEnvironmentFromUrl(config.CustomResourceManagerEndpoint)
@@ -131,10 +145,16 @@ func (c ArmClient) getBlobClient(ctx context.Context) (*blobs.Client, error) {
 		return &blobsClient, nil
 	}
 
+	if c.azureAdStorageAuth != nil {
+		blobsClient := blobs.NewWithEnvironment(c.environment)
+		c.configureClient(&blobsClient.Client, *c.azureAdStorageAuth)
+		return &blobsClient, nil
+	}
+
 	accessKey := c.accessKey
 	if accessKey == "" {
 		log.Printf("[DEBUG] Building the Blob Client from an Access Token (using user credentials)")
-		keys, err := c.storageAccountsClient.ListKeys(ctx, c.resourceGroupName, c.storageAccountName)
+		keys, err := c.storageAccountsClient.ListKeys(ctx, c.resourceGroupName, c.storageAccountName, "")
 		if err != nil {
 			return nil, fmt.Errorf("Error retrieving keys for Storage Account %q: %s", c.storageAccountName, err)
 		}
@@ -169,10 +189,17 @@ func (c ArmClient) getContainersClient(ctx context.Context) (*containers.Client,
 		c.configureClient(&containersClient.Client, storageAuth)
 		return &containersClient, nil
 	}
+
+	if c.azureAdStorageAuth != nil {
+		containersClient := containers.NewWithEnvironment(c.environment)
+		c.configureClient(&containersClient.Client, *c.azureAdStorageAuth)
+		return &containersClient, nil
+	}
+
 	accessKey := c.accessKey
 	if accessKey == "" {
 		log.Printf("[DEBUG] Building the Container Client from an Access Token (using user credentials)")
-		keys, err := c.storageAccountsClient.ListKeys(ctx, c.resourceGroupName, c.storageAccountName)
+		keys, err := c.storageAccountsClient.ListKeys(ctx, c.resourceGroupName, c.storageAccountName, "")
 		if err != nil {
 			return nil, fmt.Errorf("Error retrieving keys for Storage Account %q: %s", c.storageAccountName, err)
 		}

backend/remote-state/azure/backend.go

@@ -125,7 +125,7 @@ func New() backend.Backend {
 			"use_msi": {
 				Type:        schema.TypeBool,
 				Optional:    true,
-				Description: "Should Managed Service Identity be used?.",
+				Description: "Should Managed Service Identity be used?",
 				DefaultFunc: schema.EnvDefaultFunc("ARM_USE_MSI", false),
 			},
 			"msi_endpoint": {
@@ -135,33 +135,12 @@ func New() backend.Backend {
 				DefaultFunc: schema.EnvDefaultFunc("ARM_MSI_ENDPOINT", ""),
 			},
 
-			// Deprecated fields
-			"arm_client_id": {
-				Type:        schema.TypeString,
-				Optional:    true,
-				Description: "The Client ID.",
-				Deprecated:  "`arm_client_id` has been replaced by `client_id`",
-			},
-
-			"arm_client_secret": {
-				Type:        schema.TypeString,
-				Optional:    true,
-				Description: "The Client Secret.",
-				Deprecated:  "`arm_client_secret` has been replaced by `client_secret`",
-			},
-
-			"arm_subscription_id": {
-				Type:        schema.TypeString,
-				Optional:    true,
-				Description: "The Subscription ID.",
-				Deprecated:  "`arm_subscription_id` has been replaced by `subscription_id`",
-			},
-
-			"arm_tenant_id": {
-				Type:        schema.TypeString,
+			// Feature Flags
+			"use_azuread_auth": {
+				Type:        schema.TypeBool,
 				Optional:    true,
-				Description: "The Tenant ID.",
-				Deprecated:  "`arm_tenant_id` has been replaced by `tenant_id`",
+				Description: "Should Terraform use AzureAD Authentication to access the Blob?",
+				DefaultFunc: schema.EnvDefaultFunc("ARM_USE_AZUREAD", false),
 			},
 		},
 	}
@@ -201,6 +180,7 @@ type BackendConfig struct {
 	SubscriptionID                string
 	TenantID                      string
 	UseMsi                        bool
+	UseAzureADAuthentication      bool
 }
 
 func (b *Backend) configure(ctx context.Context) error {
@@ -215,49 +195,35 @@ func (b *Backend) configure(ctx context.Context) error {
 	b.keyName = data.Get("key").(string)
 	b.snapshot = data.Get("snapshot").(bool)
 
-	// support for previously deprecated fields
-	clientId := valueFromDeprecatedField(data, "client_id", "arm_client_id")
-	clientSecret := valueFromDeprecatedField(data, "client_secret", "arm_client_secret")
-	subscriptionId := valueFromDeprecatedField(data, "subscription_id", "arm_subscription_id")
-	tenantId := valueFromDeprecatedField(data, "tenant_id", "arm_tenant_id")
-
 	config := BackendConfig{
 		AccessKey:                     data.Get("access_key").(string),
-		ClientID:                      clientId,
+		ClientID:                      data.Get("client_id").(string),
 		ClientCertificatePassword:     data.Get("client_certificate_password").(string),
 		ClientCertificatePath:         data.Get("client_certificate_path").(string),
-		ClientSecret:                  clientSecret,
+		ClientSecret:                  data.Get("client_secret").(string),
 		CustomResourceManagerEndpoint: data.Get("endpoint").(string),
 		MetadataHost:                  data.Get("metadata_host").(string),
 		Environment:                   data.Get("environment").(string),
 		MsiEndpoint:                   data.Get("msi_endpoint").(string),
 		ResourceGroupName:             data.Get("resource_group_name").(string),
 		SasToken:                      data.Get("sas_token").(string),
 		StorageAccountName:            data.Get("storage_account_name").(string),
-		SubscriptionID:                subscriptionId,
-		TenantID:                      tenantId,
+		SubscriptionID:                data.Get("subscription_id").(string),
+		TenantID:                      data.Get("tenant_id").(string),
 		UseMsi:                        data.Get("use_msi").(bool),
+		UseAzureADAuthentication:      data.Get("use_azuread_auth").(bool),
 	}
 
 	armClient, err := buildArmClient(context.TODO(), config)
 	if err != nil {
 		return err
 	}
 
-	if config.AccessKey == "" && config.SasToken == "" && config.ResourceGroupName == "" {
-		return fmt.Errorf("Either an Access Key / SAS Token or the Resource Group for the Storage Account must be specified")
+	thingsNeededToLookupAccessKeySpecified := config.AccessKey == "" && config.SasToken == "" && config.ResourceGroupName == ""
+	if thingsNeededToLookupAccessKeySpecified && !config.UseAzureADAuthentication {
+		return fmt.Errorf("Either an Access Key / SAS Token or the Resource Group for the Storage Account must be specified - or Azure AD Authentication must be enabled")
 	}
 
 	b.armClient = armClient
 	return nil
 }
-
-func valueFromDeprecatedField(d *schema.ResourceData, key, deprecatedFieldKey string) string {
-	v := d.Get(key).(string)
-
-	if v == "" {
-		v = d.Get(deprecatedFieldKey).(string)
-	}
-
-	return v
-}

backend/remote-state/azure/backend_test.go

@@ -123,6 +123,34 @@ func TestBackendSASTokenBasic(t *testing.T) {
 	backend.TestBackendStates(t, b)
 }
 
+func TestBackendAzureADAuthBasic(t *testing.T) {
+	testAccAzureBackend(t)
+	rs := acctest.RandString(4)
+	res := testResourceNames(rs, "testState")
+	res.useAzureADAuth = true
+	armClient := buildTestClient(t, res)
+
+	ctx := context.TODO()
+	err := armClient.buildTestResources(ctx, &res)
+	defer armClient.destroyTestResources(ctx, res)
+	if err != nil {
+		armClient.destroyTestResources(ctx, res)
+		t.Fatalf("Error creating Test Resources: %q", err)
+	}
+
+	b := backend.TestBackendConfig(t, New(), backend.TestWrapConfig(map[string]interface{}{
+		"storage_account_name": res.storageAccountName,
+		"container_name":       res.storageContainerName,
+		"key":                  res.storageKeyName,
+		"access_key":           res.storageAccountAccessKey,
+		"environment":          os.Getenv("ARM_ENVIRONMENT"),
+		"endpoint":             os.Getenv("ARM_ENDPOINT"),
+		"use_azuread_auth":     true,
+	})).(*Backend)
+
+	backend.TestBackendStates(t, b)
+}
+
 func TestBackendServicePrincipalClientCertificateBasic(t *testing.T) {
 	testAccAzureBackend(t)
 

backend/remote-state/azure/client.go

@@ -110,7 +110,7 @@ func (c *RemoteClient) Delete() error {
 	ctx := context.TODO()
 	resp, err := c.giovanniBlobClient.Delete(ctx, c.accountName, c.containerName, c.keyName, options)
 	if err != nil {
-		if resp.Response.StatusCode != 404 {
+		if !resp.IsHTTPStatus(http.StatusNotFound) {
 			return err
 		}
 	}
@@ -152,7 +152,7 @@ func (c *RemoteClient) Lock(info *statemgr.LockInfo) (string, error) {
 	properties, err := c.giovanniBlobClient.GetProperties(ctx, c.accountName, c.containerName, c.keyName, blobs.GetPropertiesInput{})
 	if err != nil {
 		// error if we had issues getting the blob
-		if properties.Response.StatusCode != 404 {
+		if !properties.Response.IsHTTPStatus(http.StatusNotFound) {
 			return "", getLockInfoErr(err)
 		}
 		// if we don't find the blob, we need to build it

backend/remote-state/azure/helpers_test.go

@@ -10,7 +10,7 @@ import (
 	"time"
 
 	"github.com/Azure/azure-sdk-for-go/profiles/2017-03-09/resources/mgmt/resources"
-	armStorage "github.com/Azure/azure-sdk-for-go/profiles/2017-03-09/storage/mgmt/storage"
+	armStorage "github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2021-01-01/storage"
 	"github.com/Azure/go-autorest/autorest"
 	sasStorage "github.com/hashicorp/go-azure-helpers/storage"
 	"github.com/tombuildsstuff/giovanni/storage/2018-11-09/blob/containers"
@@ -83,6 +83,7 @@ func buildTestClient(t *testing.T, res resourceNames) *ArmClient {
 		ResourceGroupName:             res.resourceGroup,
 		StorageAccountName:            res.storageAccountName,
 		UseMsi:                        msiEnabled,
+		UseAzureADAuthentication:      res.useAzureADAuth,
 	})
 	if err != nil {
 		t.Fatalf("Failed to build ArmClient: %+v", err)
@@ -125,6 +126,7 @@ type resourceNames struct {
 	storageContainerName    string
 	storageKeyName          string
 	storageAccountAccessKey string
+	useAzureADAuth          bool
 }
 
 func testResourceNames(rString string, keyName string) resourceNames {
@@ -134,6 +136,7 @@ func testResourceNames(rString string, keyName string) resourceNames {
 		storageAccountName:   fmt.Sprintf("acctestsa%s", rString),
 		storageContainerName: "acctestcont",
 		storageKeyName:       keyName,
+		useAzureADAuth:       false,
 	}
 }
 
@@ -145,13 +148,20 @@ func (c *ArmClient) buildTestResources(ctx context.Context, names *resourceNames
 	}
 
 	log.Printf("Creating Storage Account %q in Resource Group %q", names.storageAccountName, names.resourceGroup)
-	future, err := c.storageAccountsClient.Create(ctx, names.resourceGroup, names.storageAccountName, armStorage.AccountCreateParameters{
+	storageProps := armStorage.AccountCreateParameters{
 		Sku: &armStorage.Sku{
 			Name: armStorage.StandardLRS,
 			Tier: armStorage.Standard,
 		},
 		Location: &names.location,
-	})
+	}
+	if names.useAzureADAuth {
+		allowSharedKeyAccess := false
+		storageProps.AccountPropertiesCreateParameters = &armStorage.AccountPropertiesCreateParameters{
+			AllowSharedKeyAccess: &allowSharedKeyAccess,
+		}
+	}
+	future, err := c.storageAccountsClient.Create(ctx, names.resourceGroup, names.storageAccountName, storageProps)
 	if err != nil {
 		return fmt.Errorf("failed to create test storage account: %s", err)
 	}
@@ -161,23 +171,27 @@ func (c *ArmClient) buildTestResources(ctx context.Context, names *resourceNames
 		return fmt.Errorf("failed waiting for the creation of storage account: %s", err)
 	}
 
-	log.Printf("fetching access key for storage account")
-	resp, err := c.storageAccountsClient.ListKeys(ctx, names.resourceGroup, names.storageAccountName)
-	if err != nil {
-		return fmt.Errorf("failed to list storage account keys %s:", err)
-	}
-
-	keys := *resp.Keys
-	accessKey := *keys[0].Value
-	names.storageAccountAccessKey = accessKey
-
-	storageAuth, err := autorest.NewSharedKeyAuthorizer(names.storageAccountName, accessKey, autorest.SharedKey)
-	if err != nil {
-		return fmt.Errorf("Error building Authorizer: %+v", err)
-	}
-
 	containersClient := containers.NewWithEnvironment(c.environment)
-	containersClient.Client.Authorizer = storageAuth
+	if names.useAzureADAuth {
+		containersClient.Client.Authorizer = *c.azureAdStorageAuth
+	} else {
+		log.Printf("fetching access key for storage account")
+		resp, err := c.storageAccountsClient.ListKeys(ctx, names.resourceGroup, names.storageAccountName, "")
+		if err != nil {
+			return fmt.Errorf("failed to list storage account keys %s:", err)
+		}
+
+		keys := *resp.Keys
+		accessKey := *keys[0].Value
+		names.storageAccountAccessKey = accessKey
+
+		storageAuth, err := autorest.NewSharedKeyAuthorizer(names.storageAccountName, accessKey, autorest.SharedKey)
+		if err != nil {
+			return fmt.Errorf("Error building Authorizer: %+v", err)
+		}
+
+		containersClient.Client.Authorizer = storageAuth
+	}
 
 	log.Printf("Creating Container %q in Storage Account %q (Resource Group %q)", names.storageContainerName, names.storageAccountName, names.resourceGroup)
 	_, err = containersClient.Create(ctx, names.storageAccountName, names.storageContainerName, containers.CreateInput{})