From 509aba28516ebe580ec1110147e7041ef4062f36 Mon Sep 17 00:00:00 2001 From: Hong Ooi Date: Sun, 9 May 2021 15:08:03 +1000 Subject: [PATCH 1/8] implement service sas --- NAMESPACE | 4 +++ R/sas.R | 93 ++++++++++++++++++++++++++++++++++++++++++++++++++++-- man/sas.Rd | 30 ++++++++++++++++-- 3 files changed, 122 insertions(+), 5 deletions(-) diff --git a/NAMESPACE b/NAMESPACE index 40ccf76..f67829f 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -47,6 +47,9 @@ S3method(file_share,file_endpoint) S3method(get_account_sas,az_storage) S3method(get_account_sas,default) S3method(get_account_sas,storage_endpoint) +S3method(get_service_sas,az_storage) +S3method(get_service_sas,default) +S3method(get_service_sas,storage_endpoint) S3method(get_storage_metadata,adls_filesystem) S3method(get_storage_metadata,blob_container) S3method(get_storage_metadata,file_share) @@ -145,6 +148,7 @@ export(file_share) export(get_account_sas) export(get_adls_file_acl) export(get_adls_file_status) +export(get_service_sas) export(get_storage_metadata) export(get_storage_properties) export(get_user_delegation_key) diff --git a/R/sas.R b/R/sas.R index 55d4867..f33f868 100644 --- a/R/sas.R +++ b/R/sas.R @@ -9,12 +9,12 @@ #' @param account An object representing a storage account. Depending on the generic, this can be one of the following: an Azure resource object (of class `az_storage`); a client storage endpoint (of class `storage_endpoint`); a _blob_ storage endpoint (of class `blob_endpoint`); or a string with the name of the account. #' @param key For `get_account_sas`, the _account_ key, which controls full access to the storage account. For `get_user_delegation_sas`, a _user delegation_ key, as obtained from `get_user_delegation_key`. #' @param token For `get_user_delegation_key`, an AAD token from which to obtain user details. The token must have `https://storage.azure.com` as its audience. -#' @param resource For `get_user_delegation_sas`, the resource for which the SAS is valid. This can be either the name of a blob container, or a blob. If the latter, it should include the container as well (`containername/blobname`). +#' @param resource For `get_user_delegation_sas` and `get_service_sas`, the resource for which the SAS is valid. For a user delegation SAS, this can be either the name of a blob container or an individual blob; if the latter, it should include the container as well (`containername/blobname`). A service SAS is similar but can also be used with file shares and files. #' @param start,expiry The start and end dates for the account or user delegation SAS. These should be `Date` or `POSIXct` values, or strings coercible to such. If not supplied, the default is to generate start and expiry values for a period of 8 hours, starting from 15 minutes before the current time. #' @param key_start,key_expiry For `get_user_delegation_key`, the start and end dates for the user delegation key. #' @param services For `get_account_sas`, the storage service(s) for which the SAS is valid. Defaults to `bqtf`, meaning blob (including ADLS2), queue, table and file storage. #' @param permissions For `get_account_sas` and `get_user_delegation_sas`, the permissions that the SAS grants. The default `rl` (read and list) essentially means read-only access. -#' @param resource_types The resource types for which the SAS is valid. For `get_account_sas` the default is `sco` meaning service, container and object. For `get_user_delegation_sas` the default is `c` meaning container-level access (including blobs within the container). +#' @param resource_types For an account or user delegation SAS, the resource types for which the SAS is valid. For `get_account_sas` the default is `sco` meaning service, container and object. For `get_user_delegation_sas` the default is `c` meaning container-level access (including blobs within the container). #' @param ip The IP address(es) or IP address range(s) for which the SAS is valid. The default is not to restrict access by IP. #' @param protocol The protocol required to use the SAS. Possible values are `https` meaning HTTPS-only, or `https,http` meaning HTTP is also allowed. Note that the storage account itself may require HTTPS, regardless of what the SAS allows. #' @param snapshot_time For `get_user_delegation_sas`, the blob snapshot for which the SAS is valid. Only required if `resource_types="bs"`. @@ -38,6 +38,7 @@ #' [Azure Storage Services API reference](https://docs.microsoft.com/en-us/rest/api/storageservices/), #' [Create an account SAS](https://docs.microsoft.com/en-us/rest/api/storageservices/create-account-sas), #' [Create a user delegation SAS](https://docs.microsoft.com/en-us/rest/api/storageservices/create-user-delegation-sas) +#' [Create a service SAS](https://docs.microsoft.com/en-us/rest/api/storageservices/create-service-sas) #' #' @examples #' # account SAS valid for 7 days @@ -269,6 +270,94 @@ get_user_delegation_sas.default <- function(account, key, resource, start=NULL, } +#' @rdname sas +#' @export +get_service_sas <- function(account, ...) +{ + UseMethod("get_service_sas") +} + +#' @rdname sas +#' @export +get_service_sas.az_storage <- function(account, resource, service_type=c("blob", "file"), key=account$list_keys()[1], + ...) +{ + service_type <- match.arg(service_type) + endp <- if(service_type == "blob") + account$get_blob_endpoint(key=key) + else account$get_file_endpoint(key=key) + get_service_sas(endp, resource=resource, key=key, ...) +} + +#' @rdname sas +#' @export +get_service_sas.storage_endpoint <- function(account, resource, key=account$key, ...) +{ + if(is.null(key)) + stop("Must have access key to generate SAS", call.=FALSE) + + service_type <- sub("_endpoint$", "", class(account)[1]) + acctname <- sub("\\..*", "", httr::parse_url(account$url)$hostname) + get_service_sas(acctname, resource=resource, key=key, service_type=service_type, ...) +} + + +#' @param service_type For a service SAS, the storage service for which the SAS is valid: either "blob" or "file". Currently AzureStor does not support creating a service SAS for queue or table storage. +#' @param resource_type For a service SAS, the type of resource for which the SAS is valid. For blob storage, the default value is "b" meaning a single blob. For file storage, the default value is "f" meaning a single file. Other possible values include "bs" (a blob snapshot), "c" (a blob container), or "s" (a file share). +#' @param policy For a service SAS, optionally the name of a stored access policy to correlate the SAS with. Revoking the policy will also invalidate the SAS. +#' @param snapshot_time For a service SAS, the specific blob snapshot for which the SAS is valid, if `resource_type` is "bs". +#' @rdname sas +#' @export +get_service_sas.default <- function(account, resource, key, service_type, resource_type=NULL, start=NULL, expiry=NULL, + permissions="rl", ip=NULL, protocol=NULL, policy=NULL, snapshot_time=NULL, + auth_api_version=getOption("azure_storage_api_version"), ...) +{ + if(!(service_type %in% c("blob", "file"))) + stop("Generating a service SAS currently only supported for blob and file storage", call.=FALSE) + + dates <- make_sas_dates(start, expiry) + if(is.null(resource_type)) + resource_type <- if(service_type == "blob") "b" else "f" + resource <- file.path("", service_type, account, resource) # canonicalized resource starts with / + + sig_str <- paste( + permissions, + dates$start, + dates$expiry, + resource, + policy, + ip, + protocol, + auth_api_version, + resource_type, + snapshot_time, + "", # rscc, rscd, rsce, rscl, rsct not yet implemented + "", + "", + "", + "", + sep="\n" + ) + sig <- sign_sha256(sig_str, key) + + parts <- list( + sv=auth_api_version, + sr=resource_type, + sp=permissions, + st=dates$start, + se=dates$expiry, + sip=ip, + spr=protocol, + si=policy, + # sdd=directory_depth, + sig=sig + ) + parts <- parts[!sapply(parts, is_empty)] + parts <- sapply(parts, url_encode, reserved=TRUE) + paste(names(parts), parts, sep="=", collapse="&") +} + + make_sas_dates <- function(start=NULL, expiry=NULL) { if(is.null(start)) diff --git a/man/sas.Rd b/man/sas.Rd index 5dd8a84..c688d01 100644 --- a/man/sas.Rd +++ b/man/sas.Rd @@ -17,6 +17,10 @@ \alias{get_user_delegation_sas.az_storage} \alias{get_user_delegation_sas.blob_endpoint} \alias{get_user_delegation_sas.default} +\alias{get_service_sas} +\alias{get_service_sas.az_storage} +\alias{get_service_sas.storage_endpoint} +\alias{get_service_sas.default} \title{Generate shared access signatures} \usage{ get_account_sas(account, ...) @@ -52,6 +56,19 @@ get_user_delegation_sas(account, ...) resource_types = "c", ip = NULL, protocol = NULL, snapshot_time = NULL, auth_api_version = getOption("azure_storage_api_version"), ...) + +get_service_sas(account, ...) + +\method{get_service_sas}{az_storage}(account, resource, + service_type = c("blob", "file"), key = account$list_keys()[1], ...) + +\method{get_service_sas}{storage_endpoint}(account, resource, + key = account$key, resource_type = NULL, ...) + +\method{get_service_sas}{default}(account, resource, key, service_type, + resource_type, start = NULL, expiry = NULL, permissions = "rl", + ip = NULL, protocol = NULL, policy = NULL, snapshot_time = NULL, + auth_api_version = getOption("azure_storage_api_version"), ...) } \arguments{ \item{account}{An object representing a storage account. Depending on the generic, this can be one of the following: an Azure resource object (of class \code{az_storage}); a client storage endpoint (of class \code{storage_endpoint}); a \emph{blob} storage endpoint (of class \code{blob_endpoint}); or a string with the name of the account.} @@ -66,7 +83,7 @@ get_user_delegation_sas(account, ...) \item{permissions}{For \code{get_account_sas} and \code{get_user_delegation_sas}, the permissions that the SAS grants. The default \code{rl} (read and list) essentially means read-only access.} -\item{resource_types}{The resource types for which the SAS is valid. For \code{get_account_sas} the default is \code{sco} meaning service, container and object. For \code{get_user_delegation_sas} the default is \code{c} meaning container-level access (including blobs within the container).} +\item{resource_types}{For an account or user delegation SAS, the resource types for which the SAS is valid. For \code{get_account_sas} the default is \code{sco} meaning service, container and object. For \code{get_user_delegation_sas} the default is \code{c} meaning container-level access (including blobs within the container).} \item{ip}{The IP address(es) or IP address range(s) for which the SAS is valid. The default is not to restrict access by IP.} @@ -78,9 +95,15 @@ get_user_delegation_sas(account, ...) \item{key_start, key_expiry}{For \code{get_user_delegation_key}, the start and end dates for the user delegation key.} -\item{resource}{For \code{get_user_delegation_sas}, the resource for which the SAS is valid. This can be either the name of a blob container, or a blob. If the latter, it should include the container as well (\code{containername/blobname}).} +\item{resource}{For \code{get_user_delegation_sas} and \code{get_service_sas}, the resource for which the SAS is valid. For a user delegation SAS, this can be either the name of a blob container or an individual blob; if the latter, it should include the container as well (\code{containername/blobname}). A service SAS is similar but can also be used with file shares and files.} + +\item{snapshot_time}{For a service SAS, the specific blob snapshot for which the SAS is valid, if \code{resource_type} is "bs".} + +\item{service_type}{For a service SAS, the storage service for which the SAS is valid: either "blob" or "file". Currently AzureStor does not support creating a service SAS for queue or table storage.} + +\item{resource_type}{For a service SAS, the type of resource for which the SAS is valid. For blob storage, the default value is "b" meaning a single blob. For file storage, the default value is "f" meaning a single file. Other possible values include "bs" (a blob snapshot), "c" (a blob container), or "s" (a file share).} -\item{snapshot_time}{For \code{get_user_delegation_sas}, the blob snapshot for which the SAS is valid. Only required if \code{resource_types="bs"}.} +\item{policy}{For a service SAS, optionally the name of a stored access policy to correlate the SAS with. Revoking the policy will also invalidate the SAS.} } \description{ The simplest way for a user to access files and data in a storage account is to give them the account's access key. This gives them full control of the account, and so may be a security risk. An alternative is to provide the user with a \emph{shared access signature} (SAS), which limits access to specific resources and only for a set length of time. AzureStor supports generating two kinds of SAS: account and user delegation, with the latter applying only to blob and ADLS2 storage. @@ -141,4 +164,5 @@ get_user_delegation_sas(endp, userkey, resource="mycontainer/myfile", \href{https://docs.microsoft.com/en-us/rest/api/storageservices/}{Azure Storage Services API reference}, \href{https://docs.microsoft.com/en-us/rest/api/storageservices/create-account-sas}{Create an account SAS}, \href{https://docs.microsoft.com/en-us/rest/api/storageservices/create-user-delegation-sas}{Create a user delegation SAS} +\href{https://docs.microsoft.com/en-us/rest/api/storageservices/create-service-sas}{Create a service SAS} } From 62674b329d2c28e6f8355070d4443a6068847d8e Mon Sep 17 00:00:00 2001 From: Hong Ooi Date: Sun, 9 May 2021 15:13:54 +1000 Subject: [PATCH 2/8] fix doc --- man/sas.Rd | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/man/sas.Rd b/man/sas.Rd index c688d01..f99a0ee 100644 --- a/man/sas.Rd +++ b/man/sas.Rd @@ -62,12 +62,12 @@ get_service_sas(account, ...) \method{get_service_sas}{az_storage}(account, resource, service_type = c("blob", "file"), key = account$list_keys()[1], ...) -\method{get_service_sas}{storage_endpoint}(account, resource, - key = account$key, resource_type = NULL, ...) +\method{get_service_sas}{storage_endpoint}(account, resource, key = account$key, ...) \method{get_service_sas}{default}(account, resource, key, service_type, - resource_type, start = NULL, expiry = NULL, permissions = "rl", - ip = NULL, protocol = NULL, policy = NULL, snapshot_time = NULL, + resource_type = NULL, start = NULL, expiry = NULL, + permissions = "rl", ip = NULL, protocol = NULL, policy = NULL, + snapshot_time = NULL, auth_api_version = getOption("azure_storage_api_version"), ...) } \arguments{ From 9a29ef622696fa6242d8bdf4c58f226a7ad08daf Mon Sep 17 00:00:00 2001 From: Hong Ooi Date: Sun, 9 May 2021 16:21:15 +1000 Subject: [PATCH 3/8] add R6 method, redoc --- NEWS.md | 2 ++ R/az_storage.R | 30 ++++++++++++++++++++++++++---- R/sas.R | 46 +++++++++++++++++++++++++++++----------------- man/az_storage.Rd | 19 +++++++++++++++---- man/sas.Rd | 35 ++++++++++++++++++++++++----------- 5 files changed, 96 insertions(+), 36 deletions(-) diff --git a/NEWS.md b/NEWS.md index 6a840df..66dfb81 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,7 @@ # AzureStor 3.4.1.9000 +- Add support for generating a service SAS. There is a new S3 generic `get_service_sas` with methods for `az_storage` and `storage_endpoint` objects, and a similar R6 method for `az_storage` objects. + - See `?sas` for more information. - Fix `storage_save_rds` and `storage_load_rds` to handle compression correctly. In particular, `storage_load_rds` should now correctly load files saved with `saveRDS` (#83). - Fix a bug that caused `list_blobs` to fail when leases were present. - Use a raw connection instead of a raw vector when calling `readr::read_delim` and `read_csv2`. This works around an issue introduced in readr 1.4.0 (#85, #86). diff --git a/R/az_storage.R b/R/az_storage.R index a028c4c..caa8b21 100644 --- a/R/az_storage.R +++ b/R/az_storage.R @@ -21,13 +21,22 @@ #' @section Creating a shared access signature: #' Note that you don't need to worry about this section if you have been _given_ a SAS, and only want to use it to access storage. #' -#' AzureStor supports generating two kinds of SAS: account and user delegation, with the latter applying only to blob and ADLS2 storage. To create an account SAS, call the `get_account_sas()` method. This has the following signature: +#' AzureStor supports generating three kinds of SAS: account, service and user delegation. An account SAS can be used with any type of storage. A service SAS can be used with blob and file storage, whle a user delegation SAS can be used with blob and ADLS2 storage. +#' +#' To create an account SAS, call the `get_account_sas()` method. This has the following signature: #' #' ``` #' get_account_sas(key=self$list_keys()[1], start=NULL, expiry=NULL, services="bqtf", permissions="rl", #' resource_types="sco", ip=NULL, protocol=NULL) #' ``` #' +#' To create a service SAS, call the `get_service_sas()` method, which has the following signature: +#' +#' ``` +#' get_service_sas(key=self$list_keys()[1], resource, service, start=NULL, expiry=NULL, permissions="r", +#' resource_type=NULL, ip=NULL, protocol=NULL, policy=NULL, snapshot_time=NULL) +#' ``` +#' #' To create a user delegation SAS, you must first create a user delegation _key_. This takes the place of the account's access key in generating the SAS. The `get_user_delegation_key()` method has the following signature: #' #' ``` @@ -41,6 +50,8 @@ #' resource_types="c", ip=NULL, protocol=NULL, snapshot_time=NULL) #' ``` #' +#' (Note that the `key` argument for this method is the user delegation key, _not_ the account key.) +#' #' To invalidate all user delegation keys, as well as the SAS's generated with them, call the `revoke_user_delegation_keys()` method. This has the following signature: #' #' ``` @@ -60,11 +71,14 @@ #' #' @seealso #' [blob_endpoint], [file_endpoint], -#' [create_storage_account], [get_storage_account], [delete_storage_account], [Date], [POSIXt], +#' [create_storage_account], [get_storage_account], [delete_storage_account], [Date], [POSIXt] +#' #' [Azure Storage Provider API reference](https://docs.microsoft.com/en-us/rest/api/storagerp/), -#' [Azure Storage Services API reference](https://docs.microsoft.com/en-us/rest/api/storageservices/), +#' [Azure Storage Services API reference](https://docs.microsoft.com/en-us/rest/api/storageservices/) +#' #' [Create an account SAS](https://docs.microsoft.com/en-us/rest/api/storageservices/create-account-sas), -#' [Create a user delegation SAS](https://docs.microsoft.com/en-us/rest/api/storageservices/create-user-delegation-sas) +#' [Create a user delegation SAS](https://docs.microsoft.com/en-us/rest/api/storageservices/create-user-delegation-sas), +#' [Create a service SAS](https://docs.microsoft.com/en-us/rest/api/storageservices/create-service-sas) #' #' @examples #' \dontrun{ @@ -118,6 +132,14 @@ public=list( resource_types=resource_types, ip=ip, protocol=protocol, snapshot_time=snapshot_time) }, + get_service_sas=function(key=self$list_keys()[1], resource, service, start=NULL, expiry=NULL, permissions="r", + resource_type=NULL, ip=NULL, protocol=NULL, policy=NULL, snapshot_time=NULL) + { + get_service_sas(self, key=key, resource=resource, service=service, start=start, expiry=expiry, + permissions=permissions, resource_type=resource_type, ip=ip, protocol=protocol, policy=policy, + snapshot_time=snapshot_time) + }, + get_blob_endpoint=function(key=self$list_keys()[1], sas=NULL, token=NULL) { blob_endpoint(self$properties$primaryEndpoints$blob, key=key, sas=sas, token=token) diff --git a/R/sas.R b/R/sas.R index f33f868..4b3bd5f 100644 --- a/R/sas.R +++ b/R/sas.R @@ -13,17 +13,19 @@ #' @param start,expiry The start and end dates for the account or user delegation SAS. These should be `Date` or `POSIXct` values, or strings coercible to such. If not supplied, the default is to generate start and expiry values for a period of 8 hours, starting from 15 minutes before the current time. #' @param key_start,key_expiry For `get_user_delegation_key`, the start and end dates for the user delegation key. #' @param services For `get_account_sas`, the storage service(s) for which the SAS is valid. Defaults to `bqtf`, meaning blob (including ADLS2), queue, table and file storage. -#' @param permissions For `get_account_sas` and `get_user_delegation_sas`, the permissions that the SAS grants. The default `rl` (read and list) essentially means read-only access. +#' @param permissions The permissions that the SAS grants. The default value of `rl` (read and list) essentially means read-only access. #' @param resource_types For an account or user delegation SAS, the resource types for which the SAS is valid. For `get_account_sas` the default is `sco` meaning service, container and object. For `get_user_delegation_sas` the default is `c` meaning container-level access (including blobs within the container). #' @param ip The IP address(es) or IP address range(s) for which the SAS is valid. The default is not to restrict access by IP. #' @param protocol The protocol required to use the SAS. Possible values are `https` meaning HTTPS-only, or `https,http` meaning HTTP is also allowed. Note that the storage account itself may require HTTPS, regardless of what the SAS allows. -#' @param snapshot_time For `get_user_delegation_sas`, the blob snapshot for which the SAS is valid. Only required if `resource_types="bs"`. +#' @param snapshot_time For a user delegation or service SAS, the blob snapshot for which the SAS is valid. Only required if `resource_type[s]="bs"`. #' @param auth_api_version The storage API version to use for authenticating. #' @param ... Arguments passed to lower-level functions. #' #' @details #' An **account SAS** is secured with the storage account key. An account SAS delegates access to resources in one or more of the storage services. All of the operations available via a user delegation SAS are also available via an account SAS. You can also delegate access to read, write, and delete operations on blob containers, tables, queues, and file shares. To obtain an account SAS, call `get_account_sas`. #' +#' A **service SAS** is like an account SAS, but allows finer-grained control of access. You can create a service SAS that allows access only to specific blobs in a container, or files in a file share. To obtain a service SAS, call `get_service_sas`. +#' #' A **user delegation SAS** is a SAS secured with Azure AD credentials. It's recommended that you use Azure AD credentials when possible as a security best practice, rather than using the account key, which can be more easily compromised. When your application design requires shared access signatures, use Azure AD credentials to create a user delegation SAS for superior security. #' #' Every SAS is signed with a key. To create a user delegation SAS, you must first request a **user delegation key**, which is then used to sign the SAS. The user delegation key is analogous to the account key used to sign a service SAS or an account SAS, except that it relies on your Azure AD credentials. To request the user delegation key, call `get_user_delegation_key`. With the user delegation key, you can then create the SAS with `get_user_delegation_sas`. @@ -33,11 +35,13 @@ #' See the examples and Microsoft Docs pages below for how to specify arguments like the services, permissions, and resource types. Also, while not explicitly mentioned in the documentation, ADLSgen2 storage can use any SAS that is valid for blob storage. #' @seealso #' [blob_endpoint], [file_endpoint], -#' [Date], [POSIXt], +#' [Date], [POSIXt] +#' #' [Azure Storage Provider API reference](https://docs.microsoft.com/en-us/rest/api/storagerp/), -#' [Azure Storage Services API reference](https://docs.microsoft.com/en-us/rest/api/storageservices/), +#' [Azure Storage Services API reference](https://docs.microsoft.com/en-us/rest/api/storageservices/) +#' #' [Create an account SAS](https://docs.microsoft.com/en-us/rest/api/storageservices/create-account-sas), -#' [Create a user delegation SAS](https://docs.microsoft.com/en-us/rest/api/storageservices/create-user-delegation-sas) +#' [Create a user delegation SAS](https://docs.microsoft.com/en-us/rest/api/storageservices/create-user-delegation-sas), #' [Create a service SAS](https://docs.microsoft.com/en-us/rest/api/storageservices/create-service-sas) #' #' @examples @@ -57,6 +61,15 @@ #' endp <- storage_endpoint("https://mystorage.blob.core.windows.net", key="access_key") #' get_account_sas(endp, permissions="rwcd") #' +#' # service SAS for a container with read-only permissions +#' get_service_sas(bl, "containername") +#' +#' # service SAS for a container, allowing listing of blobs +#' get_service_sas(bl, "containername", permissions="rl") +#' +#' # read/write service SAS for a blob +#' get_service_sas(endp, "containername/blobname", permissions="rw") +#' #' \dontrun{ #' #' # user delegation key valid for 24 hours @@ -279,11 +292,11 @@ get_service_sas <- function(account, ...) #' @rdname sas #' @export -get_service_sas.az_storage <- function(account, resource, service_type=c("blob", "file"), key=account$list_keys()[1], +get_service_sas.az_storage <- function(account, resource, service=c("blob", "file"), key=account$list_keys()[1], ...) { - service_type <- match.arg(service_type) - endp <- if(service_type == "blob") + service <- match.arg(service) + endp <- if(service == "blob") account$get_blob_endpoint(key=key) else account$get_file_endpoint(key=key) get_service_sas(endp, resource=resource, key=key, ...) @@ -296,29 +309,28 @@ get_service_sas.storage_endpoint <- function(account, resource, key=account$key, if(is.null(key)) stop("Must have access key to generate SAS", call.=FALSE) - service_type <- sub("_endpoint$", "", class(account)[1]) + service <- sub("_endpoint$", "", class(account)[1]) acctname <- sub("\\..*", "", httr::parse_url(account$url)$hostname) - get_service_sas(acctname, resource=resource, key=key, service_type=service_type, ...) + get_service_sas(acctname, resource=resource, key=key, service=service, ...) } -#' @param service_type For a service SAS, the storage service for which the SAS is valid: either "blob" or "file". Currently AzureStor does not support creating a service SAS for queue or table storage. +#' @param service For a service SAS, the storage service for which the SAS is valid: either "blob" or "file". Currently AzureStor does not support creating a service SAS for queue or table storage. #' @param resource_type For a service SAS, the type of resource for which the SAS is valid. For blob storage, the default value is "b" meaning a single blob. For file storage, the default value is "f" meaning a single file. Other possible values include "bs" (a blob snapshot), "c" (a blob container), or "s" (a file share). #' @param policy For a service SAS, optionally the name of a stored access policy to correlate the SAS with. Revoking the policy will also invalidate the SAS. -#' @param snapshot_time For a service SAS, the specific blob snapshot for which the SAS is valid, if `resource_type` is "bs". #' @rdname sas #' @export -get_service_sas.default <- function(account, resource, key, service_type, resource_type=NULL, start=NULL, expiry=NULL, - permissions="rl", ip=NULL, protocol=NULL, policy=NULL, snapshot_time=NULL, +get_service_sas.default <- function(account, resource, key, service, start=NULL, expiry=NULL, permissions="rl", + resource_type=NULL, ip=NULL, protocol=NULL, policy=NULL, snapshot_time=NULL, auth_api_version=getOption("azure_storage_api_version"), ...) { - if(!(service_type %in% c("blob", "file"))) + if(!(service %in% c("blob", "file"))) stop("Generating a service SAS currently only supported for blob and file storage", call.=FALSE) dates <- make_sas_dates(start, expiry) if(is.null(resource_type)) - resource_type <- if(service_type == "blob") "b" else "f" - resource <- file.path("", service_type, account, resource) # canonicalized resource starts with / + resource_type <- if(service == "blob") "b" else "f" + resource <- file.path("", service, account, resource) # canonicalized resource starts with / sig_str <- paste( permissions, diff --git a/man/az_storage.Rd b/man/az_storage.Rd index ac2f810..e2bd185 100644 --- a/man/az_storage.Rd +++ b/man/az_storage.Rd @@ -32,10 +32,16 @@ Initializing a new object of this class can either retrieve an existing storage Note that you don't need to worry about this section if you have been \emph{given} a SAS, and only want to use it to access storage. -AzureStor supports generating two kinds of SAS: account and user delegation, with the latter applying only to blob and ADLS2 storage. To create an account SAS, call the \code{get_account_sas()} method. This has the following signature:\preformatted{get_account_sas(key=self$list_keys()[1], start=NULL, expiry=NULL, services="bqtf", permissions="rl", +AzureStor supports generating three kinds of SAS: account, service and user delegation. An account SAS can be used with any type of storage. A service SAS can be used with blob and file storage, whle a user delegation SAS can be used with blob and ADLS2 storage. + +To create an account SAS, call the \code{get_account_sas()} method. This has the following signature:\preformatted{get_account_sas(key=self$list_keys()[1], start=NULL, expiry=NULL, services="bqtf", permissions="rl", resource_types="sco", ip=NULL, protocol=NULL) } +To create a service SAS, call the \code{get_service_sas()} method, which has the following signature:\preformatted{get_service_sas(key=self$list_keys()[1], resource, service, start=NULL, expiry=NULL, permissions="r", + resource_type=NULL, ip=NULL, protocol=NULL, policy=NULL, snapshot_time=NULL) +} + To create a user delegation SAS, you must first create a user delegation \emph{key}. This takes the place of the account's access key in generating the SAS. The \code{get_user_delegation_key()} method has the following signature:\preformatted{get_user_delegation_key(token=self$token, key_start=NULL, key_expiry=NULL) } @@ -43,6 +49,8 @@ Once you have a user delegation key, you can use it to obtain a user delegation resource_types="c", ip=NULL, protocol=NULL, snapshot_time=NULL) } +(Note that the \code{key} argument for this method is the user delegation key, \emph{not} the account key.) + To invalidate all user delegation keys, as well as the SAS's generated with them, call the \code{revoke_user_delegation_keys()} method. This has the following signature:\preformatted{revoke_user_delegation_keys() } @@ -80,9 +88,12 @@ stor$get_file_endpoint() } \seealso{ \link{blob_endpoint}, \link{file_endpoint}, -\link{create_storage_account}, \link{get_storage_account}, \link{delete_storage_account}, \link{Date}, \link{POSIXt}, +\link{create_storage_account}, \link{get_storage_account}, \link{delete_storage_account}, \link{Date}, \link{POSIXt} + \href{https://docs.microsoft.com/en-us/rest/api/storagerp/}{Azure Storage Provider API reference}, -\href{https://docs.microsoft.com/en-us/rest/api/storageservices/}{Azure Storage Services API reference}, +\href{https://docs.microsoft.com/en-us/rest/api/storageservices/}{Azure Storage Services API reference} + \href{https://docs.microsoft.com/en-us/rest/api/storageservices/create-account-sas}{Create an account SAS}, -\href{https://docs.microsoft.com/en-us/rest/api/storageservices/create-user-delegation-sas}{Create a user delegation SAS} +\href{https://docs.microsoft.com/en-us/rest/api/storageservices/create-user-delegation-sas}{Create a user delegation SAS}, +\href{https://docs.microsoft.com/en-us/rest/api/storageservices/create-service-sas}{Create a service SAS} } diff --git a/man/sas.Rd b/man/sas.Rd index f99a0ee..bc0f951 100644 --- a/man/sas.Rd +++ b/man/sas.Rd @@ -59,14 +59,14 @@ get_user_delegation_sas(account, ...) get_service_sas(account, ...) -\method{get_service_sas}{az_storage}(account, resource, - service_type = c("blob", "file"), key = account$list_keys()[1], ...) +\method{get_service_sas}{az_storage}(account, resource, service = c("blob", + "file"), key = account$list_keys()[1], ...) \method{get_service_sas}{storage_endpoint}(account, resource, key = account$key, ...) -\method{get_service_sas}{default}(account, resource, key, service_type, - resource_type = NULL, start = NULL, expiry = NULL, - permissions = "rl", ip = NULL, protocol = NULL, policy = NULL, +\method{get_service_sas}{default}(account, resource, key, service, + start = NULL, expiry = NULL, permissions = "rl", + resource_type = NULL, ip = NULL, protocol = NULL, policy = NULL, snapshot_time = NULL, auth_api_version = getOption("azure_storage_api_version"), ...) } @@ -81,7 +81,7 @@ get_service_sas(account, ...) \item{services}{For \code{get_account_sas}, the storage service(s) for which the SAS is valid. Defaults to \code{bqtf}, meaning blob (including ADLS2), queue, table and file storage.} -\item{permissions}{For \code{get_account_sas} and \code{get_user_delegation_sas}, the permissions that the SAS grants. The default \code{rl} (read and list) essentially means read-only access.} +\item{permissions}{The permissions that the SAS grants. The default value of \code{rl} (read and list) essentially means read-only access.} \item{resource_types}{For an account or user delegation SAS, the resource types for which the SAS is valid. For \code{get_account_sas} the default is \code{sco} meaning service, container and object. For \code{get_user_delegation_sas} the default is \code{c} meaning container-level access (including blobs within the container).} @@ -97,9 +97,9 @@ get_service_sas(account, ...) \item{resource}{For \code{get_user_delegation_sas} and \code{get_service_sas}, the resource for which the SAS is valid. For a user delegation SAS, this can be either the name of a blob container or an individual blob; if the latter, it should include the container as well (\code{containername/blobname}). A service SAS is similar but can also be used with file shares and files.} -\item{snapshot_time}{For a service SAS, the specific blob snapshot for which the SAS is valid, if \code{resource_type} is "bs".} +\item{snapshot_time}{For a user delegation or service SAS, the blob snapshot for which the SAS is valid. Only required if \code{resource_type[s]="bs"}.} -\item{service_type}{For a service SAS, the storage service for which the SAS is valid: either "blob" or "file". Currently AzureStor does not support creating a service SAS for queue or table storage.} +\item{service}{For a service SAS, the storage service for which the SAS is valid: either "blob" or "file". Currently AzureStor does not support creating a service SAS for queue or table storage.} \item{resource_type}{For a service SAS, the type of resource for which the SAS is valid. For blob storage, the default value is "b" meaning a single blob. For file storage, the default value is "f" meaning a single file. Other possible values include "bs" (a blob snapshot), "c" (a blob container), or "s" (a file share).} @@ -115,6 +115,8 @@ Note that you don't need to worry about these methods if you have been \emph{giv An \strong{account SAS} is secured with the storage account key. An account SAS delegates access to resources in one or more of the storage services. All of the operations available via a user delegation SAS are also available via an account SAS. You can also delegate access to read, write, and delete operations on blob containers, tables, queues, and file shares. To obtain an account SAS, call \code{get_account_sas}. +A \strong{service SAS} is like an account SAS, but allows finer-grained control of access. You can create a service SAS that allows access only to specific blobs in a container, or files in a file share. To obtain a service SAS, call \code{get_service_sas}. + A \strong{user delegation SAS} is a SAS secured with Azure AD credentials. It's recommended that you use Azure AD credentials when possible as a security best practice, rather than using the account key, which can be more easily compromised. When your application design requires shared access signatures, use Azure AD credentials to create a user delegation SAS for superior security. Every SAS is signed with a key. To create a user delegation SAS, you must first request a \strong{user delegation key}, which is then used to sign the SAS. The user delegation key is analogous to the account key used to sign a service SAS or an account SAS, except that it relies on your Azure AD credentials. To request the user delegation key, call \code{get_user_delegation_key}. With the user delegation key, you can then create the SAS with \code{get_user_delegation_sas}. @@ -140,6 +142,15 @@ get_account_sas("mystorage", "access_key", services="f", resource_types="o") endp <- storage_endpoint("https://mystorage.blob.core.windows.net", key="access_key") get_account_sas(endp, permissions="rwcd") +# service SAS for a container with read-only permissions +get_service_sas(bl, "containername") + +# service SAS for a container, allowing listing of blobs +get_service_sas(bl, "containername", permissions="rl") + +# read/write service SAS for a blob +get_service_sas(endp, "containername/blobname", permissions="rw") + \dontrun{ # user delegation key valid for 24 hours @@ -159,10 +170,12 @@ get_user_delegation_sas(endp, userkey, resource="mycontainer/myfile", } \seealso{ \link{blob_endpoint}, \link{file_endpoint}, -\link{Date}, \link{POSIXt}, +\link{Date}, \link{POSIXt} + \href{https://docs.microsoft.com/en-us/rest/api/storagerp/}{Azure Storage Provider API reference}, -\href{https://docs.microsoft.com/en-us/rest/api/storageservices/}{Azure Storage Services API reference}, +\href{https://docs.microsoft.com/en-us/rest/api/storageservices/}{Azure Storage Services API reference} + \href{https://docs.microsoft.com/en-us/rest/api/storageservices/create-account-sas}{Create an account SAS}, -\href{https://docs.microsoft.com/en-us/rest/api/storageservices/create-user-delegation-sas}{Create a user delegation SAS} +\href{https://docs.microsoft.com/en-us/rest/api/storageservices/create-user-delegation-sas}{Create a user delegation SAS}, \href{https://docs.microsoft.com/en-us/rest/api/storageservices/create-service-sas}{Create a service SAS} } From b168b1af262939ea3cc14f5d486ff45fe746095e Mon Sep 17 00:00:00 2001 From: Hong Ooi Date: Sun, 9 May 2021 16:23:06 +1000 Subject: [PATCH 4/8] cleanup news --- NEWS.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/NEWS.md b/NEWS.md index 66dfb81..f703dde 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,7 +1,6 @@ # AzureStor 3.4.1.9000 -- Add support for generating a service SAS. There is a new S3 generic `get_service_sas` with methods for `az_storage` and `storage_endpoint` objects, and a similar R6 method for `az_storage` objects. - - See `?sas` for more information. +- Add support for generating a service SAS. There is a new S3 generic `get_service_sas` with methods for `az_storage` and `storage_endpoint` objects, and a similar R6 method for `az_storage` objects. See `?sas` for more information. - Fix `storage_save_rds` and `storage_load_rds` to handle compression correctly. In particular, `storage_load_rds` should now correctly load files saved with `saveRDS` (#83). - Fix a bug that caused `list_blobs` to fail when leases were present. - Use a raw connection instead of a raw vector when calling `readr::read_delim` and `read_csv2`. This works around an issue introduced in readr 1.4.0 (#85, #86). From 8ced1b9d7b55ea52e08d765880ac42d76c9262e2 Mon Sep 17 00:00:00 2001 From: Hong Ooi Date: Sun, 9 May 2021 16:27:01 +1000 Subject: [PATCH 5/8] fix example --- R/sas.R | 7 ++----- man/sas.Rd | 7 ++----- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/R/sas.R b/R/sas.R index 4b3bd5f..5281522 100644 --- a/R/sas.R +++ b/R/sas.R @@ -61,11 +61,8 @@ #' endp <- storage_endpoint("https://mystorage.blob.core.windows.net", key="access_key") #' get_account_sas(endp, permissions="rwcd") #' -#' # service SAS for a container with read-only permissions -#' get_service_sas(bl, "containername") -#' -#' # service SAS for a container, allowing listing of blobs -#' get_service_sas(bl, "containername", permissions="rl") +#' # service SAS for a container +#' get_service_sas(endp, "containername") #' #' # read/write service SAS for a blob #' get_service_sas(endp, "containername/blobname", permissions="rw") diff --git a/man/sas.Rd b/man/sas.Rd index bc0f951..a80c0f6 100644 --- a/man/sas.Rd +++ b/man/sas.Rd @@ -142,11 +142,8 @@ get_account_sas("mystorage", "access_key", services="f", resource_types="o") endp <- storage_endpoint("https://mystorage.blob.core.windows.net", key="access_key") get_account_sas(endp, permissions="rwcd") -# service SAS for a container with read-only permissions -get_service_sas(bl, "containername") - -# service SAS for a container, allowing listing of blobs -get_service_sas(bl, "containername", permissions="rl") +# service SAS for a container +get_service_sas(endp, "containername") # read/write service SAS for a blob get_service_sas(endp, "containername/blobname", permissions="rw") From b38452bb12d8af21d5c3db9a9b94df6fbee606fa Mon Sep 17 00:00:00 2001 From: Hong Ooi Date: Sun, 9 May 2021 20:51:32 +1000 Subject: [PATCH 6/8] testing 1 --- tests/testthat/test10a_service_sas.R | 70 ++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 tests/testthat/test10a_service_sas.R diff --git a/tests/testthat/test10a_service_sas.R b/tests/testthat/test10a_service_sas.R new file mode 100644 index 0000000..732188a --- /dev/null +++ b/tests/testthat/test10a_service_sas.R @@ -0,0 +1,70 @@ +context("Account and user SAS") + +tenant <- Sys.getenv("AZ_TEST_TENANT_ID") +app <- Sys.getenv("AZ_TEST_APP_ID") +#cliapp <- Sys.getenv("AZ_TEST_NATIVE_APP_ID") +password <- Sys.getenv("AZ_TEST_PASSWORD") +subscription <- Sys.getenv("AZ_TEST_SUBSCRIPTION") + +if(tenant == "" || app == "" || password == "" || subscription == "") + skip("SAS tests skipped: ARM credentials not set") + +rgname <- Sys.getenv("AZ_TEST_STORAGE_RG") +storname <- Sys.getenv("AZ_TEST_STORAGE_HNS") + +if(rgname == "" || storname == "") + skip("SAS tests skipped: resource names not set") + +sub <- AzureRMR::az_rm$new(tenant=tenant, app=app, password=password)$get_subscription(subscription) +stor <- sub$get_resource_group(rgname)$get_storage_account(storname) +options(azure_storage_progress_bar=FALSE) + +dates <- c(Sys.Date() - 1, Sys.Date() + 5) +token <- AzureRMR::get_azure_token("https://storage.azure.com", tenant, app=app, password=password) + +bl0 <- stor$get_blob_endpoint() +fl0 <- stor$get_file_endpoint() + +test_that("Service SAS works 0", +{ + contname <- make_name() + key <- stor$list_keys()[1] + + bsas <- get_service_sas(storname, contname, key, service="blob", permissions="rcwl", resource_type="c") + bl <- stor$get_blob_endpoint(key=NULL, sas=bsas) + expect_silent(cont <- create_storage_container(bl0, contname)) + expect_silent(storage_upload(cont, "../resources/iris.csv")) + + fsas <- get_service_sas(storname, contname, key, service="file", permissions="rcwl", resource_type="s") + fl <- stor$get_file_endpoint(key=NULL, sas=fsas) + expect_silent(share <- create_storage_container(fl0, contname)) + expect_silent(storage_upload(share, "../resources/iris.csv")) +}) + + +test_that("Service SAS works 1", +{ + contname <- make_name() + + bsas <- stor$get_service_sas(resource=contname, service="blob", permissions="rcwl", resource_type="c") + bl <- stor$get_blob_endpoint(key=NULL, sas=bsas) + expect_silent(cont <- create_storage_container(bl0, contname)) + expect_silent(storage_upload(cont, "../resources/iris.csv")) + + fsas <- stor$get_service_sas(resource=contname, service="file", permissions="rcwl", resource_type="s") + fl <- stor$get_file_endpoint(key=NULL, sas=fsas) + expect_silent(share <- create_storage_container(fl0, contname)) + expect_silent(storage_upload(share, "../resources/iris.csv")) +}) + + +teardown({ + bl <- stor$get_blob_endpoint() + blconts <- list_storage_containers(bl) + lapply(blconts, delete_storage_container, confirm=FALSE) + + fl <- stor$get_file_endpoint() + flshares <- list_storage_containers(fl) + lapply(flshares, delete_storage_container, confirm=FALSE) +}) + From fb19f226836b9b606b7a101bd46f8ec88dc27025 Mon Sep 17 00:00:00 2001 From: Hong Ooi Date: Sun, 9 May 2021 20:54:07 +1000 Subject: [PATCH 7/8] testing 2 --- tests/testthat/test10a_service_sas.R | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/testthat/test10a_service_sas.R b/tests/testthat/test10a_service_sas.R index 732188a..0a2e2c7 100644 --- a/tests/testthat/test10a_service_sas.R +++ b/tests/testthat/test10a_service_sas.R @@ -58,6 +58,22 @@ test_that("Service SAS works 1", }) +test_that("Service SAS works 2", +{ + contname <- make_name() + + bsas <- get_service_sas(bl0, resource=contname, permissions="rcwl", resource_type="c") + bl <- stor$get_blob_endpoint(key=NULL, sas=bsas) + expect_silent(cont <- create_storage_container(bl0, contname)) + expect_silent(storage_upload(cont, "../resources/iris.csv")) + + fsas <- get_service_sas(fl0, resource=contname, permissions="rcwl", resource_type="s") + fl <- stor$get_file_endpoint(key=NULL, sas=fsas) + expect_silent(share <- create_storage_container(fl0, contname)) + expect_silent(storage_upload(share, "../resources/iris.csv")) +}) + + teardown({ bl <- stor$get_blob_endpoint() blconts <- list_storage_containers(bl) From 7115992b8eea9306ed9fa42d5bfcc232aef397f7 Mon Sep 17 00:00:00 2001 From: Hong Ooi Date: Sun, 9 May 2021 21:31:58 +1000 Subject: [PATCH 8/8] rm cruft --- tests/testthat/test10a_service_sas.R | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/testthat/test10a_service_sas.R b/tests/testthat/test10a_service_sas.R index 0a2e2c7..03a5609 100644 --- a/tests/testthat/test10a_service_sas.R +++ b/tests/testthat/test10a_service_sas.R @@ -19,9 +19,6 @@ sub <- AzureRMR::az_rm$new(tenant=tenant, app=app, password=password)$get_subscr stor <- sub$get_resource_group(rgname)$get_storage_account(storname) options(azure_storage_progress_bar=FALSE) -dates <- c(Sys.Date() - 1, Sys.Date() + 5) -token <- AzureRMR::get_azure_token("https://storage.azure.com", tenant, app=app, password=password) - bl0 <- stor$get_blob_endpoint() fl0 <- stor$get_file_endpoint()