Promote changes from private utility project into FSharp.Lu

- Add new Microsoft.FSharpLu.Azure module, upgraded to latest version of Azure nugets and migrate from WindowsAzure.Storage to Microsoft.Azure.Storage.
Includes number of F# helpers to access Azure Compute, Storage, Network, ARM API
- Add HttpCommunication module, a strongly-typed HTTP client thin library
- Add support for environment variable and error output capture in `startProcess` API
- Add Knuth-Morris-Pratt implementation for searching a string in a file stream
- Add state machine and agent module used to implement long-running operations in services
- Add Windows/Linux platform helpers
- Adds the Combine functor for the TraceTags statically-typed global logger
- Many other utility functions added: SMB, sysprep, parsing, shutdown, EXE type, IDisposable helpers, Compression, certificate, CacheMap colletion...
- Bump up version to 0.11.0
- Hardening of startProcess API to handle corner cases (e.g. process ends too quickly or too slowly)

c

.gitattributes

@@ -1 +1,2 @@
-*.fs text eol=input
+*.fs text eol=input
+/FSharpLu.Tests/*.ini text eol=crlf

.gitignore

@@ -127,4 +127,5 @@ ModelManifest.xml
 .nuget/CredentialProvider.VSS.exe
 
 # Scripts for FSI
-FSharpLu.Json/Scripts
+FSharpLu.Json/Scripts
+/.ionide/symbolCache.db

CI-vsts.yml

@@ -15,7 +15,7 @@ queue:
 
 variables:
   # Increase version number before every official release (if PublishRelease is set to true)
-  NUGET_PACKAGE_VERSION: 0.10.33
+  NUGET_PACKAGE_VERSION: 0.11.0
 
 steps:
 - task: DotNetCoreInstaller@0

FSharpLu.Azure.Test/Azure.Test.Keyvault.fs

@@ -0,0 +1,41 @@
+[<Xunit.Trait("TestCategory", "Keyvault")>]
+module Microsoft.FSharpLu.Azure.Test.Keyvault
+
+open System
+open Microsoft.FSharpLu.Azure.Vault
+open Xunit
+
+[<Fact>]
+let ``Parse keyvault URL`` () =
+    let x, y, z = parseKeyvaultSecretUrl "https://foo.vault.azure.net:443/secrets/bla/13564s6df"
+    Assert.Equal(x, "https://foo.vault.azure.net:443/")
+    Assert.Equal(y, "bla")
+    Assert.Equal(z, "13564s6df")
+
+[<Fact>]
+let ``Parse keyvault URL 2`` () =
+    let x, y, z = parseKeyvaultSecretUrl "https://foo.vault.azure.net:443/secrets/bla/13564s6df/"
+    Assert.Equal(x, "https://foo.vault.azure.net:443/")
+    Assert.Equal(y, "bla")
+    Assert.Equal(z, "13564s6df")
+
+[<Fact>]
+let ``Parse keyvault URL without version`` () =
+    let x, y, z = parseKeyvaultSecretUrl "https://bar.vault.azure.net:443/secrets/foo"
+    Assert.Equal(x, "https://bar.vault.azure.net:443/")
+    Assert.Equal(y, "foo")
+    Assert.Equal(z, "")
+
+[<Fact>]
+let ``Parse keyvault URL without version 2`` () =
+    let x, y, z = parseKeyvaultSecretUrl "https://bar.vault.azure.net:443/secrets/foo/"
+    Assert.Equal(x, "https://bar.vault.azure.net:443/")
+    Assert.Equal(y, "foo")
+    Assert.Equal(z, "")
+
+[<Fact>]
+let ``Throws when parsing incorrect keyvault URLs`` () =
+    let _ = Assert.Throws(Action(fun () -> parseKeyvaultSecretUrl "https://bar.vault.azure.net:443/secretsasd/foo/" |> ignore))
+    let _ = Assert.Throws(Action(fun () -> parseKeyvaultSecretUrl "https://bar.vault.azure.net:443/secretsasdfoosdf" |> ignore))
+    let _ = Assert.Throws(Action(fun () -> parseKeyvaultSecretUrl "https://bar.vault.azure.net:443/secrets/1/2/345" |> ignore))
+    ()

FSharpLu.Azure.Test/Azure.Test.Queue.fs

@@ -0,0 +1,59 @@
+namespace Microsoft.FSharpLu.Azure.Test.Queue
+
+open System
+open Microsoft.Azure.Storage
+open Microsoft.Azure.Storage.Queue
+open Microsoft.FSharpLu.Azure.Queue
+open Microsoft.FSharpLu.Async
+
+[<AutoOpen>]
+module Tests =
+
+    let validateHardCodedMaximumVisibilityTimeout(storageAccount: string, storageKey: string) =
+        async {
+            let auth = Auth.StorageCredentials(storageAccount, storageKey)
+            let storage = CloudStorageAccount(auth, true)
+
+            let queueClient = storage.CreateCloudQueueClient()
+
+            let queueName = "maxvisibilitytest"
+
+            let q = queueClient.GetQueueReference(queueName)
+            let! exists = q.ExistsAsync().AsAsync
+            if exists then
+                failwithf "Queue %s already exists, use a different queue name" queueName
+            let! _ = q.CreateIfNotExistsAsync().AsAsync
+
+            let msg = box "wubalubadubdub"
+            let rec validate (t: TimeSpan) =
+                async{
+                    try
+                        do! schedulePostMessage q msg t
+                        printfn "Scheduled with visibility: %O" t
+                        return! validate (t.Add(TimeSpan.FromDays 1.0))
+                    with
+                    e ->
+                        printfn "Failed to schedule with visibility: %O" t
+                        try
+                            let t = t.Subtract(TimeSpan.FromDays 1.0)
+                            // expected to succeed
+                            do! schedulePostMessage q msg t
+                            return t
+                        with e ->
+                            return failwithf "Failed to schedule post with smaller timespan %O" t
+
+                }
+
+            let! c =
+                Async.Catch(
+                    async {
+                        let! t = 1.0 |> TimeSpan.FromDays |> validate
+                        if t <> AzureHardCodedMaximumVisibilityTimeout then
+                            failwithf "Expected maximum visibility timeout to be: %O but got: %O" AzureHardCodedMaximumVisibilityTimeout t
+                    })
+            do! q.DeleteAsync().AsAsync
+
+            match c with
+            | Choice1Of2 () -> ()
+            | Choice2Of2 e -> reraise e
+        }

FSharpLu.Azure.Test/Azure.Test.Storage.fs

@@ -0,0 +1,137 @@
+namespace Microsoft.FSharpLu.Azure.Test.Storage
+
+open System
+open FsCheck
+open Xunit
+
+module Tests =
+    open System.Collections.Generic
+    open Microsoft.FSharpLu.Azure.Storage.Impl
+    open Microsoft.FSharpLu.Azure.Test.Generators
+    open Microsoft.Azure.Management.Storage
+
+    let accountsByTag (listResourcesWithTag:ListResourcesWithTag<ListResources>) azure resourceGroupName =
+        let withTag, tagged =
+            async{
+                let (ListResourcesWithTag (listByResourceGroup, (tagKey, tagValue), tagged)) = listResourcesWithTag
+
+                let! withTag =
+                    getStorageAccountsByTag listByResourceGroup azure resourceGroupName (tagKey, tagValue)
+
+                return withTag, tagged
+            } |> Async.RunSynchronously
+
+        (fun () -> Seq.length withTag = Seq.length tagged)
+        |> Prop.classify (Seq.isEmpty withTag) "Empty"
+        |> Prop.classify (Seq.length withTag = Seq.length tagged) "Same length"
+        |> Prop.classify (Seq.length withTag <> Seq.length tagged) "Lengths do not match"
+
+
+    let endpointsByTag (listResourcesWithTag:ListResourcesWithTag<ListResources>) azure resourceGroupName =
+        let endpointsWithTag, tagged, sameEndpoints =
+            async{
+                let (ListResourcesWithTag (listByResourceGroup, (tagKey, tagValue), tagged)) = listResourcesWithTag
+
+                let! endpointsWithTag =
+                    getStorageEnpointsByTag listByResourceGroup azure resourceGroupName (tagKey, tagValue)
+
+                let sameEndpoints = (tagged, endpointsWithTag)
+                                    ||> Seq.forall2 (fun acc (name, _, ep) -> acc.Name = name && acc.PrimaryEndpoints = ep)
+
+                return endpointsWithTag, tagged, sameEndpoints
+            } |> Async.RunSynchronously
+        (fun () ->
+            Seq.length endpointsWithTag = Seq.length tagged && sameEndpoints)
+        |> Prop.classify (Seq.isEmpty endpointsWithTag) "Empty"
+        |> Prop.classify (sameEndpoints) "Same Endpoints"
+        |> Prop.classify (not sameEndpoints) "Endpoints do not match"
+        |> Prop.classify (Seq.length endpointsWithTag = Seq.length tagged) "Same length"
+        |> Prop.classify (Seq.length endpointsWithTag <> Seq.length tagged) "Different lengths"
+
+
+    let accountsByKeys (resources: Resources<ListResources * Models.StorageAccount[] * Dictionary<string, Models.StorageAccountListKeysResult>>) azure resourceGroupName =
+        let sameResourceGroup, xs, accounts, sameKey1s =
+            async {
+                let (Resources (listResources, accounts, keys)) = resources
+
+                let! xs = getAllAccountsWithKeys listResources azure resourceGroupName
+                let sameResourceGroup = xs |> Seq.forall(fun (g, _, _, _) -> g = resourceGroupName)
+                let sameKey1s = xs |> Seq.forall(fun (_, name, _, key) -> keys.[name].Keys.[0].Value = key)
+                return sameResourceGroup, xs, accounts, sameKey1s
+            } |> Async.RunSynchronously
+
+        (fun () -> Seq.length xs = Seq.length accounts && sameResourceGroup && sameKey1s)
+        |> Prop.classify(Seq.isEmpty xs) "Empty"
+        |> Prop.classify(Seq.length xs = Seq.length accounts) "Same lengths"
+        |> Prop.classify(sameResourceGroup) "Same resource group"
+        |> Prop.classify(sameKey1s) "Same keys"
+        |> Prop.classify(Seq.length xs <> Seq.length accounts) "Lengths do not match"
+        |> Prop.classify(not sameResourceGroup) "Resource Group does not match"
+        |> Prop.classify(not sameKey1s) "Keys do not match"
+
+
+    let endpointByName (resources: Resources<ListResources * Models.StorageAccount[] * Dictionary<string, Models.StorageAccountListKeysResult> * string Set>) azure resourceGroupName =
+        let (Resources (listResources, accounts, keys, names)) = resources
+        let name = if Set.isEmpty names then String.Empty else Seq.head names
+        let x, y =
+            async {
+                let! s, listKeys = listResources azure resourceGroupName
+                let! xs = tryGetStorageEndpointByName listResources azure resourceGroupName name
+                return s |> Seq.tryFind(fun x -> x.Name = name), xs
+            } |> Async.RunSynchronously
+        (fun () ->
+            match x, y with
+            | None, None | Some _, Some _ -> true
+            | Some _, None | None, Some _ -> false)
+        |> Prop.classify (match x, y with None, None -> true | _ -> false) "Expected not to find"
+        |> Prop.classify (match x, y with Some n, Some(m, _, _) when n.Name = m -> true | _ -> false) "Found"
+        |> Prop.classify (match x, y with Some n, Some(m, _, _) when n.Name <> m -> true | _ -> false) "Found wrong one"
+        |> Prop.classify (match x, y with Some _, None -> true | _ -> false) "Not found when expected to find"
+        |> Prop.classify (match x, y with None, Some _ -> true | _ -> false) "Found when not supposed to find"
+
+    /// TODO This test is broken. It had a bug that made it pass, the fix breaks the test. disabling it for now.
+    let mapEndpointsByName (resources: Resources<ListResources * Models.StorageAccount[] * Dictionary<string, Models.StorageAccountListKeysResult> * string Set>) azure resourceGroupName =
+        let names, map =
+            async {
+                let (Resources (listResources, accounts, keys, names)) = resources
+                let! s, listKeys = listResources azure resourceGroupName
+                let! xs = mapStorageEndpointsByName listResources azure resourceGroupName names
+                return names, xs
+            } |> Async.RunSynchronously
+        (fun () ->
+            names.Count = map.Count &&
+            (Seq.forall(fun name ->
+                (Map.tryFind name map |> Option.bind id).IsSome) names)
+                )
+        |> Prop.classify (names.IsEmpty) "Empty"
+        |> Prop.classify (names.Count > map.Count) "Final map does not have all the names"
+        |> Prop.classify (names.Count < map.Count) "Final map has more names than supplied"
+        |> Prop.classify (names.Count = map.Count) "Same length"
+
+
+open Tests
+
+[<Trait("TestCategory", "Azure Storage")>]
+type AzureStorageTests () =
+    let generatorRegistration = Arb.register<Microsoft.FSharpLu.Azure.Test.Generators.Generators>()
+
+    [<Fact>]
+    member __.StorageAccountsWithKeys() =
+        Check.QuickThrowOnFailure accountsByKeys
+
+    [<Fact>]
+    member x.StorageAccountsByTag() =
+        Check.QuickThrowOnFailure accountsByTag
+
+    [<Fact>]
+    member x.StorageEndpointsByTag() =
+        Check.QuickThrowOnFailure endpointsByTag
+
+    [<Fact>]
+    member x.StorageEndpointByName() =
+        Check.QuickThrowOnFailure endpointByName
+
+    /// TODO: 2680 Broken test
+    // [<Fact>]
+    //member __.MapStorageEndpointsByName() =
+    //    Check.QuickThrowOnFailure mapEndpointsByName

FSharpLu.Azure.Test/CommunicationTests.fs

@@ -0,0 +1,116 @@
+namespace Microsoft.FSharpLu.HttpCommunication.Test
+
+open Microsoft.VisualStudio.TestTools.UnitTesting
+open System
+open Microsoft.FSharpLu.HttpCommunication
+
+[<AutoOpen>]
+module CommunicationTests =
+
+    let randomSeed = (new Random()).Next()
+    let random = new Random(randomSeed)
+    let testResponseText = "a response"
+
+    let getOpenPort startPort endPort =
+        let properties = System.Net.NetworkInformation.IPGlobalProperties.GetIPGlobalProperties()
+        let usedPortsWithinRange =
+            properties.GetActiveTcpListeners()
+            |> Seq.map (fun p -> p.Port)
+            |> Seq.filter (fun p -> p >= startPort && p <= endPort)
+            |> Set.ofSeq
+        let openPort = set [startPort .. endPort] - usedPortsWithinRange |> Seq.tryHead
+        match openPort with
+        | Some port -> port
+        | None -> failwith (sprintf "Error getting an open port, all ports within range from %i to %i are busy. Try increasing the range." startPort endPort)
+
+    let rec startListener cancellationTokenSource =
+        try
+            let serverUri = Uri (sprintf "http://localhost:%d" (getOpenPort 8980 9000))
+            Server.listener serverUri cancellationTokenSource (fun req resp ->
+                async {
+                    printfn "Listener start on %s" serverUri.AbsoluteUri
+                    let relativeUri = Server.getRelativeUri serverUri req
+
+                    printfn "URI: %s" relativeUri
+                    let statusCode =
+                        match relativeUri with
+                        | "SuccessTest" ->
+                            async {
+                                printfn "Running success test."
+                                let messageText = Server.getRequestContent req
+
+                                match messageText with
+                                | Ok messageText ->
+                                    printfn "Found message: %s" messageText
+                                    return System.Net.HttpStatusCode.OK
+                                | Error exn ->
+                                    printfn "Error reading request content: %O" exn
+                                    return System.Net.HttpStatusCode.BadRequest
+                            } |> Async.RunSynchronously
+                        | unknownUri ->
+                            printfn "Unknown request type: %A" unknownUri
+                            System.Net.HttpStatusCode.BadRequest
+
+                    printfn "Returning code: %A" statusCode
+
+                    let dataToReturn = if random.NextDouble() > 0.5 then None else (Some testResponseText)
+                    Server.sendResponse resp statusCode dataToReturn
+                }), serverUri
+        with
+        | :? System.Net.HttpListenerException as e ->
+            printfn "An exception has occurred: %s. Retrying start of the listener." e.Message
+            startListener cancellationTokenSource
+
+    let sendRequest serverUri requestContent relativeUri =
+        Client.sendRequest serverUri requestContent relativeUri
+
+[<TestClass>]
+type UtilitiesCommunicationTests() =
+
+    static let cancellationTokenSource = new System.Threading.CancellationTokenSource()
+    static let httpListener, serverUri = startListener cancellationTokenSource
+
+    [<ClassInitialize>]
+    static member init(context : TestContext) =
+        printfn "Using random seed %d" randomSeed
+
+    [<ClassCleanup>]
+    static member cleanup() =
+        (httpListener :> IDisposable).Dispose()
+
+    [<TestMethod>]
+    [<Description("Test 'ServerRequestSuccess' behavior")>]
+    [<TestCategory("Utilities")>]
+    member this.ServerRequestSuccess() =
+
+        let response = sendRequest serverUri "hello" "SuccessTest"
+        printfn "Received response text: %s" response
+        if not (String.IsNullOrEmpty response || response.Equals(testResponseText)) then
+            invalidOp (sprintf "test failed, response text %s does not match expected %s" response testResponseText)
+
+    [<TestMethod>]
+    [<Description("Test 'ServerRequestError' behavior")>]
+    [<TestCategory("Utilities")>]
+    member this.ServerRequestError() =
+        let requestSucceeded =
+            try
+                sendRequest serverUri "hello" "NonExistentPath" |> ignore
+                true
+            with
+                e ->
+                    printfn "Got %s" e.Message
+                    false
+        if requestSucceeded then
+            invalidOp "test failed, request should fail"
+
+
+    [<TestMethod>]
+    [<Description("Ping Google and Bing")>]
+    [<TestCategory("Utilities")>]
+    member this.InternetAccess() =
+        let urlsToPing = ["http://www.google.com"; "http://www.bing.com"]
+        let access = urlsToPing |> Utils.hasInternetAccess |> Async.Parallel |> Async.RunSynchronously
+        if access |> Array.exists (fun x -> match x with Utils.InternetAccessResult.Access _ -> false | _ -> true) then
+            List.zip urlsToPing (List.ofArray access)
+            |> invalidOp "Failed to ping at least one URL: %A"
+