Skip to content

Commit

Permalink
Add modules to deployment template & use edge util logging (Azure#123)
Browse files Browse the repository at this point in the history 
* Add analyzer and snitch modules to deployment template & use edge util logging

* Added 2 more loadgen modules

* Address review comments
  • Loading branch information
@ancaantochi
ancaantochi authored Aug 5, 2018
1 parent 02f193a commit cd1fa09
Show file tree
Hide file tree
Showing 5 changed files with 140 additions and 24 deletions.
8 changes: 4 additions & 4 deletions edge-modules/MessagesAnalyzer/MessagesAnalyzer.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,10 +24,10 @@
<PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="2.1.1" />
<PackageReference Include="Microsoft.Extensions.Configuration.FileExtensions" Version="2.1.1" />
<PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="2.1.1" />
<PackageReference Include="Microsoft.Extensions.Logging" Version="2.1.1" />
<PackageReference Include="Serilog" Version="2.7.1" />
<PackageReference Include="Serilog.Extensions.Logging" Version="2.0.2" />
<PackageReference Include="Serilog.Sinks.Console" Version="3.1.1" />
</ItemGroup>

<ItemGroup>
<ProjectReference Include="..\..\edge-util\src\Microsoft.Azure.Devices.Edge.Util\Microsoft.Azure.Devices.Edge.Util.csproj" />
</ItemGroup>

<ItemGroup>
Expand Down
9 changes: 5 additions & 4 deletions edge-modules/MessagesAnalyzer/PartitionReceiverHandler.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,13 @@ namespace MessagesAnalyzer
using System;
using System.Collections.Generic;
using System.Threading.Tasks;
using Microsoft.Azure.Devices.Edge.Util;
using Microsoft.Azure.EventHubs;
using Serilog;
using Microsoft.Extensions.Logging;

class PartitionReceiveHandler : IPartitionReceiveHandler
{
static readonly ILogger Log = Logger.Factory.CreateLogger<PartitionReceiveHandler>();
const string DeviceIdPropertyName = "iothub-connection-device-id";
const string ModuleIdPropertyName = "iothub-connection-module-id";
const string SequenceNumberPropertyName = "sequenceNumber";
Expand Down Expand Up @@ -54,8 +56,7 @@ public Task ProcessEventsAsync(IEnumerable<EventData> events)
}
else
{
Log.Debug($"Message for moduleId: {modId} doesn't contain required properties");

Log.LogDebug($"Message for moduleId: {modId} doesn't contain required properties");
}
}
}
Expand All @@ -66,7 +67,7 @@ public Task ProcessEventsAsync(IEnumerable<EventData> events)

public Task ProcessErrorAsync(Exception error)
{
Log.Error(error.StackTrace);
Log.LogError(error.StackTrace);
return Task.CompletedTask;
}

Expand Down
19 changes: 5 additions & 14 deletions edge-modules/MessagesAnalyzer/Program.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,16 +10,17 @@ namespace MessagesAnalyzer
using Microsoft.AspNetCore;
using Microsoft.AspNetCore.Hosting;
using Microsoft.Azure.Devices.Common;
using Microsoft.Azure.Devices.Edge.Util;
using Microsoft.Azure.EventHubs;
using Microsoft.Extensions.Logging;
using Serilog;

class Program
{
static readonly ILogger Log = Logger.Factory.CreateLogger<Program>();

static async Task Main(string[] args)
{
InitLogger().CreateLogger("analyzer");
Log.Information($"Starting analyzer for deviceId: {Settings.Current.DeviceId}, exclude-modules: {string.Join(", ", Settings.Current.ExcludedModuleIds.ToArray())}");
Log.LogInformation($"Starting analyzer for deviceId: {Settings.Current.DeviceId}, exclude-modules: {string.Join(", ", Settings.Current.ExcludedModuleIds.ToArray())}");

await ReceiveMessages();

Expand All @@ -40,7 +41,7 @@ static IWebHostBuilder CreateWebHostBuilder(string[] args) =>
static async Task ReceiveMessages()
{
var builder = new EventHubsConnectionStringBuilder(Settings.Current.EventHubConnectionString);
Log.Information($"Receiving events from device '{Settings.Current.DeviceId}' on Event Hub '{builder.EntityPath}'");
Log.LogInformation($"Receiving events from device '{Settings.Current.DeviceId}' on Event Hub '{builder.EntityPath}'");

EventHubClient eventHubClient =
EventHubClient.CreateFromConnectionString(builder.ToString());
Expand All @@ -52,15 +53,5 @@ static async Task ReceiveMessages()

eventHubReceiver.SetReceiveHandler(new PartitionReceiveHandler(Settings.Current.DeviceId, Settings.Current.ExcludedModuleIds));
}

static ILoggerFactory InitLogger()
{
Log.Logger = new LoggerConfiguration()
.MinimumLevel.Debug()
.WriteTo.Console(outputTemplate: "[{Timestamp:yyyy-MM-dd HH:mm:ss.fff} {Level:u3}] {Message:lj}{NewLine}{Exception}")
.CreateLogger();

return new LoggerFactory().AddSerilog();
}
}
}
10 changes: 10 additions & 0 deletions edge-modules/MessagesAnalyzer/Reporter.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,13 @@ namespace MessagesAnalyzer
using System;
using System.Collections.Generic;
using System.Linq;
using Microsoft.Azure.Devices.Edge.Util;
using Microsoft.Extensions.Logging;

class Reporter
{
static readonly ILogger Log = Logger.Factory.CreateLogger<Reporter>();

public static DeviceReport GetReceivedMessagesReport(double toleranceInMilliseconds)
{
DateTime enDateTime = DateTime.UtcNow;
Expand All @@ -24,6 +28,8 @@ public static DeviceReport GetReceivedMessagesReport(double toleranceInMilliseco

static ModuleReport GetReceivedMessagesReport(string moduleId, double toleranceInMilliseconds, IList<SortedSet<MessageDetails>> batchesSnapshot, DateTime endDateTime)
{
Log.LogInformation($"Report for {moduleId}");

long missingCounter = 0;
long totalMessagesCounter = 0;
IList<MissedMessagesDetails> missingIntervals = new List<MissedMessagesDetails>();
Expand All @@ -44,10 +50,14 @@ static ModuleReport GetReceivedMessagesReport(string moduleId, double toleranceI
{
// ignore messages enqued after endTime
if (DateTime.Compare(endDateTime, msg.EnquedDateTime) < 0)
{
Log.LogDebug($"Ignore message for {moduleId} enqued at {msg.EnquedDateTime} because is after {endDateTime}");
break;
}

if (msg.SequenceNumber - 1 != prevSequenceNumber)
{
Log.LogInformation($"Missing messages for {moduleId} from {prevSequenceNumber} to {msg.SequenceNumber}");
missingCounter += msg.SequenceNumber - prevSequenceNumber;
missingIntervals.Add(new MissedMessagesDetails(msg.SequenceNumber - prevSequenceNumber, prevEnquedDateTime, msg.EnquedDateTime));
}
Expand Down
118 changes: 116 additions & 2 deletions stress/deployment.template.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,22 +29,134 @@
"type": "docker",
"status": "running",
"restartPolicy": "always",
"env": {
"CollectMetrics": {
"value": true
}
},
"settings": {
"image": "edgebuilds.azurecr.io/microsoft/azureiotedge-hub:<Build.BuildNumber>-linux-amd64",
"createOptions": "{\"HostConfig\":{\"PortBindings\":{\"8883/tcp\":[{\"HostPort\":\"8883\"}],\"443/tcp\":[{\"HostPort\":\"443\"}]}}}"
}
}
},
"modules": {
"loadGen": {
"loadGen1": {
"version": "1.0",
"type": "docker",
"status": "running",
"restartPolicy": "always",
"env": {
"messageFrequency": {
"value": "<LoadGen.MessageFrequency>"
}
},
"settings": {
"image": "edgebuilds.azurecr.io/microsoft/azureiotedge-load-gen:<Build.BuildNumber>-linux-amd64",
"createOptions": ""
}
},
"loadGen2": {
"version": "1.0",
"type": "docker",
"status": "running",
"restartPolicy": "always",
"env": {
"messageFrequency": {
"value": "<LoadGen.MessageFrequency>"
}
},
"settings": {
"image": "edgebuilds.azurecr.io/microsoft/azureiotedge-load-gen:<Build.BuildNumber>-linux-amd64",
"createOptions": ""
}
},
"loadGen3": {
"version": "1.0",
"type": "docker",
"status": "running",
"restartPolicy": "always",
"env": {
"messageFrequency": {
"value": "<LoadGen.MessageFrequency>"
}
},
"settings": {
"image": "edgebuilds.azurecr.io/microsoft/azureiotedge-load-gen:<Build.BuildNumber>-linux-amd64",
"createOptions": ""
}
},
"snitcher": {
"version": "2.0",
"type": "docker",
"status": "running",
"restartPolicy": "always",
"env": {
"BUILD_ID": {
"value": "<Build.BuildNumber>"
},
"TEST_DURATION_IN_SECS": {
"value": "<Snitch.TestDurationInSecs>"
},
"REPORTING_INTERVAL_IN_SECS": {
"value": "<Snitch.ReportingIntervalInSecs>"
},
"ALERT_URL": {
"value": "<Snitch.AlertUrl>"
},
"INFLUX_URL": {
"value": "http://influxdb:8086"
},
"INFLUX_DB_NAME": {
"value": "_internal"
},
"INFLUX_QUERY_all": {
"value": "select * from /.*/"
},
"ANALYZER_URL": {
"value": "http://analyzer:15000/api/report"
},
"BLOB_STORAGE_ACCOUNT": {
"value": "<Snitch.StorageAccount>"
},
"BLOB_STORAGE_MASTER_KEY": {
"value": "<Snitch.StorageMasterKey>"
},
"BLOB_CONTAINER_NAME": {
"value": "loadtest1"
}
},
"settings": {
"image": "edgebuilds.azurecr.io/microsoft/snitcher:<Snitch.BuildNumber>",
"createOptions": "{\"HostConfig\":{\"Binds\":[\"/var/run/docker.sock:/var/run/docker.sock\"]}}"
}
},
"analyzer": {
"type": "docker",
"status": "running",
"restartPolicy": "never",
"env": {
"EventHubConnectionString": {
"value": "<Analyzer.EventHubConnectionString>"
},
"DeviceId": {
"value": "<Analyzer.DeviceID>"
}
},
"settings": {
"image": "edgebuilds.azurecr.io/microsoft/azureiotedge-analyzer:<Build.BuildNumber>-linux-amd64",
"createOptions": "{\"HostConfig\":{\"PortBindings\":{\"15000/tcp\":[{\"HostPort\":\"15000\"}]}}}"
}
},
"influxdb": {
"version": "1.0",
"type": "docker",
"status": "running",
"restartPolicy": "on-unhealthy",
"settings": {
"image": "influxdb:1.5",
"createOptions": "{\"ExposedPorts\":{\"8086/tcp\":{}},\"HostConfig\":{\"PortBindings\":{\"8086/tcp\":[{\"HostPort\":\"8086\"}]}}}"
}
}
}
}
Expand All @@ -53,7 +165,9 @@
"properties.desired": {
"schemaVersion": "1.0",
"routes": {
"upstream": "FROM * INTO $upstream"
"upstreamLoadGen1": "FROM /messages/modules/loadGen1/outputs/output1 INTO $upstream",
"upstreamLoadGen2": "FROM /messages/modules/loadGen2/outputs/output1 INTO $upstream",
"upstreamLoadGen3": "FROM /messages/modules/loadGen3/outputs/output1 INTO $upstream"
},
"storeAndForwardConfiguration": {
"timeToLiveSecs": 7200
Expand Down

0 comments on commit cd1fa09

Please sign in to comment.