tried to decouple the worker from kinesis while upgrading the framework and tooling.

humblelistener · humblelistener · commit 017ba5a3cedd · 2017-06-21T02:15:26.000+10:00
diff --git a/.gitignore b/.gitignore
@@ -1,4 +1,4 @@
-## Ignore Visual Studio temporary files, build results, and
+﻿## Ignore Visual Studio temporary files, build results, and
 ## files generated by popular Visual Studio add-ons.
 
 # User-specific files
@@ -23,7 +23,8 @@ bld/
 [Oo]bj/
 
 # Visual Studio 2015 cache/options directory
-.vs/
+.vs/
+.vscode/
 # Uncomment if you have tasks that create the project's static files in wwwroot
 #wwwroot/
 
diff --git a/.vscode/settings.json b/.vscode/settings.json
diff --git a/KinesisNet/AsyncHelper.cs b/KinesisNet/AsyncHelper.cs
@@ -2,7 +2,7 @@
 using System.Threading;
 using System.Threading.Tasks;
 
-namespace WorkerService.KinesisNet
+namespace KinesisNet
 {
     /// <summary>
     /// Used to task a async task and run it without the async part.
diff --git a/KinesisNet/Consumer.cs b/KinesisNet/Consumer.cs
@@ -6,11 +6,11 @@
 using Amazon.Kinesis;
 using Amazon.Kinesis.Model;
 using Serilog;
-using WorkerService.KinesisNet.Extensions;
-using WorkerService.KinesisNet.Interface;
-using WorkerService.KinesisNet.Model;
+using KinesisNet.Extensions;
+using KinesisNet.Interface;
+using KinesisNet.Model;
 
-namespace WorkerService.KinesisNet
+namespace KinesisNet
 {
     internal class Consumer : IConsumer
     {
diff --git a/KinesisNet/Extensions/AsyncExtensions.cs b/KinesisNet/Extensions/AsyncExtensions.cs
@@ -2,7 +2,7 @@
 using System.Threading;
 using System.Threading.Tasks;
 
-namespace WorkerService.KinesisNet.Extensions
+namespace KinesisNet.Extensions
 {
     public static class AsyncExtensions
     {
diff --git a/KinesisNet/Interface/IConsumer.cs b/KinesisNet/Interface/IConsumer.cs
@@ -1,7 +1,8 @@
 ﻿using System.Threading.Tasks;
-using WorkerService.KinesisNet.Model;
+using KinesisNet.Interface;
+using KinesisNet.Model;
 
-namespace WorkerService.KinesisNet.Interface
+namespace KinesisNet.Interface
 {
     public interface IConsumer
     {
diff --git a/KinesisNet/Interface/IDynamoDB.cs b/KinesisNet/Interface/IDynamoDB.cs
@@ -1,9 +1,9 @@
 ﻿using System;
 using System.Collections.Generic;
 using System.Threading.Tasks;
-using WorkerService.KinesisNet.Model;
+using KinesisNet.Model;
 
-namespace WorkerService.KinesisNet.Interface
+namespace KinesisNet.Interface
 {
     internal interface IDynamoDB
     {
diff --git a/KinesisNet/Interface/IKManager.cs b/KinesisNet/Interface/IKManager.cs
@@ -1,4 +1,4 @@
-﻿namespace WorkerService.KinesisNet.Interface
+﻿namespace KinesisNet.Interface
 {
     public interface IKManager
     {
diff --git a/KinesisNet/Interface/IProducer.cs b/KinesisNet/Interface/IProducer.cs
@@ -1,7 +1,7 @@
 ﻿using System.Threading.Tasks;
 using Amazon.Kinesis.Model;
 
-namespace WorkerService.KinesisNet.Interface
+namespace KinesisNet.Interface
 {
     public interface IProducer
     {
diff --git a/KinesisNet/Interface/IRecordProcessor.cs b/KinesisNet/Interface/IRecordProcessor.cs
@@ -2,7 +2,7 @@
 using System.Collections.Generic;
 using Amazon.Kinesis.Model;
 
-namespace WorkerService.KinesisNet.Interface
+namespace KinesisNet.Interface
 {
     public interface IRecordProcessor
     {
diff --git a/KinesisNet/Interface/IUtilities.cs b/KinesisNet/Interface/IUtilities.cs
@@ -3,7 +3,7 @@
 using Amazon.Kinesis.Model;
 using Serilog;
 
-namespace WorkerService.KinesisNet.Interface
+namespace KinesisNet.Interface
 {
     public interface IUtilities
     {
diff --git a/KinesisNet/KManager.cs b/KinesisNet/KManager.cs
@@ -3,10 +3,10 @@
 using Amazon.DynamoDBv2;
 using Amazon.Kinesis;
 using Serilog;
-using WorkerService.KinesisNet.Interface;
-using WorkerService.KinesisNet.Persistance;
+using KinesisNet.Interface;
+using KinesisNet.Persistance;
 
-namespace WorkerService.KinesisNet
+namespace KinesisNet
 {
     public class KManager : IKManager
     {
@@ -73,7 +73,7 @@ public KManager(string awsKey, string awsSecret, AmazonKinesisConfig config, str
         {
             if (workerId == null)
             {
-                workerId = System.Environment.MachineName;
+                workerId = Environment.MachineName;
             }
 
             _client = new AmazonKinesisClient(awsKey, awsSecret, awsSessionToken, config);
@@ -92,7 +92,7 @@ public KManager(IAmazonDynamoDB dynamoClient, IAmazonKinesis kinesisClient, stri
             Log.Debug("Entering KManager ctor");
             if (workerId == null)
             {
-                workerId = System.Environment.MachineName;
+                workerId = Environment.MachineName;
             }
 
             _client = kinesisClient;
@@ -116,7 +116,7 @@ private void Init(string workerId = null)
         {
             if (workerId == null)
             {
-                workerId = System.Environment.MachineName;
+                workerId = Environment.MachineName;
             }
 
             _utilities = new Utilities(_client, workerId);
diff --git a/KinesisNet/KinesisNet.csproj b/KinesisNet/KinesisNet.csproj
@@ -0,0 +1,13 @@
+﻿<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>netstandard1.6</TargetFramework>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="AWSSDK.DynamoDBv2" Version="3.3.4.13" />
+    <PackageReference Include="AWSSDK.Kinesis" Version="3.3.2.4" />
+    <PackageReference Include="Serilog" Version="2.4.0" />
+  </ItemGroup>
+
+</Project>
diff --git a/KinesisNet/Model/KShard.cs b/KinesisNet/Model/KShard.cs
@@ -3,7 +3,7 @@
 using Amazon.Kinesis;
 using Amazon.Kinesis.Model;
 
-namespace WorkerService.KinesisNet.Model
+namespace KinesisNet.Model
 {
     internal class KShard
     {
diff --git a/KinesisNet/Model/RecordResponse.cs b/KinesisNet/Model/RecordResponse.cs
@@ -1,7 +1,7 @@
 ﻿using System.Threading;
 using Amazon.Kinesis.Model;
 
-namespace WorkerService.KinesisNet.Model
+namespace KinesisNet.Model
 {
     internal class RecordResponse
     {
diff --git a/KinesisNet/Model/Result.cs b/KinesisNet/Model/Result.cs
@@ -1,4 +1,4 @@
-﻿namespace WorkerService.KinesisNet.Model
+﻿namespace KinesisNet.Model
 {
     public class Result
     {
diff --git a/KinesisNet/Persistance/DynamoDB.cs b/KinesisNet/Persistance/DynamoDB.cs
@@ -9,10 +9,10 @@
 using Amazon.DynamoDBv2;
 using Amazon.DynamoDBv2.Model;
 using Serilog;
-using WorkerService.KinesisNet.Interface;
-using WorkerService.KinesisNet.Model;
+using KinesisNet.Interface;
+using KinesisNet.Model;
 
-namespace WorkerService.KinesisNet.Persistance
+namespace KinesisNet.Persistance
 {
     internal class DynamoDB : IDynamoDB
     {
diff --git a/KinesisNet/Producer.cs b/KinesisNet/Producer.cs
@@ -5,9 +5,9 @@
 using System.Threading.Tasks;
 using Amazon.Kinesis;
 using Amazon.Kinesis.Model;
-using WorkerService.KinesisNet.Interface;
+using KinesisNet.Interface;
 
-namespace WorkerService.KinesisNet
+namespace KinesisNet
 {
     internal class Producer : IProducer
     {
diff --git a/KinesisNet/Utilities.cs b/KinesisNet/Utilities.cs
@@ -8,9 +8,9 @@
 using Amazon.Kinesis;
 using Amazon.Kinesis.Model;
 using Serilog;
-using WorkerService.KinesisNet.Interface;
+using KinesisNet.Interface;
 
-namespace WorkerService.KinesisNet
+namespace KinesisNet
 {
     internal class Utilities : IUtilities
     {
diff --git a/WebService.UnitTests/WebService.UnitTests.csproj b/WebService.UnitTests/WebService.UnitTests.csproj
@@ -21,4 +21,8 @@
     <PackageReference Include="xunit" Version="2.2.0" />
   </ItemGroup>
 
+  <ItemGroup>
+    <Service Include="{82a7f48d-3b50-4b1e-b82e-3ada8210c358}" />
+  </ItemGroup>
+
 </Project>
diff --git a/WorkerService/EventProcessors/RolesEventProcessor.cs b/WorkerService/EventProcessors/RolesEventProcessor.cs
@@ -1,8 +1,8 @@
-using System;
+﻿using System;
 using System.Collections.Generic;
 using System.Text;
 using Amazon.Kinesis.Model;
-using WorkerService.KinesisNet.Interface;
+using KinesisNet.Interface;
 
 namespace WorkerService.EventProcessors
 {
diff --git a/WorkerService/Program.cs b/WorkerService/Program.cs
@@ -1,29 +1,21 @@
-using System;
-using System.Reflection;
+﻿using System.Reflection;
 using System.Threading.Tasks;
 using Amazon;
 using Amazon.DynamoDBv2;
 using Amazon.Kinesis;
 using Microsoft.Extensions.Configuration;
 using Serilog;
-using Serilog.Events;
-using Serilog.Formatting.Json;
-using SeriLog.LogSanitizingFormatter;
 using WorkerService.EventProcessors;
-using WorkerService.KinesisNet;
-using WorkerService.KinesisNet.Interface;
-using WorkerService.KinesisNet.Model;
-using WorkerService.KinesisNet.Persistance;
+using KinesisNet;
 using static System.Threading.Thread;
 using static System.Threading.Timeout;
-using static System.Console;
 
 namespace WorkerService
 {
     public class Program
     {
-        private static Environment _environment;
-        private static KManager _kManager;
+        private static Environment environment;
+        private static KManager kManager;
 
         public static void Main(string[] args)
         {
@@ -34,8 +26,7 @@ public static void Main(string[] args)
 
             Log.Debug("Testing debug logging");
 
-            //new LoggerConfiguration().WriteTo.Co
-            _environment = new Environment
+            environment = new Environment
             (
                 System.Environment.GetEnvironmentVariable("REGION"),
                 System.Environment.GetEnvironmentVariable("DC"),
@@ -47,7 +38,7 @@ public static void Main(string[] args)
                 .InformationalVersion
             );
             
-            Log.Information("{@Environment}", _environment);
+            Log.Information("{@Environment}", environment);
 
             Log.Information("Starting configuration");
 
@@ -67,29 +58,29 @@ public static void Main(string[] args)
 
         private static void StopListeningToEvents()
         {
-            _kManager.Consumer.Stop();
+            kManager.Consumer.Stop();
         }
 
         public static void StartListeningForEvents()
         {
             Log.Debug("Entering StartListeningForEvents");
             var kinesisStreamName = "RoleStream.LIVE";
-            var kinesisWorkerId = Assembly.GetEntryAssembly().GetName().Name + "-" + _environment.Env;
+            var kinesisWorkerId = Assembly.GetEntryAssembly().GetName().Name + "-" + environment.Env;
 
             Log.Information("KManager variables:");
             Log.Information($"  StreamName: {kinesisStreamName}");
             Log.Information($"  WorkerId: {kinesisWorkerId}");
 
             Log.Debug("Creating Dyanmo client");
-            var dynamoClient = new AmazonDynamoDBClient(new AmazonDynamoDBConfig{RegionEndpoint = RegionEndpoint.GetBySystemName(_environment.AwsRegion)});
+            var dynamoClient = new AmazonDynamoDBClient(new AmazonDynamoDBConfig{RegionEndpoint = RegionEndpoint.GetBySystemName(environment.AwsRegion)});
             Log.Debug("Creating Kinesis client");
-            var kinesisClient = new AmazonKinesisClient(new AmazonKinesisConfig{ RegionEndpoint = RegionEndpoint.GetBySystemName(_environment.AwsRegion )});
+            var kinesisClient = new AmazonKinesisClient(new AmazonKinesisConfig{ RegionEndpoint = RegionEndpoint.GetBySystemName(environment.AwsRegion )});
 
             Log.Debug("Starting Kmanager");
-            _kManager = new KManager(dynamoClient, kinesisClient, kinesisStreamName, kinesisWorkerId);
+            kManager = new KManager(dynamoClient, kinesisClient, kinesisStreamName, kinesisWorkerId);
 
             Log.Debug("Starting consumer");
-            Task.Run(() => _kManager.Consumer.Start(new RolesEventProcessor()));
+            Task.Run(() => kManager.Consumer.Start(new RolesEventProcessor()));
         }
     }
 }
diff --git a/WorkerService/WorkerService.csproj b/WorkerService/WorkerService.csproj
@@ -11,15 +11,19 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="1.0.2" />
-    <PackageReference Include="Microsoft.Extensions.Configuration.CommandLine" Version="1.0.2" />
-    <PackageReference Include="Microsoft.Extensions.Configuration.FileExtensions" Version="1.0.2" />
-    <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="1.0.2" />
-    <PackageReference Include="AWSSDK.DynamoDBv2" Version="3.3.2" />
-    <PackageReference Include="AWSSDK.Kinesis" Version="3.3.1.3" />
-    <PackageReference Include="Serilog" Version="2.4.0-dev-00767" />
-    <PackageReference Include="Serilog.Sinks.Literate" Version="2.1.0-dev-00034" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="1.1.2" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.CommandLine" Version="1.1.2" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.FileExtensions" Version="1.1.2" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="1.1.2" />
+    <PackageReference Include="AWSSDK.DynamoDBv2" Version="3.3.4.13" />
+    <PackageReference Include="AWSSDK.Kinesis" Version="3.3.2.4" />
+    <PackageReference Include="Serilog" Version="2.4.0" />
+    <PackageReference Include="Serilog.Sinks.Literate" Version="2.1.0" />
     <PackageReference Include="SeriLog.LogSanitizingFormatter" Version="1.0.1" />
   </ItemGroup>
 
+  <ItemGroup>
+    <ProjectReference Include="..\KinesisNet\KinesisNet.csproj" />
+  </ItemGroup>
+
 </Project>
diff --git a/microservice.sln b/microservice.sln
@@ -6,6 +6,7 @@ MinimumVisualStudioVersion = 10.0.40219.1
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{44FEDB3C-7C7E-4D87-9536-3E4B6A516073}"
 	ProjectSection(SolutionItems) = preProject
 		.editorconfig = .editorconfig
+		.gitignore = .gitignore
 		.travis.yml = .travis.yml
 		deploy.sh = deploy.sh
 		docker-compose.yml = docker-compose.yml
@@ -22,6 +23,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "WorkerService", "WorkerServ
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "WorkerService.UnitTests", "WorkerService.UnitTests\WorkerService.UnitTests.csproj", "{DF1D9DC2-C641-4D89-959B-7364F039EC1A}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "KinesisNet", "KinesisNet\KinesisNet.csproj", "{D3ADBFD7-656F-4B99-9CDB-C4DEB11E6FB9}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -44,6 +47,10 @@ Global
 		{DF1D9DC2-C641-4D89-959B-7364F039EC1A}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{DF1D9DC2-C641-4D89-959B-7364F039EC1A}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{DF1D9DC2-C641-4D89-959B-7364F039EC1A}.Release|Any CPU.Build.0 = Release|Any CPU
+		{D3ADBFD7-656F-4B99-9CDB-C4DEB11E6FB9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{D3ADBFD7-656F-4B99-9CDB-C4DEB11E6FB9}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{D3ADBFD7-656F-4B99-9CDB-C4DEB11E6FB9}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{D3ADBFD7-656F-4B99-9CDB-C4DEB11E6FB9}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

Original file line number	Diff line number	Diff line change
`@@ -2,7 +2,7 @@`
`2`	`2`	`using System.Threading;`
`3`	`3`	`using System.Threading.Tasks;`
`4`	`4`
`5`		`-namespace WorkerService.KinesisNet`
	`5`	`+namespace KinesisNet`
`6`	`6`	`{`
`7`	`7`	`/// <summary>`
`8`	`8`	`/// Used to task a async task and run it without the async part.`
Original file line number	Diff line number	Diff line change
`@@ -1,7 +1,8 @@`
`1`	`1`	`using System.Threading.Tasks;`
`2`		`-using WorkerService.KinesisNet.Model;`
	`2`	`+using KinesisNet.Interface;`
	`3`	`+using KinesisNet.Model;`
`3`	`4`
`4`		`-namespace WorkerService.KinesisNet.Interface`
	`5`	`+namespace KinesisNet.Interface`
`5`	`6`	`{`
`6`	`7`	`public interface IConsumer`
`7`	`8`	`{`
Original file line number	Diff line number	Diff line change
`@@ -1,9 +1,9 @@`
`1`	`1`	`using System;`
`2`	`2`	`using System.Collections.Generic;`
`3`	`3`	`using System.Threading.Tasks;`
`4`		`-using WorkerService.KinesisNet.Model;`
	`4`	`+using KinesisNet.Model;`
`5`	`5`
`6`		`-namespace WorkerService.KinesisNet.Interface`
	`6`	`+namespace KinesisNet.Interface`
`7`	`7`	`{`
`8`	`8`	`internal interface IDynamoDB`
`9`	`9`	`{`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-namespace WorkerService.KinesisNet.Interface`
	`1`	`+namespace KinesisNet.Interface`
`2`	`2`	`{`
`3`	`3`	`public interface IKManager`
`4`	`4`	`{`
Original file line number	Diff line number	Diff line change
`@@ -1,7 +1,7 @@`
`1`	`1`	`using System.Threading.Tasks;`
`2`	`2`	`using Amazon.Kinesis.Model;`
`3`	`3`
`4`		`-namespace WorkerService.KinesisNet.Interface`
	`4`	`+namespace KinesisNet.Interface`
`5`	`5`	`{`
`6`	`6`	`public interface IProducer`
`7`	`7`	`{`
Original file line number	Diff line number	Diff line change
`@@ -3,7 +3,7 @@`
`3`	`3`	`using Amazon.Kinesis.Model;`
`4`	`4`	`using Serilog;`
`5`	`5`
`6`		`-namespace WorkerService.KinesisNet.Interface`
	`6`	`+namespace KinesisNet.Interface`
`7`	`7`	`{`
`8`	`8`	`public interface IUtilities`
`9`	`9`	`{`
Original file line number	Diff line number	Diff line change
`@@ -3,10 +3,10 @@`
`3`	`3`	`using Amazon.DynamoDBv2;`
`4`	`4`	`using Amazon.Kinesis;`
`5`	`5`	`using Serilog;`
`6`		`-using WorkerService.KinesisNet.Interface;`
`7`		`-using WorkerService.KinesisNet.Persistance;`
	`6`	`+using KinesisNet.Interface;`
	`7`	`+using KinesisNet.Persistance;`
`8`	`8`
`9`		`-namespace WorkerService.KinesisNet`
	`9`	`+namespace KinesisNet`
`10`	`10`	`{`
`11`	`11`	`public class KManager : IKManager`
`12`	`12`	`{`
`@@ -73,7 +73,7 @@ public KManager(string awsKey, string awsSecret, AmazonKinesisConfig config, str`
`73`	`73`	`{`
`74`	`74`	`if (workerId == null)`
`75`	`75`	`{`
`76`		`- workerId = System.Environment.MachineName;`
	`76`	`+ workerId = Environment.MachineName;`
`77`	`77`	`}`
`78`	`78`
`79`	`79`	`_client = new AmazonKinesisClient(awsKey, awsSecret, awsSessionToken, config);`
`@@ -92,7 +92,7 @@ public KManager(IAmazonDynamoDB dynamoClient, IAmazonKinesis kinesisClient, stri`
`92`	`92`	`Log.Debug("Entering KManager ctor");`
`93`	`93`	`if (workerId == null)`
`94`	`94`	`{`
`95`		`- workerId = System.Environment.MachineName;`
	`95`	`+ workerId = Environment.MachineName;`
`96`	`96`	`}`
`97`	`97`
`98`	`98`	`_client = kinesisClient;`
`@@ -116,7 +116,7 @@ private void Init(string workerId = null)`
`116`	`116`	`{`
`117`	`117`	`if (workerId == null)`
`118`	`118`	`{`
`119`		`- workerId = System.Environment.MachineName;`
	`119`	`+ workerId = Environment.MachineName;`
`120`	`120`	`}`
`121`	`121`
`122`	`122`	`_utilities = new Utilities(_client, workerId);`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-namespace WorkerService.KinesisNet.Model`
	`1`	`+namespace KinesisNet.Model`
`2`	`2`	`{`
`3`	`3`	`public class Result`
`4`	`4`	`{`
Original file line number	Diff line number	Diff line change
`@@ -9,10 +9,10 @@`
`9`	`9`	`using Amazon.DynamoDBv2;`
`10`	`10`	`using Amazon.DynamoDBv2.Model;`
`11`	`11`	`using Serilog;`
`12`		`-using WorkerService.KinesisNet.Interface;`
`13`		`-using WorkerService.KinesisNet.Model;`
	`12`	`+using KinesisNet.Interface;`
	`13`	`+using KinesisNet.Model;`
`14`	`14`
`15`		`-namespace WorkerService.KinesisNet.Persistance`
	`15`	`+namespace KinesisNet.Persistance`
`16`	`16`	`{`
`17`	`17`	`internal class DynamoDB : IDynamoDB`
`18`	`18`	`{`
Original file line number	Diff line number	Diff line change
`@@ -5,9 +5,9 @@`
`5`	`5`	`using System.Threading.Tasks;`
`6`	`6`	`using Amazon.Kinesis;`
`7`	`7`	`using Amazon.Kinesis.Model;`
`8`		`-using WorkerService.KinesisNet.Interface;`
	`8`	`+using KinesisNet.Interface;`
`9`	`9`
`10`		`-namespace WorkerService.KinesisNet`
	`10`	`+namespace KinesisNet`
`11`	`11`	`{`
`12`	`12`	`internal class Producer : IProducer`
`13`	`13`	`{`