IoT Samrt Grig v2 - 1.4M rows/sec

Author: perrysk-msft

samples/applications/iot-smart-grid/App.config

@@ -4,21 +4,26 @@
     <supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.5.2" />
   </startup>
   <connectionStrings>
-    <!--<add name="Db" connectionString="Server=tcp:<server>.database.windows.net,1433;Database=PowerConsumption;User ID=<user>;Password=<password>;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;"/>-->
+    <clear />
+    <!--<add name="Db" connectionString="Server=tcp:(YOUR_SQLSERVERNAME).database.windows.net,1433;Database=(YOUR_DATABASE_NAME);User ID=(YOUR_USERNAME);Password=(YOUR_PASSWORD);Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;"/>-->
     <add name="Db" connectionString="Server=.;Database=PowerConsumption;Integrated Security=True"/>
   </connectionStrings>
   <appSettings>
-    <add key="insertSPName" value="InsertMeterMeasurement"/>  <!--Stored Procedure Name-->
-    <add key="numberOfTasks" value="50"/>                     <!--Number of concurrent async tasks that the Data Generator will use-->
-    <add key="numberOfMeters" value="1000000"/>               <!--Number of unique meter Ids-->
-    <add key="batchSize" value="1000"/>                       <!--Row Batch Size that every task produces-->
-    <add key="commandDelay" value="1500"/>                    <!--Delay between sql commands. You can set this to 0 for max high volume workload-->
-    <add key="commandTimeout" value="600"/>                   <!--SQL Command Timeout-->
-    <add key="enableShock" value="1"/>                        <!--Flag that turns on/off the data shock. This should be set to 0 for max high volume workload-->
-    <add key="shockFrequency" value="35000"/>                 <!--The Frequency that the code sets the commandDelay to 0-->
-    <add key="shockDuration" value="4000"/>                   <!--The Duration that the code keeps the commandDelay to 0-->
-    <add key="rpsFrequency" value="2000"/>                    <!--How frequently the Data Generator Rows Per Second(RPS) is polled-->
-    <add key="logFileName" value="log.txt"/>                  <!--Log File Path-->
-    <add key="powerBIDesktopPath" value="C:\Program Files\Microsoft Power BI Desktop\bin\PBIDesktop.exe"/><!--PowerBIDesktop.exe path-->    
+    <add key="insertSPName" value="InsertMeterMeasurement"/>        <!--Stored Procedure Name-->
+    <add key="numberOfDataLoadTasks" value="70"/>                   <!--Number of concurrent async tasks that the Data Generator will use (per SQL Connection)-->
+    <add key="dataLoadCommandDelay" value="2000"/>                  <!--Delay between sql commands. You can set this to 0 for max high volume workload-->
+    <add key="batchSize" value="70000"/>                            <!--Insert: Row Batch Size that every task produces-->    
+    <add key="deleteSPName" value="InsertMeterMeasurementHistory"/> <!--Stored Procedure Name-->
+    <add key="numberOfOffLoadTasks" value="50"/>                    <!--Number of concurrent async tasks that the Data Generator will use (per SQL Connection)-->
+    <add key="offLoadCommandDelay" value="0"/>                      <!--Delay between sql commands. You can set this to 0 for max high volume workload-->
+    <add key="deleteBatchSize" value="1000000"/>                    <!--Delete: Row Batch Size that every task produces-->
+    <add key="numberOfMeters" value="75000000"/>                    <!--Number of unique meters-->
+    <add key="commandTimeout" value="600"/>                         <!--SQL Command Timeout-->
+    <add key="rpsFrequency" value="500"/>                           <!--How frequently the Data Generator Rows Per Second(RPS) is polled-->
+    <add key="logFileName" value="log.txt"/>                        <!--Log File Path-->
+    <add key="delayStart" value="0"/>                               <!--Delay Graph Interval-->
+    <add key="appRunDuration" value="1800000"/>                     <!--Run App Time: 30 mins-->
+    <add key="numberOfRowsOfloadLimit" value="1400000"/>            <!--Number Of Rows Ofload Limit-->
+    <add key="powerBIDesktopPath" value="C:\Program Files\Microsoft Power BI Desktop\bin\PBIDesktop.exe"/>  <!--PowerBIDesktop.exe path-->
   </appSettings>
 </configuration>

samples/applications/iot-smart-grid/ConsoleClient/Program.cs

@@ -36,40 +36,39 @@ This code sample simulates an IoT Smart Grid scenario where multiple IoT power m
 shock absorber scenario: https://blogs.technet.microsoft.com/dataplatforminsider/2013/09/19/in-memory-oltp-common-design-pattern-high-data-input-rateshock-absorber/. 
 Every async task in the Data Generator produces a batch of records with random values in order to simulate the data of an IoT power meter. 
 It then calls a natively compiled stored procedure, that accepts an memory optimized table valued parameter (TVP), to insert the data into an memory optimized SQL Server table. 
-In addition to the in-memory features, the sample is leveraging System-Versioned Temporal Tables: https://msdn.microsoft.com/en-us/library/dn935015.aspx for building version history, 
-Clustered Columnstore Index: https://msdn.microsoft.com/en-us/library/dn817827.aspx) for enabling real time operational analytics, and 
+In addition to the in-memory features, the sample is offloading historical values to a Clustered Columnstore Index: https://msdn.microsoft.com/en-us/library/dn817827.aspx) for enabling real time operational analytics, and 
 Power BI: https://powerbi.microsoft.com/en-us/desktop/ for data visualization. 
 */
 namespace ConsoleClient
 {
     class Program
     {
         static SqlDataGenerator dataGenerator;
-        static string connection;
+        static string[] connection;
         static string spName;
         static string logFileName;
         static string powerBIDesktopPath;
-        static int tasks;
         static int meters;
         static int batchSize;
-        static int delay;
         static int commandTimeout;
-        static int shockFrequency;
-        static int shockDuration;
-        static int rpsFrequency;
-        static int enableShock;
-        static Timer mainTimer = new Timer();
+        static int rpsFrequency;    
+        static int numberOfDataLoadTasks;
+        static int numberOfOffLoadTasks;
+        static int deleteBatchSize;
+        static string deleteSPName;
+        static int dataLoadCommandDelay;
+        static int offLoadCommandDelay;
+        static int delayStart;
+        static int appRunDuration;
+        static int numberOfRowsOfloadLimit;
         static Timer rpsTimer = new Timer();
-        static Timer shockTimer = new Timer();
 
         static void Main(string[] args)
         {
             Init();
-            dataGenerator = new SqlDataGenerator(connection, spName, commandTimeout, meters, tasks, delay, batchSize, ExceptionCallback);
-
-            mainTimer.Elapsed += mainTimer_Tick;
+            dataGenerator = new SqlDataGenerator(connection, spName, commandTimeout, meters, numberOfDataLoadTasks, dataLoadCommandDelay, batchSize, deleteSPName, numberOfOffLoadTasks, offLoadCommandDelay, deleteBatchSize, numberOfRowsOfloadLimit, ExceptionCallback);
             rpsTimer.Elapsed += rpsTimer_Tick;
-            shockTimer.Elapsed += shockTimer_Tick;
+
 
             string commandString = string.Empty;
             Console.ForegroundColor = ConsoleColor.White;
@@ -129,9 +128,7 @@ static async void Start()
             {
                 if (!dataGenerator.IsRunning)
                 {
-                    if(enableShock == 1) mainTimer.Start();
                     rpsTimer.Start();
-
                     await dataGenerator.RunAsync();
                 }
             }
@@ -144,10 +141,7 @@ static async void Stop()
             {
                 if (dataGenerator.IsRunning)
                 {
-                    if (enableShock == 1) mainTimer.Stop();
                     rpsTimer.Stop();
-                    if (enableShock == 1) shockTimer.Stop();
-
                     await dataGenerator.StopAsync();
                     dataGenerator.RpsReset();
                 }
@@ -177,47 +171,47 @@ static void Init()
         {
             try
             {
+                int numberOfSqlConnections = ConfigurationManager.ConnectionStrings.Count;
+                connection = new string[numberOfSqlConnections];
                 // Read Config Settings
-                connection = ConfigurationManager.ConnectionStrings["Db"].ConnectionString;
+                for (int i = 0; i < numberOfSqlConnections; i++)
+                {
+                    connection[i] = ConfigurationManager.ConnectionStrings[i].ConnectionString;
+                }
+
                 spName = ConfigurationManager.AppSettings["insertSPName"];
                 logFileName = ConfigurationManager.AppSettings["logFileName"];
-                powerBIDesktopPath = ConfigurationManager.AppSettings["powerBIDesktopPath"];
-                tasks = int.Parse(ConfigurationManager.AppSettings["numberOfTasks"]);
-                meters = int.Parse(ConfigurationManager.AppSettings["numberOfMeters"]);
+                numberOfDataLoadTasks = int.Parse(ConfigurationManager.AppSettings["numberOfDataLoadTasks"]);
+                dataLoadCommandDelay = int.Parse(ConfigurationManager.AppSettings["dataLoadCommandDelay"]);
                 batchSize = int.Parse(ConfigurationManager.AppSettings["batchSize"]);
-                delay = int.Parse(ConfigurationManager.AppSettings["commandDelay"]);
+                deleteSPName = ConfigurationManager.AppSettings["deleteSPName"];
+                numberOfOffLoadTasks = int.Parse(ConfigurationManager.AppSettings["numberOfOffLoadTasks"]);
+                offLoadCommandDelay = int.Parse(ConfigurationManager.AppSettings["offLoadCommandDelay"]);
+                deleteBatchSize = int.Parse(ConfigurationManager.AppSettings["deleteBatchSize"]);
+                meters = int.Parse(ConfigurationManager.AppSettings["numberOfMeters"]);
                 commandTimeout = int.Parse(ConfigurationManager.AppSettings["commandTimeout"]);
-                shockFrequency = int.Parse(ConfigurationManager.AppSettings["shockFrequency"]);
-                shockDuration = int.Parse(ConfigurationManager.AppSettings["shockDuration"]);
-                enableShock = int.Parse(ConfigurationManager.AppSettings["enableShock"]);
-
+                delayStart = int.Parse(ConfigurationManager.AppSettings["delayStart"]);
+                appRunDuration = int.Parse(ConfigurationManager.AppSettings["appRunDuration"]);
                 rpsFrequency = int.Parse(ConfigurationManager.AppSettings["rpsFrequency"]);
+                numberOfRowsOfloadLimit = int.Parse(ConfigurationManager.AppSettings["numberOfRowsOfloadLimit"]);
+                powerBIDesktopPath = ConfigurationManager.AppSettings["powerBIDesktopPath"];
 
                 // Initialize Timers
-                mainTimer.Interval = shockFrequency;
-                shockTimer.Interval = shockDuration;
                 rpsTimer.Interval = rpsFrequency;
 
                 if (batchSize <= 0) throw new SqlDataGeneratorException("The Batch Size cannot be less or equal to zero.");
 
-                if (tasks <= 0) throw new SqlDataGeneratorException("Number Of Tasks cannot be less or equal to zero.");
+                if (numberOfDataLoadTasks <= 0) throw new SqlDataGeneratorException("Number Of Tasks cannot be less or equal to zero.");
 
-                if (delay < 0) throw new SqlDataGeneratorException("Delay cannot be less than zero");
+                if (dataLoadCommandDelay < 0) throw new SqlDataGeneratorException("Delay cannot be less than zero");
 
                 if (meters <= 0) throw new SqlDataGeneratorException("Number Of Meters cannot be less than zero");
 
-                if (meters < batchSize * tasks) throw new SqlDataGeneratorException("Number Of Meters cannot be less than (Tasks * BatchSize).");
+                if (meters < batchSize * numberOfDataLoadTasks) throw new SqlDataGeneratorException("Number Of Meters cannot be less than (Tasks * BatchSize).");
+
             }
             catch (Exception exception) { HandleException(exception); }
         }
-        static void mainTimer_Tick(object sender, ElapsedEventArgs e)
-        {
-            if (dataGenerator.IsRunning)
-            {
-                dataGenerator.Delay = 0;
-                shockTimer.Start();
-            }
-        }
         static void rpsTimer_Tick(object sender, ElapsedEventArgs e)
         {
             try
@@ -236,11 +230,5 @@ static void rpsTimer_Tick(object sender, ElapsedEventArgs e)
             }
             catch (Exception exception) { HandleException(exception); }
         }
-        static void shockTimer_Tick(object sender, ElapsedEventArgs e)
-        {
-            Random rand = new Random();
-            dataGenerator.Delay = rand.Next(1500, 3000);
-            shockTimer.Stop();
-        }
     }
 }