Some fixes changes made during benchmarking to improve ICTCore perfor…

…mance
fizaict · Mar 2, 2018 · ab3f060 · ab3f060
1 parent e6868d6
commit ab3f060
Show file tree

Hide file tree

Showing 8 changed files with 123 additions and 20 deletions.
diff --git a/bin/campaign b/bin/campaign
@@ -77,6 +77,12 @@ class CampaignCli
    */
   private $is_active = false;
 
+  /**
+   * Delay between each transmission
+   * @var integer $delay
+   */
+  private $delay = 1000000; // one second
+
   public function __construct($campaign_id)
   {
     $this->campaign_id = $campaign_id;
@@ -85,7 +91,7 @@ class CampaignCli
     // set stop signal handler
     pcntl_signal(SIGQUIT, array($this, "signal_stop"));
     pcntl_signal(SIGABRT, array($this, "signal_stop"));
-    pcntl_signal(SIGKILL, array($this, "signal_stop"));
+    /* produce error  pcntl_signal(SIGKILL, array($this, "signal_stop")); */
     pcntl_signal(SIGTERM, array($this, "signal_stop"));
 
     // set reload signal handler
@@ -116,6 +122,9 @@ class CampaignCli
       throw new CoreException(423, 'Campaign already completed');
     }
     \ICT\Core\do_login($this->oCampaign->created_by);
+
+    $cpm = empty($this->oCampaign->cpm) ? 2 : $this->oCampaign->cpm;
+    $this->delay = (1000000 * 60) / $cpm;
   }
 
   public function prepare()
@@ -129,7 +138,7 @@ class CampaignCli
         $this->prepare_transmission($oProgram, $aContact['contact_id']);
         $count++;
       } catch (CoreException $e) {
-        Corelog::log('Unable to create transmission skipping. error:' . $e->getMessage, Corelog::WARNING);
+        Corelog::log('Unable to create transmission skipping. error:' . $e->getMessage(), Corelog::WARNING);
       }
     }
 
@@ -182,6 +191,8 @@ class CampaignCli
 
   private function process($listTransmission = array())
   {
+    $recent_attempt = round(microtime(true) * 1000);
+
     foreach ($listTransmission as $aTransmission) {
 
       // Stop processing if campaign is not active
@@ -190,12 +201,15 @@ class CampaignCli
       }
 
       // delay is required for system stability
-      usleep($this->oCampaign->delay);
+      $current_time = round(microtime(true) * 1000);
+      usleep($this->delay - ($current_time - $recent_attempt));
+      $recent_attempt = $current_time;
 
       try { // now send the transmission, in case of error simply skip that transmission
+        Corelog::log('Sending transmission with id: ' . $aTransmission['transmission_id'], Corelog::INFO);
         $this->transmission_send($aTransmission['transmission_id']);
       } catch (CoreException $e) {
-        Corelog::log('Unable to send transmission skipping. error:' . $e->getMessage, Corelog::WARNING);
+        Corelog::log('Unable to send transmission skipping. error:' . $e->getMessage(), Corelog::WARNING);
       }
     }
 
@@ -207,4 +221,4 @@ class CampaignCli
     $oTransmission = new Transmission($transmission_id);
     return $oTransmission->send();
   }
-}
+}
diff --git a/core/Api/CampaignApi.php b/core/Api/CampaignApi.php
@@ -69,6 +69,9 @@ public function update($campaign_id, $data = array())
     $this->set($oCampaign, $data);
 
     if ($oCampaign->save()) {
+      if ($oCampaign->status == Campaign::STATUS_RUNNING) {
+        $oCampaign->reload();
+      }
       return $oCampaign;
     } else {
       throw new CoreException(417, 'Campaign update failed');

diff --git a/core/Campaign.php b/core/Campaign.php
@@ -25,7 +25,7 @@ class Campaign
       'campaign_id ',
       'program_id',
       'group_id',
-      'delay',
+      'cpm',
       'try_allowed',
       'account_id',
       'created_by',
@@ -51,7 +51,7 @@ class Campaign
   public $group_id = NULL;
 
   /** @var string */
-  public $delay = NULL;
+  public $cpm = NULL;
 
   /** @var string */
   public $try_allowed = NULL;
@@ -116,7 +116,7 @@ private function load()
       $this->campaign_id = $data['campaign_id'];
       $this->program_id  = $data['program_id'];
       $this->group_id    = $data['group_id'];
-      $this->delay       = $data['delay'];
+      $this->cpm         = $data['cpm'];
       $this->try_allowed = $data['try_allowed'];
       $this->account_id  = $data['account_id'];
       $this->status      = $data['status'];
@@ -167,6 +167,16 @@ public function __set($field, $value)
     }
   }
 
+  public function set_delay($delay)
+  {
+    $this->cpm = $delay;
+  }
+
+  public function get_delay()
+  {
+    return $this->delay;
+  }
+
   public function get_id()
   {
     return $this->campaign_id;
@@ -182,7 +192,7 @@ public function save()
         'campaign_id' => $this->campaign_id,
         'program_id' => $this->program_id,
         'group_id' => $this->group_id,
-        'delay' => $this->delay,
+        'cpm' => $this->cpm,
         'try_allowed' => $this->try_allowed,
         'account_id' => $this->account_id,
         'status' => $this->status
@@ -211,14 +221,19 @@ public function stop()
     return $this->daemon('stop');
   }
 
+  public function reload()
+  {
+    return $this->daemon('reload');
+  }
+
   public function daemon($action = 'start')
   {
     global $path_root;
     $daemon = $path_root . DIRECTORY_SEPARATOR . 'bin' . DIRECTORY_SEPARATOR . 'campaign';
     /* Excutable file start / stop deamon */
     $output = array();
     $result = false;
-    exec("$daemon $this->campaign_id  $action", $output, $result);
+    exec("$daemon $this->campaign_id $action", $output, $result);
     if ($result != 0) {
       return false;
     } else {

diff --git a/core/Gateway.php b/core/Gateway.php
@@ -95,4 +95,4 @@ public function config_reload()
     Corelog::log('Gateway->config_reload', Corelog::WARNING);
   }
 
-}
+}
diff --git a/core/Gateway/Freeswitch.php b/core/Gateway/Freeswitch.php
@@ -49,19 +49,43 @@ public function __construct()
 
   protected function connect()
   {
+    static $fs_conn = NULL;
+    static $last_check = NULL;
 
-    $this->conn = fsockopen($this->host, $this->port);
-    socket_set_blocking($this->conn, false);
+    if (empty($this->conn) && $fs_conn !== NULL) {
+      $this->conn = $fs_conn;
+    }
+
+    // try to use existing connection
     if ($this->conn) {
-      while (!feof($this->conn)) {
-        $buffer = fgets($this->conn, 1024);
+      if (($last_check + 300) > time()) {
+        return $this->conn;
+      } else {
+        $status = socket_get_status($this->conn);
+        $last_check = time();
+        if ($status['timed_out'] == false && $status['blocked'] == false) {
+          return $this->conn;
+        }
+      }
+    }
+
+    $fs_socket = "tcp://$this->host:$this->port";
+    $error_no  = 0;
+    $error_msg = '';
+    if ($socket = stream_socket_client($fs_socket, $error_no, $error_msg)) {
+      stream_set_blocking($socket, false); // none blocking
+      stream_set_timeout($socket, 3);
+      while (!feof($socket)) {
+        $buffer = fgets($socket, 1024);
         usleep(100); //allow time for reponse
         if (trim($buffer) == "Content-Type: auth/request") {
-          fputs($this->conn, "auth $this->password\n\n");
+          fputs($socket, "auth $this->password\n\n");
           break;
         }
       }
       Corelog::log("Freeswitch connected successfully", Corelog::DEBUG);
+      $fs_conn = $socket;
+      $this->conn = $socket;
       return $this->conn;
     } else {
       Corelog::log("Freeswitch connection failed", Corelog::ERROR);
@@ -106,7 +130,7 @@ private function _send($command)
       fputs($this->conn, $command . "\n\n");
     }
 
-    $this->dissconnect();
+    //$this->dissconnect();
   }
 
   private function _read() {

diff --git a/db/database.sql b/db/database.sql
@@ -528,8 +528,8 @@ CREATE TABLE campaign
    program_id                int(11)                NOT NULL,
    group_id                  int(11)                NOT NULL ,
    account_id                int(11)                default NULL,
-   delay                     varchar(128)           NOT NULL default '',
-   try_allowed               varchar(128)           NOT NULL default '',
+   cpm                       int(11)                NOT NULL default 2,
+   try_allowed               int(11)                NOT NULL default 1,
    contact_total             int(11)                NOT NULL default 0,
    contact_done              int(11)                NOT NULL default 0,
    status                    varchar(128)           NOT NULL default '',

diff --git a/docs/ApiGuide.md b/docs/ApiGuide.md
@@ -162,7 +162,7 @@ HOST: http://ictcore.example.com
 ## Campaign (object)
 + program_id: 1 (number)
 + group_id: 2 (number)
-+ delay: 2 (number) - pause between transmissions `milliseconds`
++ cpm: 2 (number) - transmissions / cycles per second
 + try_allowed: 2 (number)
 + account_id: 1 (number) - account_id of associated account
 + status: active (string) - current status of campaign

diff --git a/docs/dimensioning.md b/docs/dimensioning.md
@@ -0,0 +1,47 @@
+ICTCore Dimensioning
+====================
+We have tested ICTCore on 4 CPU / 8 GB RAM / SSD HD instance and have collected following results
+
+Note: before testing it is required to increase file limit for freeswitch, which can be done by including following line in /lib/systemd/system/freeswitch.service
+
+    LimitNOFILE=100000
+    LimitNPROC=60000
+
+With single campaign
+--------------------
+A single campaign has produced 18 call per seconds which resulted in 400 active calls while tested with a message having 24 seconds duration. it mean system will be able to produce 800 concurrent calls if call duration will be 60 seconds
+
+* cps: 18
+* concurrent calls: 800
+
+
+With multiple campaigns
+-----------------------
+Multiple campaigns have produced 24 call per seconds which resulted in 600 active calls while tested with a message having 24 seconds duration. it mean system will be able to produce 1200 concurrent calls if call duration will be 60 seconds
+
+* cps: 26
+* concurrent calls: 1200
+
+
+Bottlenecks
+-----------
+
+We have identified following bottlenecks in our testing
+
+* Database server (MySQL)
+* Freeswitch dialplan (lua script)
+
+Recommendations
+---------------
+To further improve ICTCore performance it is recommended to 
+
+### Freeswitch dialplan
+* Use multiple and distributed Freeswitch instances
+* Replace lua scripts with something faster, or rewrite those script to improve performance
+
+### Database server
+* Use SSD hard-disks
+* Use a separate server for database (MySQL) hosting
+* For larger setups database load balancer can be used
+* Avoid database queries where ever possible
+
-Original file line number
+Diff line change
@@ Expand Up / @@ -95,4 +95,4 @@ public function config_reload() @@
         Corelog::log('Gateway->config_reload', Corelog::WARNING);
       }
-    }
+    }