Merge pull request #51 from CESNET/postgresql

melanger · web-flow · commit 63f4b49846a5 · 2021-11-22T12:05:58.000+01:00
feat: adapt for PostgreSQL
diff --git a/composer.json b/composer.json
@@ -16,8 +16,6 @@
   ],
   "require": {
     "php": ">=7.1",
-    "simplesamlphp/simplesamlphp": "~1.17",
-    "ext-pdo": "*",
     "ext-json": "*"
   },
   "suggest": {
diff --git a/config-templates/tables.sql b/config-templates/tables.sql
@@ -6,14 +6,14 @@ CREATE TABLE `statistics_sums` (
   `year` YEAR NOT NULL,
   `month` TINYINT UNSIGNED DEFAULT NULL,
   `day` TINYINT UNSIGNED DEFAULT NULL,
-  `idpId` INT UNSIGNED NOT NULL DEFAULT 0,
-  `spId` INT UNSIGNED NOT NULL DEFAULT 0,
+  `idp_id` INT UNSIGNED NOT NULL DEFAULT 0,
+  `sp_id` INT UNSIGNED NOT NULL DEFAULT 0,
   `logins` INT UNSIGNED DEFAULT NULL,
   `users` INT UNSIGNED DEFAULT NULL,
   PRIMARY KEY (`id`),
-  UNIQUE KEY `year` (`year`,`month`,`day`,`idpId`,`spId`),
-  KEY `idpId` (`idpId`),
-  KEY `spId` (`spId`)
+  UNIQUE KEY `year` (`year`,`month`,`day`,`idp_id`,`sp_id`),
+  KEY `idp_id` (`idp_id`),
+  KEY `sp_id` (`sp_id`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8;
 -- ROWS
 -- each row contains daily users = COUNT(1) and daily logins = SUM(logins) from statistics_per_user
@@ -27,29 +27,29 @@ CREATE TABLE `statistics_sums` (
 -- data is being kept for ~1 month
 CREATE TABLE `statistics_per_user` (
   `day` date NOT NULL,
-  `idpId` INT UNSIGNED NOT NULL,
-  `spId` INT UNSIGNED NOT NULL,
+  `idp_id` INT UNSIGNED NOT NULL,
+  `sp_id` INT UNSIGNED NOT NULL,
   `user` VARCHAR(255) NOT NULL,
   `logins` INT UNSIGNED DEFAULT '1',
-  PRIMARY KEY (`day`,`idpId`,`spId`,`user`),
-  KEY `idpId` (`idpId`),
-  KEY `spId` (`spId`)
+  PRIMARY KEY (`day`,`idp_id`,`sp_id`,`user`),
+  KEY `idp_id` (`idp_id`),
+  KEY `sp_id` (`sp_id`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8;
 
 -- identity providers
 CREATE TABLE `statistics_idp` (
-  `idpId` INT UNSIGNED NOT NULL AUTO_INCREMENT,
+  `idp_id` INT UNSIGNED NOT NULL AUTO_INCREMENT,
   `identifier` VARCHAR(255) NOT NULL,
   `name` VARCHAR(255) NOT NULL,
-  PRIMARY KEY (`idpId`),
+  PRIMARY KEY (`idp_id`),
   UNIQUE KEY `identifier` (`identifier`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8;
 
 -- services
 CREATE TABLE `statistics_sp` (
-  `spId` INT UNSIGNED NOT NULL AUTO_INCREMENT,
+  `sp_id` INT UNSIGNED NOT NULL AUTO_INCREMENT,
   `identifier` VARCHAR(255) NOT NULL,
   `name` VARCHAR(255) NOT NULL,
-  PRIMARY KEY (`spId`),
+  PRIMARY KEY (`sp_id`),
   UNIQUE KEY `identifier` (`identifier`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8;
diff --git a/lib/DatabaseCommand.php b/lib/DatabaseCommand.php
@@ -24,8 +24,8 @@ class DatabaseCommand
     ];
 
     private const TABLE_IDS = [
-        self::TABLE_IDP => 'idpId',
-        self::TABLE_SP => 'spId',
+        self::TABLE_IDP => 'idp_id',
+        self::TABLE_SP => 'sp_id',
     ];
 
     private $tables = [
@@ -41,11 +41,15 @@ class DatabaseCommand
 
     private $mode;
 
+    private $escape_char = '`';
+
     public function __construct()
     {
         $this->config = Config::getInstance();
         $this->conn = Database::getInstance($this->config->getStore());
-        assert($this->conn !== null);
+        if ($this->conn->getDriver() === 'pgsql') {
+            $this->escape_char = '"';
+        }
         $this->tables = array_merge($this->tables, $this->config->getTables());
         $this->mode = $this->config->getMode();
     }
@@ -84,7 +88,7 @@ public function getNameById($side, $id)
     {
         $table = self::TABLE_SIDES[$side];
         return $this->read(
-            'SELECT IFNULL(name, identifier) ' .
+            'SELECT COALESCE(name, identifier) ' .
             'FROM ' . $this->tables[$table] . ' ' .
             'WHERE ' . self::TABLE_IDS[$table] . '=:id',
             [
@@ -96,16 +100,20 @@ public function getNameById($side, $id)
     public function getLoginCountPerDay($days, $where = [])
     {
         $params = [];
-        $query = 'SELECT UNIX_TIMESTAMP(STR_TO_DATE(CONCAT(year,"-",month,"-",day), "%Y-%m-%d")) AS day, ' .
-                 'logins AS count, users ' .
+        if ($this->conn->getDriver() === 'pgsql') {
+            $query = "SELECT EXTRACT(epoch FROM TO_DATE(CONCAT(year,'-',month,'-',day), 'YYYY-MM-DD')) AS day, ";
+        } else {
+            $query = "SELECT UNIX_TIMESTAMP(STR_TO_DATE(CONCAT(year,'-',month,'-',day), '%Y-%m-%d')) AS day, ";
+        }
+        $query .= 'logins AS count, users ' .
                  'FROM ' . $this->tables[self::TABLE_SUM] . ' ' .
                  'WHERE ';
         $where = array_merge([
             Config::MODE_SP => null,
             Config::MODE_IDP => null,
         ], $where);
-        self::addWhereId($where, $query, $params);
-        self::addDaysRange($days, $query, $params);
+        $this->addWhereId($where, $query, $params);
+        $this->addDaysRange($days, $query, $params);
         $query .= //'GROUP BY day ' .
                   'ORDER BY day ASC';
 
@@ -117,13 +125,13 @@ public function getAccessCount($side, $days, $where = [])
     {
         $table = self::TABLE_SIDES[$side];
         $params = [];
-        $query = 'SELECT IFNULL(name,identifier) AS name, ' . self::TABLE_IDS[$table] . ', SUM(logins) AS count ' .
+        $query = 'SELECT COALESCE(name,identifier) AS name, ' . self::TABLE_IDS[$table] . ', SUM(logins) AS count ' .
                  'FROM ' . $this->tables[$table] . ' ' .
                  'LEFT OUTER JOIN ' . $this->tables[self::TABLE_SUM] . ' ' .
                  'USING (' . self::TABLE_IDS[$table] . ') ' .
                  'WHERE ';
-        self::addWhereId($where, $query, $params);
-        self::addDaysRange($days, $query, $params);
+        $this->addWhereId($where, $query, $params);
+        $this->addDaysRange($days, $query, $params);
         $query .= 'GROUP BY ' . self::TABLE_IDS[$table] . ' ';
         $query .= 'ORDER BY SUM(logins) DESC';
 
@@ -133,22 +141,32 @@ public function getAccessCount($side, $days, $where = [])
 
     public function aggregate()
     {
-        foreach ([self::TABLE_IDS[self::TABLE_IDP], null] as $idpId) {
-            foreach ([self::TABLE_IDS[self::TABLE_SP], null] as $spId) {
-                $ids = [$idpId, $spId];
+        foreach ([self::TABLE_IDS[self::TABLE_IDP], null] as $idp_id) {
+            foreach ([self::TABLE_IDS[self::TABLE_SP], null] as $sp_id) {
+                $ids = [$idp_id, $sp_id];
                 $msg = 'Aggregating daily statistics per ' . implode(' and ', array_filter($ids));
                 Logger::info($msg);
                 $query = 'INSERT INTO ' . $this->tables[self::TABLE_SUM] . ' '
-                    . '(`year`,`month`,`day`,idpId,spId,logins,users) '
-                    . 'SELECT YEAR(`day`), MONTH(`day`), DAY(`day`), ';
+                    . '(' . $this->escape_cols(['year', 'month', 'day', 'idp_id', 'sp_id', 'logins', 'users']) . ') '
+                    . 'SELECT EXTRACT(YEAR FROM ' . $this->escape_col(
+                        'day'
+                    ) . '), EXTRACT(MONTH FROM ' . $this->escape_col(
+                        'day'
+                    ) . '), EXTRACT(DAY FROM ' . $this->escape_col('day') . '), ';
                 foreach ($ids as $id) {
                     $query .= ($id === null ? '0' : $id) . ',';
                 }
-                $query .= 'SUM(logins), COUNT(DISTINCT user) '
+                $query .= 'SUM(logins), COUNT(DISTINCT ' . $this->escape_col('user') . ') '
                     . 'FROM ' . $this->tables[self::TABLE_PER_USER] . ' '
                     . 'WHERE day<DATE(NOW()) '
-                    . 'GROUP BY ' . self::getAggregateGroupBy($ids) . ' '
-                    . 'ON DUPLICATE KEY UPDATE id=id;';
+                    . 'GROUP BY ' . $this->getAggregateGroupBy($ids) . ' ';
+                if ($this->conn->getDriver() === 'pgsql') {
+                    $query .= 'ON CONFLICT (' . $this->escape_cols(
+                        ['year', 'month', 'day', 'idp_id', 'sp_id']
+                    ) . ') DO NOTHING;';
+                } else {
+                    $query .= 'ON DUPLICATE KEY UPDATE id=id;';
+                }
                 // do nothing if row already exists
                 if (! $this->conn->write($query)) {
                     Logger::warning($msg . ' failed');
@@ -160,30 +178,52 @@ public function aggregate()
 
         $msg = 'Deleting detailed statistics';
         Logger::info($msg);
-        // INNER JOIN ensures that only aggregated stats are deleted
+        if ($this->conn->getDriver() === 'pgsql') {
+            $make_date = 'MAKE_DATE(' . $this->escape_cols(['year', 'month', 'day']) . ')';
+            $date_clause = sprintf('CURRENT_DATE - INTERVAL \'%s DAY\' ', $keepPerUserDays);
+            $params = [];
+        } else {
+            $make_date = 'STR_TO_DATE(CONCAT(' . $this->escape_col('year') . ",'-'," . $this->escape_col(
+                'month'
+            ) . ",'-'," . $this->escape_col('day') . "), '%Y-%m-%d')";
+            $date_clause = 'CURDATE() - INTERVAL :days DAY';
+            $params = [
+                'days' => $keepPerUserDays,
+            ];
+        }
+        $query = 'DELETE FROM ' . $this->tables[self::TABLE_PER_USER] . ' WHERE ' . $this->escape_col(
+            'day'
+        ) . ' < ' . $date_clause
+        . ' AND ' . $this->escape_col(
+            'day'
+        ) . ' IN (SELECT ' . $make_date . ' FROM ' . $this->tables[self::TABLE_SUM] . ')';
         if (
-            ! $this->conn->write(
-                'DELETE u FROM ' . $this->tables[self::TABLE_PER_USER] . ' AS u '
-                . 'INNER JOIN ' . $this->tables[self::TABLE_SUM] . ' AS s '
-                . 'ON YEAR(u.`day`)=s.`year` AND MONTH(u.`day`)=s.`month` AND DAY(u.`day`)=s.`day`'
-                . 'WHERE u.`day` < CURDATE() - INTERVAL :days DAY',
-                [
-                    'days' => $keepPerUserDays,
-                ]
-            )
+            ! $this->conn->write($query, $params)
         ) {
             Logger::warning($msg . ' failed');
         }
     }
 
+    private function escape_col($col_name)
+    {
+        return $this->escape_char . $col_name . $this->escape_char;
+    }
+
+    private function escape_cols($col_names)
+    {
+        return $this->escape_char . implode(
+            $this->escape_char . ',' . $this->escape_char,
+            $col_names
+        ) . $this->escape_char;
+    }
+
     private function read($query, $params)
     {
         return $this->conn->read($query, $params);
     }
 
-    private static function addWhereId($where, &$query, &$params)
+    private function addWhereId($where, &$query, &$params)
     {
-        assert(count(array_filter($where)) <= 1); //placeholder would be overwritten
         $parts = [];
         foreach ($where as $side => $value) {
             $table = self::TABLE_SIDES[$side];
@@ -216,8 +256,15 @@ private function writeLogin($date, $ids, $user)
         ]);
         $fields = array_keys($params);
         $placeholders = array_map(['self', 'prependColon'], $fields);
-        $query = 'INSERT INTO ' . $this->tables[self::TABLE_PER_USER] . ' (' . implode(', ', $fields) . ')' .
-                 ' VALUES (' . implode(', ', $placeholders) . ') ON DUPLICATE KEY UPDATE logins = logins + 1';
+        $query = 'INSERT INTO ' . $this->tables[self::TABLE_PER_USER] . ' (' . $this->escape_cols($fields) . ')' .
+                 ' VALUES (' . implode(', ', $placeholders) . ') ';
+        if ($this->conn->getDriver() === 'pgsql') {
+            $query .= 'ON CONFLICT (' . $this->escape_cols(
+                ['day', 'idp_id', 'sp_id', 'user']
+            ) . ') DO UPDATE SET "logins" = ' . $this->tables[self::TABLE_PER_USER] . '.logins + 1;';
+        } else {
+            $query .= 'ON DUPLICATE KEY UPDATE logins = logins + 1;';
+        }
 
         return $this->conn->write($query, $params);
     }
@@ -262,47 +309,60 @@ private function getIdFromIdentifier($table, $entity, $idColumn)
     {
         $identifier = $entity['id'];
         $name = $entity['name'];
-        $this->conn->write(
-            'INSERT INTO ' . $this->tables[$table]
-            . '(identifier, name) VALUES (:identifier, :name1) ON DUPLICATE KEY UPDATE name = :name2',
-            [
-                'identifier' => $identifier,
-                'name1' => $name,
-                'name2' => $name,
-            ]
-        );
+        $query = 'INSERT INTO ' . $this->tables[$table] . '(identifier, name) VALUES (:identifier, :name1) ';
+        if ($this->conn->getDriver() === 'pgsql') {
+            $query .= 'ON CONFLICT (identifier) DO UPDATE SET name = :name2;';
+        } else {
+            $query .= 'ON DUPLICATE KEY UPDATE name = :name2';
+        }
+        $this->conn->write($query, [
+            'identifier' => $identifier,
+            'name1' => $name,
+            'name2' => $name,
+        ]);
         return $this->read('SELECT ' . $idColumn . ' FROM ' . $this->tables[$table]
             . ' WHERE identifier=:identifier', [
                 'identifier' => $identifier,
             ])
             ->fetchColumn();
     }
 
-    private static function addDaysRange($days, &$query, &$params, $not = false)
+    private function addDaysRange($days, &$query, &$params, $not = false)
     {
         if ($days !== 0) {    // 0 = all time
             if (stripos($query, 'WHERE') === false) {
                 $query .= 'WHERE';
             } else {
                 $query .= 'AND';
             }
-            $query .= ' CONCAT(year,"-",LPAD(month,2,"00"),"-",LPAD(day,2,"00")) ';
+            if ($this->conn->getDriver() === 'pgsql') {
+                $query .= ' MAKE_DATE(year,month,day) ';
+            } else {
+                $query .= " CONCAT(year,'-',LPAD(month,2,'00'),'-',LPAD(day,2,'00')) ";
+            }
             if ($not) {
                 $query .= 'NOT ';
             }
-            $query .= 'BETWEEN CURDATE() - INTERVAL :days DAY AND CURDATE() ';
-            $params['days'] = $days;
+            if ($this->conn->getDriver() === 'pgsql') {
+                if (! is_int($days) && ! ctype_digit($days)) {
+                    throw new \Exception('days have to be an integer');
+                }
+                $query .= sprintf('BETWEEN CURRENT_DATE - INTERVAL \'%s DAY\' AND CURRENT_DATE ', $days);
+            } else {
+                $query .= 'BETWEEN CURDATE() - INTERVAL :days DAY AND CURDATE() ';
+                $params['days'] = $days;
+            }
         }
     }
 
-    private static function getAggregateGroupBy($ids)
+    private function getAggregateGroupBy($ids)
     {
         $columns = ['day'];
         foreach ($ids as $id) {
             if ($id !== null) {
                 $columns[] = $id;
             }
         }
-        return '`' . implode('`,`', $columns) . '`';
+        return $this->escape_cols($columns);
     }
 }
diff --git a/scripts/migrate_v6_mysql.sql b/scripts/migrate_v6_mysql.sql
@@ -0,0 +1,9 @@
+ALTER TABLE statistics_sums CHANGE idpId idp_id INT UNSIGNED NOT NULL DEFAULT 0;
+ALTER TABLE statistics_sums CHANGE spId sp_id INT UNSIGNED NOT NULL DEFAULT 0;
+
+ALTER TABLE statistics_per_user CHANGE idpId idp_id INT UNSIGNED NOT NULL DEFAULT 0;
+ALTER TABLE statistics_per_user CHANGE spId sp_id INT UNSIGNED NOT NULL DEFAULT 0;
+
+ALTER TABLE statistics_idp CHANGE idpId idp_id INT UNSIGNED NOT NULL DEFAULT 0;
+
+ALTER TABLE statistics_sp CHANGE spId sp_id INT UNSIGNED NOT NULL DEFAULT 0;
diff --git a/scripts/migrate_v6_pgsql.sql b/scripts/migrate_v6_pgsql.sql
@@ -0,0 +1,9 @@
+ALTER TABLE statistics_sums RENAME "idpId" TO idp_id;
+ALTER TABLE statistics_sums RENAME "spId" TO sp_id;
+
+ALTER TABLE statistics_per_user RENAME "idpId" TO idp_id;
+ALTER TABLE statistics_per_user RENAME "spId" TO sp_id;
+
+ALTER TABLE statistics_idp RENAME "idpId" TO idp_id;
+
+ALTER TABLE statistics_sp RENAME "spId" TO sp_id;
diff --git a/www/assets/js/index.js b/www/assets/js/index.js
@@ -8,8 +8,10 @@ function getStatisticsData(name) {
 
 function getStatisticsDataYMDC(name, field) {
   return getStatisticsData(name).map(function mapItemToDate(item) {
+    var d = new Date(item.day * 1000);
+    d.setHours(0);
     return {
-      x: new Date(item.day * 1000),
+      x: d,
       y: item[field]
     };
   });
@@ -22,7 +24,7 @@ function getTranslation(str) {
 function extendData(data, minX, maxX) {
   var i = 0;
   var extendedData = [];
-  for (var d = new Date(minX); d <= maxX; d.setDate(d.getDate() + 1)) {
+  for (var d = new Date(minX); d <= maxX; d.setDate(d.getDate() + 1),d.setHours(0)) {
     if (data[i].x.getTime() === d.getTime()) {
       extendedData.push(data[i]);
       i += 1;
