diff --git a/console b/console index b9ade49..2f594bc 100755 --- a/console +++ b/console @@ -11,6 +11,7 @@ $console->setSilex($app); $console->add(new Pinboard\Command\InitCommand()); $console->add(new Pinboard\Command\AggregateCommand()); +$console->add(new Pinboard\Command\CleanCommand()); $console->add(new Pinboard\Command\AddUserCommand()); $app->register( diff --git a/src/Pinboard/Command/AggregateCommand.php b/src/Pinboard/Command/AggregateCommand.php index 8e78491..abf432a 100644 --- a/src/Pinboard/Command/AggregateCommand.php +++ b/src/Pinboard/Command/AggregateCommand.php @@ -28,6 +28,12 @@ protected function configure() ; } + protected function currentTime() + { + $now = new \DateTime(); + return $now->format('Y-m-d H:i:s'); + } + protected function initMailer() { if (isset($this->params['smtp'])) { @@ -137,6 +143,8 @@ protected function execute(InputInterface $input, OutputInterface $output) $db = $this->app['db']; + $output->writeln('[' . $this->currentTime() . '] Starting aggregation'); + try { $this->initMailer(); } @@ -182,46 +190,18 @@ protected function execute(InputInterface $input, OutputInterface $output) $now = new \DateTime(); $now = $now->format('Y-m-d H:i:s'); - $delta = new \DateInterval(isset($this->params['records_lifetime']) ? $this->params['records_lifetime'] : 'P1M'); - $date = new \DateTime(); - $date->sub($delta); - - $params = array( - 'created_at' => $date->format('Y-m-d H:i:s'), - ); - - $tablesForClear = array( - "ipm_report_2_by_hostname_and_server", - "ipm_report_by_hostname", - "ipm_report_by_hostname_and_server", - "ipm_report_by_server_name", - "ipm_req_time_details", - "ipm_mem_peak_usage_details", - "ipm_status_details", - "ipm_cpu_usage_details", - "ipm_timer", - ); - $sql = ''; + $sql .= 'TRUNCATE TABLE `ipm_request`;'; + $sql .= 'INSERT INTO `ipm_request` SELECT * FROM `request`;'; - foreach ($tablesForClear as $value) { - $sql .= ' - DELETE - FROM - ' . $value . ' - WHERE - created_at < :created_at - ;'; - } - if ($sql != '') - $db->executeQuery($sql, $params); + $db->executeQuery($sql); if (isset($this->params['notification']['enable']) && $this->params['notification']['enable']) { $sql = ' SELECT server_name, script_name, status, max(hostname) AS hostname, count(*) AS count FROM - request + ipm_request WHERE status >= 500 GROUP BY @@ -247,7 +227,7 @@ protected function execute(InputInterface $input, OutputInterface $output) SELECT server_name, hostname, COUNT(*) AS cnt FROM - request + ipm_request GROUP BY server_name, hostname '; @@ -259,7 +239,7 @@ protected function execute(InputInterface $input, OutputInterface $output) SELECT r.%s FROM - request r + ipm_request r WHERE r.server_name = r2.server_name AND r.hostname = r2.hostname ORDER BY @@ -297,7 +277,7 @@ protected function execute(InputInterface $input, OutputInterface $output) ' . sprintf($subselectTemplate, 'doc_size', 'doc_size', $server['cnt'] * (1 - 1.00), 'doc_size_100') . ', \'' . $now . '\' FROM - request r2 + ipm_request r2 WHERE r2.server_name = "' . $server['server_name'] . '" and r2.hostname = "' . $server['hostname'] . '" LIMIT 1 @@ -451,7 +431,7 @@ protected function execute(InputInterface $input, OutputInterface $output) SELECT server_name, hostname, script_name, status, tags, tags_cnt, FROM_UNIXTIME(max(timestamp)) FROM - request + ipm_request WHERE status >= 500 GROUP BY @@ -478,7 +458,7 @@ protected function execute(InputInterface $input, OutputInterface $output) SELECT id, server_name, hostname, script_name, max(req_time), max(mem_peak_usage), max(tags), max(tags_cnt), max(timers_cnt), FROM_UNIXTIME(timestamp) FROM - request + ipm_request WHERE server_name = "' . $server['server_name'] . '" AND hostname = "' . $server['hostname'] . '" AND req_time > ' . (float)$maxReqTime . ' GROUP BY @@ -518,7 +498,7 @@ protected function execute(InputInterface $input, OutputInterface $output) FROM timer t JOIN - request r ON t.request_id = r.id + ipm_request r ON t.request_id = r.id JOIN timertag tt ON tt.timer_id = t.id JOIN @@ -547,7 +527,7 @@ protected function execute(InputInterface $input, OutputInterface $output) SELECT server_name, hostname, script_name, max(mem_peak_usage), max(tags), max(tags_cnt), FROM_UNIXTIME(max(timestamp)) FROM - request + ipm_request WHERE server_name = "' . $server['server_name'] . '" AND hostname = "' . $server['hostname'] . '" AND mem_peak_usage > ' . (int)$maxMemoryUsage . ' GROUP BY @@ -577,7 +557,7 @@ protected function execute(InputInterface $input, OutputInterface $output) SELECT server_name, hostname, script_name, max(ru_utime), max(tags), max(tags_cnt), FROM_UNIXTIME(max(timestamp)) FROM - request + ipm_request WHERE server_name = "' . $server['server_name'] . '" AND hostname = "' . $server['hostname'] . '" AND ru_utime > ' . (int)$maxCPUUsage . ' GROUP BY @@ -595,7 +575,7 @@ protected function execute(InputInterface $input, OutputInterface $output) $values = $this->getBorderOutValues($db, $servers); $this->sendBorderOutEmails($values); - $output->writeln('Data are aggregated successfully'); + $output->writeln('[' . $this->currentTime() . '] Data are aggregated successfully'); if (!unlink( __FILE__ . '.lock')) { $output->writeln('Error: cannot remove ' . __FILE__ . '.lock file, you must remove it manually and check server settings.'); diff --git a/src/Pinboard/Command/CleanCommand.php b/src/Pinboard/Command/CleanCommand.php new file mode 100644 index 0000000..97bcf2b --- /dev/null +++ b/src/Pinboard/Command/CleanCommand.php @@ -0,0 +1,89 @@ +setName('clean') + ->setDescription('Clean old data from tables') + ; + } + + protected function currentTime() + { + $now = new \DateTime(); + return $now->format('Y-m-d H:i:s'); + } + + protected function execute(InputInterface $input, OutputInterface $output) + { + $this->app = $this->getApplication()->getSilex(); + $this->app->boot(); + $this->params = $this->app['params']; + $this->output = $output; + + $db = $this->app['db']; + + $output->writeln('[' . $this->currentTime() . '] Starting clean'); + + try { + $db->connect(); + } + catch(\PDOException $e) { + $output->writeln('Can\'t connect to MySQL server'); + + return; + } + + $delta = new \DateInterval(isset($this->params['records_lifetime']) ? $this->params['records_lifetime'] : 'P1M'); + $date = new \DateTime(); + $date->sub($delta); + + $params = array( + 'created_at' => $date->format('Y-m-d H:i:s'), + ); + + $tablesForClear = array( + "ipm_report_2_by_hostname_and_server", + "ipm_report_by_hostname", + "ipm_report_by_hostname_and_server", + "ipm_report_by_server_name", + "ipm_req_time_details", + "ipm_mem_peak_usage_details", + "ipm_status_details", + "ipm_cpu_usage_details", + "ipm_timer", + "ipm_tag_info", + ); + + + foreach ($tablesForClear as $value) { + $output->write('[' . $this->currentTime() . '] Clean "' . $value . '"....'); + $sql = ' + DELETE + FROM + ' . $value . ' + WHERE + created_at < :created_at + ;'; + + if ($db->executeQuery($sql, $params)->closeCursor()) { + $output->writeln('Done'); + } + } + + $output->writeln('[' . $this->currentTime() . '] Finished'); + } +} diff --git a/src/Pinboard/Command/InitCommand.php b/src/Pinboard/Command/InitCommand.php index aa32de1..7af2225 100644 --- a/src/Pinboard/Command/InitCommand.php +++ b/src/Pinboard/Command/InitCommand.php @@ -20,6 +20,7 @@ protected function configure() protected function execute(InputInterface $input, OutputInterface $output) { + $changeFlag = False; $output->writeln('Defining crontab task...'); $output->writeln('Please enter the frequency of data aggregating (frequency must be equal "pinba_stats_history" of the pinba engine config).'); @@ -46,24 +47,42 @@ function ($answer) { $crontabString = $process->isSuccessful() ? $process->getOutput() : ''; $path = realpath(__DIR__ . '/../../../console'); - $command = '*/' . $frequency . ' * * * * ' . $path . ' aggregate'; + $crontabString .= "\n# Pinboard aggregating & clean\n"; + + // Add the aggregate command in to crontab if not exist + $execTime = '*/' . $frequency . ' * * * * '; + $command = $path . ' aggregate'; if (strpos($crontabString, $command) === false) { - $crontabString .= "\n" . $command . "\n"; + $crontabString .= $execTime . $command . "\n"; + $changeFlag = True; } - $file = tempnam(sys_get_temp_dir(), 'ipm'); - file_put_contents($file, $crontabString); + // Add the clean command in to crontab if not exist + $execTime = '3 0 * * * '; + $command = $path . ' clean'; + if (strpos($crontabString, $command) === false) { + $crontabString .= $execTime . $command . "\n"; + $changeFlag = True; + } - $process = new Process('crontab ' . $file); - $process->setTimeout(20); - $process->run(); + if ($changeFlag) { + $file = tempnam(sys_get_temp_dir(), 'ipm'); + file_put_contents($file, $crontabString); - if (!$process->isSuccessful()) { - throw new \RuntimeException($process->getErrorOutput()); - } + $process = new Process('crontab ' . $file); + $process->setTimeout(20); + $process->run(); + + if (!$process->isSuccessful()) { + throw new \RuntimeException($process->getErrorOutput()); + } - $output->writeln('Crontab task are defined successfully'); - $output->writeln('Please set parameter "aggregation_period" to value "PT' . $frequency . 'M" in config/parameters.yml'); + $output->writeln('Crontab task are defined successfully'); + $output->writeln('Please set parameter "aggregation_period" to value "PT' . $frequency . 'M" in config/parameters.yml'); + } + else { + $output->writeln('The crons already exist'); + } } -} \ No newline at end of file +} diff --git a/src/Pinboard/Controller/server.php b/src/Pinboard/Controller/server.php old mode 100755 new mode 100644 diff --git a/src/Pinboard/DoctrineMigrations/Version20191018165422.php b/src/Pinboard/DoctrineMigrations/Version20191018165422.php new file mode 100644 index 0000000..49f703b --- /dev/null +++ b/src/Pinboard/DoctrineMigrations/Version20191018165422.php @@ -0,0 +1,41 @@ +addSql(" + CREATE TABLE `ipm_request` ( + `id` int(11) NOT NULL DEFAULT '0', + `hostname` varchar(32) DEFAULT NULL, + `req_count` int(11) DEFAULT NULL, + `server_name` varchar(64) DEFAULT NULL, + `script_name` varchar(128) DEFAULT NULL, + `doc_size` float DEFAULT NULL, + `mem_peak_usage` float DEFAULT NULL, + `req_time` float DEFAULT NULL, + `ru_utime` float DEFAULT NULL, + `ru_stime` float DEFAULT NULL, + `timers_cnt` int(11) DEFAULT NULL, + `status` int(11) DEFAULT NULL, + `memory_footprint` float DEFAULT NULL, + `schema` varchar(16) DEFAULT NULL, + `tags_cnt` int(11) DEFAULT NULL, + `tags` varchar(1024) DEFAULT NULL, + `timestamp` int(11) DEFAULT NULL, + KEY `ir_sss` (`server_name`,`script_name`,`status`), + KEY `ir_shst` (`server_name`,`hostname`,`script_name`,`timestamp`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8 + "); + } + + public function down(Schema $schema) + { + $this->addSql("DROP TABLE `ipm_request`;"); + } +} diff --git a/src/Pinboard/DoctrineMigrations/Version20191021124252.php b/src/Pinboard/DoctrineMigrations/Version20191021124252.php new file mode 100644 index 0000000..567cc47 --- /dev/null +++ b/src/Pinboard/DoctrineMigrations/Version20191021124252.php @@ -0,0 +1,22 @@ +addSql("ALTER TABLE `ipm_tag_info` ADD INDEX `iti_c` (`created_at`);"); + $this->addSql("ALTER TABLE `ipm_timer` ADD INDEX `it_c` (`created_at`);"); + } + + public function down(Schema $schema) + { + $this->addSql("ALTER TABLE `ipm_tag_info` DROP INDEX `iti_c`;"); + $this->addSql("ALTER TABLE `ipm_timer` DROP INDEX `it_c`;"); + } + +} diff --git a/views/live.html.twig b/views/live.html.twig old mode 100755 new mode 100644 diff --git a/views/server.html.twig b/views/server.html.twig index 46b371d..b8d0ee1 100644 --- a/views/server.html.twig +++ b/views/server.html.twig @@ -8,7 +8,7 @@

Request time view details

{% if req|length > 0 %} -
+