diff --git a/console b/console
index b9ade49..2f594bc 100755
--- a/console
+++ b/console
@@ -11,6 +11,7 @@ $console->setSilex($app);
$console->add(new Pinboard\Command\InitCommand());
$console->add(new Pinboard\Command\AggregateCommand());
+$console->add(new Pinboard\Command\CleanCommand());
$console->add(new Pinboard\Command\AddUserCommand());
$app->register(
diff --git a/src/Pinboard/Command/AggregateCommand.php b/src/Pinboard/Command/AggregateCommand.php
index 8e78491..abf432a 100644
--- a/src/Pinboard/Command/AggregateCommand.php
+++ b/src/Pinboard/Command/AggregateCommand.php
@@ -28,6 +28,12 @@ protected function configure()
;
}
+ protected function currentTime()
+ {
+ $now = new \DateTime();
+ return $now->format('Y-m-d H:i:s');
+ }
+
protected function initMailer()
{
if (isset($this->params['smtp'])) {
@@ -137,6 +143,8 @@ protected function execute(InputInterface $input, OutputInterface $output)
$db = $this->app['db'];
+ $output->writeln('[' . $this->currentTime() . '] Starting aggregation');
+
try {
$this->initMailer();
}
@@ -182,46 +190,18 @@ protected function execute(InputInterface $input, OutputInterface $output)
$now = new \DateTime();
$now = $now->format('Y-m-d H:i:s');
- $delta = new \DateInterval(isset($this->params['records_lifetime']) ? $this->params['records_lifetime'] : 'P1M');
- $date = new \DateTime();
- $date->sub($delta);
-
- $params = array(
- 'created_at' => $date->format('Y-m-d H:i:s'),
- );
-
- $tablesForClear = array(
- "ipm_report_2_by_hostname_and_server",
- "ipm_report_by_hostname",
- "ipm_report_by_hostname_and_server",
- "ipm_report_by_server_name",
- "ipm_req_time_details",
- "ipm_mem_peak_usage_details",
- "ipm_status_details",
- "ipm_cpu_usage_details",
- "ipm_timer",
- );
-
$sql = '';
+ $sql .= 'TRUNCATE TABLE `ipm_request`;';
+ $sql .= 'INSERT INTO `ipm_request` SELECT * FROM `request`;';
- foreach ($tablesForClear as $value) {
- $sql .= '
- DELETE
- FROM
- ' . $value . '
- WHERE
- created_at < :created_at
- ;';
- }
- if ($sql != '')
- $db->executeQuery($sql, $params);
+ $db->executeQuery($sql);
if (isset($this->params['notification']['enable']) && $this->params['notification']['enable']) {
$sql = '
SELECT
server_name, script_name, status, max(hostname) AS hostname, count(*) AS count
FROM
- request
+ ipm_request
WHERE
status >= 500
GROUP BY
@@ -247,7 +227,7 @@ protected function execute(InputInterface $input, OutputInterface $output)
SELECT
server_name, hostname, COUNT(*) AS cnt
FROM
- request
+ ipm_request
GROUP BY
server_name, hostname
';
@@ -259,7 +239,7 @@ protected function execute(InputInterface $input, OutputInterface $output)
SELECT
r.%s
FROM
- request r
+ ipm_request r
WHERE
r.server_name = r2.server_name AND r.hostname = r2.hostname
ORDER BY
@@ -297,7 +277,7 @@ protected function execute(InputInterface $input, OutputInterface $output)
' . sprintf($subselectTemplate, 'doc_size', 'doc_size', $server['cnt'] * (1 - 1.00), 'doc_size_100') . ',
\'' . $now . '\'
FROM
- request r2
+ ipm_request r2
WHERE
r2.server_name = "' . $server['server_name'] . '" and r2.hostname = "' . $server['hostname'] . '"
LIMIT 1
@@ -451,7 +431,7 @@ protected function execute(InputInterface $input, OutputInterface $output)
SELECT
server_name, hostname, script_name, status, tags, tags_cnt, FROM_UNIXTIME(max(timestamp))
FROM
- request
+ ipm_request
WHERE
status >= 500
GROUP BY
@@ -478,7 +458,7 @@ protected function execute(InputInterface $input, OutputInterface $output)
SELECT
id, server_name, hostname, script_name, max(req_time), max(mem_peak_usage), max(tags), max(tags_cnt), max(timers_cnt), FROM_UNIXTIME(timestamp)
FROM
- request
+ ipm_request
WHERE
server_name = "' . $server['server_name'] . '" AND hostname = "' . $server['hostname'] . '" AND req_time > ' . (float)$maxReqTime . '
GROUP BY
@@ -518,7 +498,7 @@ protected function execute(InputInterface $input, OutputInterface $output)
FROM
timer t
JOIN
- request r ON t.request_id = r.id
+ ipm_request r ON t.request_id = r.id
JOIN
timertag tt ON tt.timer_id = t.id
JOIN
@@ -547,7 +527,7 @@ protected function execute(InputInterface $input, OutputInterface $output)
SELECT
server_name, hostname, script_name, max(mem_peak_usage), max(tags), max(tags_cnt), FROM_UNIXTIME(max(timestamp))
FROM
- request
+ ipm_request
WHERE
server_name = "' . $server['server_name'] . '" AND hostname = "' . $server['hostname'] . '" AND mem_peak_usage > ' . (int)$maxMemoryUsage . '
GROUP BY
@@ -577,7 +557,7 @@ protected function execute(InputInterface $input, OutputInterface $output)
SELECT
server_name, hostname, script_name, max(ru_utime), max(tags), max(tags_cnt), FROM_UNIXTIME(max(timestamp))
FROM
- request
+ ipm_request
WHERE
server_name = "' . $server['server_name'] . '" AND hostname = "' . $server['hostname'] . '" AND ru_utime > ' . (int)$maxCPUUsage . '
GROUP BY
@@ -595,7 +575,7 @@ protected function execute(InputInterface $input, OutputInterface $output)
$values = $this->getBorderOutValues($db, $servers);
$this->sendBorderOutEmails($values);
- $output->writeln('Data are aggregated successfully');
+ $output->writeln('[' . $this->currentTime() . '] Data are aggregated successfully');
if (!unlink( __FILE__ . '.lock')) {
$output->writeln('Error: cannot remove ' . __FILE__ . '.lock file, you must remove it manually and check server settings.');
diff --git a/src/Pinboard/Command/CleanCommand.php b/src/Pinboard/Command/CleanCommand.php
new file mode 100644
index 0000000..97bcf2b
--- /dev/null
+++ b/src/Pinboard/Command/CleanCommand.php
@@ -0,0 +1,89 @@
+setName('clean')
+ ->setDescription('Clean old data from tables')
+ ;
+ }
+
+ protected function currentTime()
+ {
+ $now = new \DateTime();
+ return $now->format('Y-m-d H:i:s');
+ }
+
+ protected function execute(InputInterface $input, OutputInterface $output)
+ {
+ $this->app = $this->getApplication()->getSilex();
+ $this->app->boot();
+ $this->params = $this->app['params'];
+ $this->output = $output;
+
+ $db = $this->app['db'];
+
+ $output->writeln('[' . $this->currentTime() . '] Starting clean');
+
+ try {
+ $db->connect();
+ }
+ catch(\PDOException $e) {
+ $output->writeln('Can\'t connect to MySQL server');
+
+ return;
+ }
+
+ $delta = new \DateInterval(isset($this->params['records_lifetime']) ? $this->params['records_lifetime'] : 'P1M');
+ $date = new \DateTime();
+ $date->sub($delta);
+
+ $params = array(
+ 'created_at' => $date->format('Y-m-d H:i:s'),
+ );
+
+ $tablesForClear = array(
+ "ipm_report_2_by_hostname_and_server",
+ "ipm_report_by_hostname",
+ "ipm_report_by_hostname_and_server",
+ "ipm_report_by_server_name",
+ "ipm_req_time_details",
+ "ipm_mem_peak_usage_details",
+ "ipm_status_details",
+ "ipm_cpu_usage_details",
+ "ipm_timer",
+ "ipm_tag_info",
+ );
+
+
+ foreach ($tablesForClear as $value) {
+ $output->write('[' . $this->currentTime() . '] Clean "' . $value . '"....');
+ $sql = '
+ DELETE
+ FROM
+ ' . $value . '
+ WHERE
+ created_at < :created_at
+ ;';
+
+ if ($db->executeQuery($sql, $params)->closeCursor()) {
+ $output->writeln('Done');
+ }
+ }
+
+ $output->writeln('[' . $this->currentTime() . '] Finished');
+ }
+}
diff --git a/src/Pinboard/Command/InitCommand.php b/src/Pinboard/Command/InitCommand.php
index aa32de1..7af2225 100644
--- a/src/Pinboard/Command/InitCommand.php
+++ b/src/Pinboard/Command/InitCommand.php
@@ -20,6 +20,7 @@ protected function configure()
protected function execute(InputInterface $input, OutputInterface $output)
{
+ $changeFlag = False;
$output->writeln('Defining crontab task...');
$output->writeln('Please enter the frequency of data aggregating (frequency must be equal "pinba_stats_history" of the pinba engine config).');
@@ -46,24 +47,42 @@ function ($answer) {
$crontabString = $process->isSuccessful() ? $process->getOutput() : '';
$path = realpath(__DIR__ . '/../../../console');
- $command = '*/' . $frequency . ' * * * * ' . $path . ' aggregate';
+ $crontabString .= "\n# Pinboard aggregating & clean\n";
+
+ // Add the aggregate command in to crontab if not exist
+ $execTime = '*/' . $frequency . ' * * * * ';
+ $command = $path . ' aggregate';
if (strpos($crontabString, $command) === false) {
- $crontabString .= "\n" . $command . "\n";
+ $crontabString .= $execTime . $command . "\n";
+ $changeFlag = True;
}
- $file = tempnam(sys_get_temp_dir(), 'ipm');
- file_put_contents($file, $crontabString);
+ // Add the clean command in to crontab if not exist
+ $execTime = '3 0 * * * ';
+ $command = $path . ' clean';
+ if (strpos($crontabString, $command) === false) {
+ $crontabString .= $execTime . $command . "\n";
+ $changeFlag = True;
+ }
- $process = new Process('crontab ' . $file);
- $process->setTimeout(20);
- $process->run();
+ if ($changeFlag) {
+ $file = tempnam(sys_get_temp_dir(), 'ipm');
+ file_put_contents($file, $crontabString);
- if (!$process->isSuccessful()) {
- throw new \RuntimeException($process->getErrorOutput());
- }
+ $process = new Process('crontab ' . $file);
+ $process->setTimeout(20);
+ $process->run();
+
+ if (!$process->isSuccessful()) {
+ throw new \RuntimeException($process->getErrorOutput());
+ }
- $output->writeln('Crontab task are defined successfully');
- $output->writeln('Please set parameter "aggregation_period" to value "PT' . $frequency . 'M" in config/parameters.yml');
+ $output->writeln('Crontab task are defined successfully');
+ $output->writeln('Please set parameter "aggregation_period" to value "PT' . $frequency . 'M" in config/parameters.yml');
+ }
+ else {
+ $output->writeln('The crons already exist');
+ }
}
-}
\ No newline at end of file
+}
diff --git a/src/Pinboard/Controller/server.php b/src/Pinboard/Controller/server.php
old mode 100755
new mode 100644
diff --git a/src/Pinboard/DoctrineMigrations/Version20191018165422.php b/src/Pinboard/DoctrineMigrations/Version20191018165422.php
new file mode 100644
index 0000000..49f703b
--- /dev/null
+++ b/src/Pinboard/DoctrineMigrations/Version20191018165422.php
@@ -0,0 +1,41 @@
+addSql("
+ CREATE TABLE `ipm_request` (
+ `id` int(11) NOT NULL DEFAULT '0',
+ `hostname` varchar(32) DEFAULT NULL,
+ `req_count` int(11) DEFAULT NULL,
+ `server_name` varchar(64) DEFAULT NULL,
+ `script_name` varchar(128) DEFAULT NULL,
+ `doc_size` float DEFAULT NULL,
+ `mem_peak_usage` float DEFAULT NULL,
+ `req_time` float DEFAULT NULL,
+ `ru_utime` float DEFAULT NULL,
+ `ru_stime` float DEFAULT NULL,
+ `timers_cnt` int(11) DEFAULT NULL,
+ `status` int(11) DEFAULT NULL,
+ `memory_footprint` float DEFAULT NULL,
+ `schema` varchar(16) DEFAULT NULL,
+ `tags_cnt` int(11) DEFAULT NULL,
+ `tags` varchar(1024) DEFAULT NULL,
+ `timestamp` int(11) DEFAULT NULL,
+ KEY `ir_sss` (`server_name`,`script_name`,`status`),
+ KEY `ir_shst` (`server_name`,`hostname`,`script_name`,`timestamp`)
+ ) ENGINE=InnoDB DEFAULT CHARSET=utf8
+ ");
+ }
+
+ public function down(Schema $schema)
+ {
+ $this->addSql("DROP TABLE `ipm_request`;");
+ }
+}
diff --git a/src/Pinboard/DoctrineMigrations/Version20191021124252.php b/src/Pinboard/DoctrineMigrations/Version20191021124252.php
new file mode 100644
index 0000000..567cc47
--- /dev/null
+++ b/src/Pinboard/DoctrineMigrations/Version20191021124252.php
@@ -0,0 +1,22 @@
+addSql("ALTER TABLE `ipm_tag_info` ADD INDEX `iti_c` (`created_at`);");
+ $this->addSql("ALTER TABLE `ipm_timer` ADD INDEX `it_c` (`created_at`);");
+ }
+
+ public function down(Schema $schema)
+ {
+ $this->addSql("ALTER TABLE `ipm_tag_info` DROP INDEX `iti_c`;");
+ $this->addSql("ALTER TABLE `ipm_timer` DROP INDEX `it_c`;");
+ }
+
+}
diff --git a/views/live.html.twig b/views/live.html.twig
old mode 100755
new mode 100644
diff --git a/views/server.html.twig b/views/server.html.twig
index 46b371d..b8d0ee1 100644
--- a/views/server.html.twig
+++ b/views/server.html.twig
@@ -8,7 +8,7 @@
{% if req|length > 0 %}
-
+