Skip to content

Commit 5039386

Browse files
evens-stoneevens-stone
committed
Improved sniffer, scrapers and providers
1 parent fa5ce8e commit 5039386

File tree

9 files changed

+189
-43
lines changed

9 files changed

+189
-43
lines changed

‎README.md

Lines changed: 25 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -109,15 +109,24 @@ $urlSeeds = GoogleProvider::create($config)->fetch($query, $meta);
109109
1. Your provider must implement `Snippetify\SnippetSniffer\Providers\ProviderInterface`
110110
2. Take a look at `Snippetify\SnippetSniffer\Providers\GoogleProvider` to get you helped
111111
3. Your fetch method must return an array of `GuzzleHttp\Psr7\Uri`
112-
4. Pass your new provider in the configuration parameter as follow
112+
4. Pass your new provider in the configuration parameter or use the `addProvider` method
113113

114114
```php
115-
// Configurations
115+
use Snippetify\SnippetSniffer\SnippetSniffer;
116+
117+
// Use Configurations
116118
$config = [
117119
"providers" => [
118-
"provider_name" => "ProviderClass::class"
120+
"provider_name" => "ProviderClass::class",
121+
"provider_2_name" => "Provider2Class::class" // You can add as many as you want
119122
]
120123
];
124+
125+
// Or use addProvider method as follow
126+
SnippetSniffer::create(...)
127+
->addProvider('provider_name', 'ProviderClass::class')
128+
->addProvider('provider_2_name', 'Provider2Class::class') // You can add as many as you want
129+
...
121130
```
122131

123132
#### Scrapers
@@ -168,16 +177,26 @@ $snippets = (new DefaultScraper($config))->fetch(new Uri($urlSeed));
168177
1. Your scraper must implement `Snippetify\SnippetSniffer\Scrapers\ScraperInterface`
169178
2. Take a look at `Snippetify\SnippetSniffer\Scrapers\StackoverflowScraper` to get you helped
170179
3. Your fetch method must return an array of `Snippetify\SnippetSniffer\Common\Snippet`
171-
4. Pass your new scraper in the configuration parameter as follow
180+
4. Pass your new scraper in the configuration parameter or use the `addScraper` method
172181

173182
```php
174-
// Configurations
183+
use Snippetify\SnippetSniffer\SnippetSniffer;
184+
175185
// Important: Scrapper's name must be the website uri without the scheme. i.e. vuejs.org
186+
187+
// Configurations
176188
$config = [
177189
"scrapers" => [
178-
"scraper_name" => "ScraperClass::class"
190+
"scraper_name" => "ScraperClass::class",
191+
"scraper_2_name" => "Scraper2Class::class" // You can add as many as you want
179192
]
180193
];
194+
195+
// Or use addProvider method as follow
196+
SnippetSniffer::create(...)
197+
->addScraper('scraper_name', 'ScraperClass::class')
198+
->addScraper('scraper_2_name', 'Scraper2Class::class') // You can add as many as you want
199+
...
181200
```
182201

183202
## Changelog

‎src/Common/Logger.php

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ public function __construct(array $config = [])
5151
$this->config = $config;
5252

5353
if (empty($config['name'])) $this->config['name'] = self::NAME;
54-
if (empty($config['file'])) $this->config['file'] = dirname(__FILE__).'/../../logs/'.self::LOG_FILE;
54+
if (empty($config['file'])) $this->config['file'] = dirname(__FILE__) . '/../../logs/'.self::LOG_FILE;
5555

5656
$this->logger = new BaseLogger($this->config['name']);
5757
}

‎src/Core.php

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ class Core
1717
*/
1818
public static $providers = [
1919
'google' => \Snippetify\SnippetSniffer\Providers\GoogleProvider::class,
20-
'gigablast' => \Snippetify\SnippetSniffer\Providers\GigablastProvider::class,
20+
// 'gigablast' => \Snippetify\SnippetSniffer\Providers\GigablastProvider::class,
2121
];
2222

2323
/**

‎src/Providers/GoogleProvider.php

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
use Symfony\Component\HttpClient\Exception\ClientException;
88
use Symfony\Component\HttpClient\Exception\InvalidArgumentException;
99

10-
class GoogleProvider implements ProviderInterface
10+
final class GoogleProvider implements ProviderInterface
1111
{
1212
const API_URI = 'https://www.googleapis.com/customsearch/v1';
1313

@@ -16,7 +16,7 @@ class GoogleProvider implements ProviderInterface
1616
*
1717
* @var string
1818
*/
19-
protected $config;
19+
private $config;
2020

2121
/**
2222
* Singletion.
@@ -85,7 +85,7 @@ public function fetch(string $query, array $meta = []): array
8585
'start' => empty($meta['page']) || empty($meta['limit']) ? 1 :
8686
(($meta['page'] * $meta['limit']) - $meta['limit']) + 1
8787
]);
88-
$response = HttpClient::create()->request('GET', self::API_URI."?$query");
88+
$response = HttpClient::create()->request('GET', self::API_URI . "?{$query}");
8989

9090
if (200 !== $response->getStatusCode()) {
9191
throw new ClientException($response);

‎src/Scrapers/AbstractScraper.php

Lines changed: 26 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
use Psr\Http\Message\UriInterface;
88
use Snippetify\SnippetSniffer\Core;
99
use Symfony\Component\DomCrawler\Crawler;
10+
use Snippetify\SnippetSniffer\Common\Logger;
1011
use Snippetify\SnippetSniffer\Common\Snippet;
1112
use Snippetify\ProgrammingLanguages\Facades\Languages;
1213

@@ -19,6 +20,13 @@ abstract class AbstractScraper implements ScraperInterface
1920
*/
2021
protected $config;
2122

23+
/**
24+
* Logger.
25+
*
26+
* @var Snippetify\SnippetSniffer\Common\Logger
27+
*/
28+
protected $logger;
29+
2230
/**
2331
* Snippets.
2432
*
@@ -39,10 +47,13 @@ public function __construct(array $config = [])
3947
if (empty($this->config['logger'])) $this->config['logger'] = [];
4048

4149
if (empty($config['app'])) {
42-
$this->config['app']['name'] = Core::APP_NAME;
43-
$this->config['app']['type'] = Core::APP_TYPE;
44-
$this->config['app']['version'] = Core::APP_VERSION;
50+
$this->config['app']['name'] = Core::APP_NAME;
51+
$this->config['app']['type'] = Core::APP_TYPE;
52+
$this->config['app']['version'] = Core::APP_VERSION;
53+
$this->config['app']['user_agent'] = Core::USER_AGENT;
4554
}
55+
56+
$this->logger = Logger::create($this->config['logger']);
4657
}
4758

4859
/**
@@ -53,7 +64,7 @@ public function __construct(array $config = [])
5364
*/
5465
protected function getCrawler(UriInterface $uri): Crawler
5566
{
56-
return (new Client)->request('GET', $uri, ['headers' => ['User-Agent' => Core::USER_AGENT]]);
67+
return (new Client)->request('GET', $uri, ['headers' => ['User-Agent' => $this->config['app']['user_agent']]]);
5768
}
5869

5970
/**
@@ -165,4 +176,15 @@ protected function fetchWebsiteMetadata(Crawler $node, Crawler $crawler): array
165176
'url' => (new Uri($node->getUri()))->getHost(),
166177
];
167178
}
179+
180+
/**
181+
* Log error.
182+
*
183+
* @param string $message
184+
* @return void
185+
*/
186+
protected function logError(string $message): void
187+
{
188+
$this->logger->log($message, Logger::ERROR);
189+
}
168190
}

‎src/Scrapers/DefaultScraper.php

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@
33
namespace Snippetify\SnippetSniffer\Scrapers;
44

55
use Psr\Http\Message\UriInterface;
6-
use Snippetify\SnippetSniffer\Common\Logger;
76

87
final class DefaultScraper extends AbstractScraper
98
{
@@ -35,7 +34,7 @@ public function fetch(UriInterface $uri, array $options = []): array
3534
$this->hydrateSnippets($node, $crawler, $uri);
3635
});
3736
} catch (\Exception $e) {
38-
Logger::create($this->config['logger'])->log($e, Logger::ERROR);
37+
$this->logError($e);
3938
}
4039

4140
return $this->snippets;

‎src/Scrapers/StackoverflowScraper.php

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@
66
use GuzzleHttp\Psr7\Uri;
77
use Psr\Http\Message\UriInterface;
88
use Symfony\Component\DomCrawler\Crawler;
9-
use Snippetify\SnippetSniffer\Common\Logger;
109
use Snippetify\SnippetSniffer\Common\Snippet;
1110

1211
final class StackoverflowScraper extends AbstractScraper
@@ -60,7 +59,7 @@ public function fetch(UriInterface $uri, array $options = []): array
6059

6160
});
6261
} catch (\Exception $e) {
63-
Logger::create($this->config['logger'])->log($e, Logger::ERROR);
62+
$this->logError($e);
6463
}
6564

6665
return $this->snippets;

‎src/SnippetSniffer.php

Lines changed: 83 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -5,14 +5,28 @@
55
use Snippetify\SnippetSniffer\Scrapers\ScraperInterface;
66
use Snippetify\SnippetSniffer\Providers\ProviderInterface;
77

8-
class SnippetSniffer
8+
final class SnippetSniffer
99
{
1010
/**
11-
* The config.
11+
* Configuration.
1212
*
1313
* @var string
1414
*/
15-
protected $config;
15+
private $config;
16+
17+
/**
18+
* Scrapers.
19+
*
20+
* @var array
21+
*/
22+
private $scrapers;
23+
24+
/**
25+
* Providers.
26+
*
27+
* @var array
28+
*/
29+
private $providers;
1630

1731
/**
1832
* Singletion.
@@ -33,7 +47,18 @@ public function __construct(array $config)
3347
throw new \InvalidArgumentException("Invalid arguments");
3448
}
3549

36-
$this->config = $config;
50+
$this->config = $config;
51+
$this->scrapers = Core::$scrapers;
52+
$this->providers = Core::$providers;
53+
54+
if (!empty($config['scrapers'])) {
55+
$this->scrapers = array_merge($this->scrapers, $config['scrapers']);
56+
}
57+
58+
if (!empty($config['providers'])) {
59+
$this->providers = array_merge($this->providers, $config['providers']);
60+
}
61+
3762
}
3863

3964
/**
@@ -49,6 +74,42 @@ public static function create(array $config): self
4974
return self::$instance;
5075
}
5176

77+
/**
78+
* Add scraper.
79+
*
80+
* @param string $name
81+
* @param string $class
82+
* @return self
83+
*/
84+
public function addScraper(string $name, string $class): self
85+
{
86+
if (empty(trim($name)) || empty(trim($class))) {
87+
throw new \InvalidArgumentException("Arguments cannot be empty.");
88+
}
89+
90+
$this->scrapers[$name] = $class;
91+
92+
return $this;
93+
}
94+
95+
/**
96+
* Add provider.
97+
*
98+
* @param string $name
99+
* @param string $class
100+
* @return self
101+
*/
102+
public function addProvider(string $name, string $class): self
103+
{
104+
if (empty(trim($name)) || empty(trim($class))) {
105+
throw new \InvalidArgumentException("Arguments cannot be empty.");
106+
}
107+
108+
$this->providers[$name] = $class;
109+
110+
return $this;
111+
}
112+
52113
/**
53114
* Fetch snippets.
54115
*
@@ -75,22 +136,21 @@ public function fetch(string $query, array $meta = []): array
75136
*/
76137
protected function provider(): ProviderInterface
77138
{
78-
$providers = Core::$providers;
79-
80-
if (!empty($this->config['providers'])) {
81-
foreach ($this->config['providers'] as $key => $value) {
82-
if (!class_exists($value)) {
83-
throw new \RuntimeException("Provider class not exists");
84-
}
85-
$providers[$key] = $value;
139+
if (empty($this->providers)) {
140+
throw new \RuntimeException("Providers cannot be empty.");
141+
}
142+
143+
foreach ($this->providers as $key => $value) {
144+
if (!class_exists($value)) {
145+
throw new \RuntimeException("Provider class not exists");
86146
}
87147
}
88148

89-
if (!array_key_exists($this->config['provider']['name'], $providers)) {
149+
if (!array_key_exists($this->config['provider']['name'], $this->providers)) {
90150
throw new \RuntimeException("Provider not exists");
91151
}
92152

93-
$provider = $providers[$this->config['provider']['name']]::create($this->config['provider']);
153+
$provider = $this->providers[$this->config['provider']['name']]::create($this->config['provider']);
94154

95155
if (!$provider instanceof ProviderInterface) {
96156
throw new \RuntimeException("Provider class must implement the ProviderInterface");
@@ -106,20 +166,19 @@ protected function provider(): ProviderInterface
106166
*/
107167
protected function scraper(string $name): ScraperInterface
108168
{
109-
$scrapers = Core::$scrapers;
110-
111-
if (!empty($this->config['scrapers'])) {
112-
foreach ($this->config['scrapers'] as $key => $value) {
113-
if (!class_exists($value)) {
114-
throw new \RuntimeException("Scraper class not exists");
115-
}
116-
$scrapers[$key] = $value;
169+
if (empty($this->scrapers)) {
170+
throw new \RuntimeException("Scrapers cannot be empty.");
171+
}
172+
173+
foreach ($this->scrapers as $key => $value) {
174+
if (!class_exists($value)) {
175+
throw new \RuntimeException("Scraper class not exists");
117176
}
118177
}
119178

120-
$name = array_key_exists($name, $scrapers) ? $name : 'default';
179+
$name = array_key_exists($name, $this->scrapers) ? $name : 'default';
121180

122-
$scraper = new $scrapers[$name]($this->config);
181+
$scraper = new $this->scrapers[$name]($this->config);
123182

124183
if (!$scraper instanceof ScraperInterface) {
125184
throw new \RuntimeException("Scraper class must implement the ScraperInterface");

0 commit comments

Comments
 (0)