Web crawler implemented

Author: evens-stone

README.md

@@ -15,6 +15,8 @@ This library allows you
 $ composer require snippetify/snippet-sniffer
 ```
 
+### Snippet Sniffer
+
 ```php
 use Snippetify\SnippetSniffer\SnippetSniffer;
 
@@ -199,13 +201,93 @@ SnippetSniffer::create(...)
   ...
 ```
 
+### Sniptbot
+
+Sniptbot allows you to extract all snippets from a website by crawling it.
+
+```php
+use Snippetify\SnippetSniffer\WebCrawler;
+
+// Optional
+$config = [...];
+
+// @return Snippetify\SnippetSniffer\Common\MetaSnippetCollection[]
+$snippets = WebCrawler::create($config)->fetch(['your uri']);
+```
+
+
+
+#### Configuration reference
+
+```php
+$config = [
+  // Required 
+  // Search engine api configuration keys
+  'provider' => [
+    "cx" => "your google Search engine ID",
+    "key" => "your google API key"
+    'name' => 'provider name (google)',
+  ],
+  // Optional
+  // Useful for adding meta information to each snippet
+  'app' => [
+    "name" => "your App name",
+    'version' => 'your App version',
+  ],
+  // Optional
+  // Useful for logging
+  'logger' => [
+    "name" => "logger name",
+    'file' => 'logger file path',
+  ],
+  // Optional
+  // Useful for scraping
+  "html_tags" => [
+    "snippet" => "pre[class] code, div[class] code, .highlight pre, code[class]", // Tags to fetch snippets
+    "index" => "h1, h2, h3, h4, h5, h6, p, li" // Tags to index
+  ],
+  // Optional
+  // Useful for adding new scrapers
+  // The name must be the website host without the scheme i.e. not https://foo.com but foo.com
+  "scrapers" => [
+    "scraper_name" => ScraperClass::class,
+    "scraper_2_name" => Scraper2Class::class // You can add as many as you want
+  ],
+  // Optional
+  // Useful for adding new providers
+  "providers" => [
+    "provider_name" => ProviderClass::class,
+    "provider_2_name" => Provider2Class::class // You can add as many as you want
+  ],
+  // Optional
+  // Useful for web crawling
+  // Please follow the link below for more information as we use Spatie crawler
+  // https://github.com/spatie/crawler
+  "crawler" => [
+    "langs" => ['en'],
+    "profile" => CrawlSubdomainsAndUniqueUri::class,
+    "user_agent" => 'your user agent',
+    "concurrency" => 10,
+    "ignore_robots" => false,
+    "maximum_depth" => null,
+    "execute_javascript" => false,
+    "maximum_crawl_count" => null,
+    "parseable_mime_types" => 'text/html',
+    "maximum_response_size" => 1024 * 1024 * 3,
+    "delay_between_requests" => 250,
+  ]
+];
+```
+
 ## Changelog
 
 Please see [CHANGELOG](https://github.com/snippetify/snippet-sniffer/blob/master/CHANGELOG.md) for more information what has changed recently.
 
 ## Testing
 
- You must set the **PROVIDER_NAME**, **PROVIDER_CX**, **PROVIDER_KEY** keys in phpunit.xml file before running tests.
+ You must set the **PROVIDER_NAME**, **PROVIDER_CX**, **PROVIDER_KEY**, **CRAWLER_URI**, **DEFAULT_SCRAPER_URI**, **STACKOVERFLOW_SCRAPER_URI** keys in phpunit.xml file before running tests.
+
+**Important:** Those links must contains at least one snippet otherwise the tests will failed. The **Stackoverflow** uri must be a question link with an accepted answer otherwise the tests will failed.
 
 ```bash
 composer test

phpunit.xml.dist

@@ -21,5 +21,8 @@
         <server name="PROVIDER_CX" value=""/>
         <server name="PROVIDER_KEY" value=""/>
         <server name="APP_ENV" value="testing"/>
+        <server name="CRAWLER_URI" value=""/>
+        <server name="DEFAULT_SCRAPER_URI" value=""/>
+        <server name="STACKOVERFLOW_SCRAPER_URI" value=""/>
     </php>
 </phpunit>

src/Common/Snippet.php

@@ -18,7 +18,7 @@
  */
 class Snippet
 {
-    const WIKI_TYPE = 'wiki';
+    const ROBOT_TYPE = 'robot';
 
     /**
      * @var string

src/Common/WebPage.php

@@ -19,22 +19,17 @@ class WebPage
     /**
      * @var string
      */
-    public $siteName;
-
-    /**
-     * @var Psr\Http\Message\UriInterface
-     */
-    public $siteUri;
+    public $title;
 
     /**
      * @var string
      */
-    public $title;
-
+    public $summary;
+    
     /**
      * @var string
      */
-    public $summary;
+    public $lang;
 
     /**
      * @var Psr\Http\Message\UriInterface

src/Core.php

@@ -11,23 +11,26 @@
 
 namespace Snippetify\SnippetSniffer;
 
+use Snippetify\SnippetSniffer\Profiles\CrawlSubdomainsAndUniqueUri;
+
 class Core
 {
     public const APP_NAME      = 'Snippet sniffer';
     public const APP_TYPE      = 'snippetify-sniffer';
     public const APP_VERSION   = '1.1.0';
 
     // Crawler
-    public const CRAWLER_PROFILE                = \Spatie\Crawler\CrawlSubdomains::class;
+    public const CRAWLER_LANG                   = 'en';
+    public const CRAWLER_PROFILE                = CrawlSubdomainsAndUniqueUri::class;
     public const CRAWLER_CONCURENCY             = 10;
     public const CRAWLER_IGNORE_ROBOTS          = true;
-    public const CRAWLER_MAXIMUM_DEPTH          = 50;
+    public const CRAWLER_MAXIMUM_DEPTH          = null;
     public const CRAWLER_EXECUTE_JAVASCRIPT     = false;
-    public const CRAWLER_MAXIMUM_CRAWL_COUNT    = 1500;
+    public const CRAWLER_MAXIMUM_CRAWL_COUNT    = null;
     public const CRAWLER_PARSEABLE_MIME_TYPES   = 'text/html';
     public const CRAWLER_MAXIMUM_RESPONSE_SIZE  = 1024 * 1024 * 3;
     public const CRAWLER_DELAY_BETWEEN_REQUESTS = 250;
-    public const CRAWLER_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:65.0) Gecko/20100101 Firefox/65.0';
+    public const CRAWLER_USER_AGENT = 'Mozilla/5.0 (compatible; Sniptbot/1.0; +http://www.snippetify.com/bot)';
 
     /**
      * Html Snippet tags

src/Observers/SnippetCrawlObserver.php

@@ -48,32 +48,51 @@ public function __construct(WebCrawler $webCrawler)
      */
     public function crawled(UriInterface $url, ResponseInterface $response, ?UriInterface $foundOnUrl = null)
     {
-        $crawler = new Crawler((string)$response->getBody());
+        // Must crawl uri once
+        if ($this->webCrawler->isCrawled($url)) return;
 
+        // Create crawler from reponse body
+        $crawler = new Crawler((string) $response->getBody());
+
+        // Must contains snippets
         if (0 === $crawler->filter($this->webCrawler->getConfig()['html_tags']['snippet'])->count()) return;
 
+        // Only crawl specified langs
+        if (!$this->hasLang($crawler)) return;
+
+        // New meta snippet
+        $metaSnippet = new MetaSnippetCollection([
+            'uri' => (string) $url,
+            'snippets' => $this->webCrawler->getScraper($url->getHost())->fetchFromDocument($crawler)
+        ]);
+
+        // Must contains snippets
+        if (0 === count($metaSnippet->snippets)) return;
+
+        // Get page description
         $summTags = 'meta[name="description"], meta[property="og:description"]';
         $summary  = 0 === $crawler->filter($summTags)->count() ? '' : $crawler->filter($summTags)->attr('content');
 
-        $metaSnippet = new MetaSnippetCollection(['uri' => $url]);
-
         try {
             $metaSnippet->page = new WebPage([
-                'link'      => $url,
                 'summary'   => $summary,
+                'link'      => (string) $url,
                 'title'     => $crawler->filter('title')->text(),
+                'lang'      => $crawler->filter('html')->attr('lang'),
                 'metaTags'  => $crawler->filter('meta')
                                 ->each(function ($v) { return [$v->attr('name') => $v->attr('content')]; }),
-                'plainText' => $crawler->filter($this->webCrawler->getConfig()['html_tags']['index'])
-                                        ->each(function ($v) { return ' ' . $v->text(); }),
+                'plainText' => implode(' ', $crawler->filter($this->webCrawler->getConfig()['html_tags']['index'])
+                                        ->each(function ($v) { return $v->text(); })),
             ]);
         } catch(\Exception $e) {
             $this->webCrawler->logError($requestException);
         }
 
-        $metaSnippet->snippets = $this->webCrawler->getScraper($url->getHost())->fetchFromDocument($crawler);
+        // Save meta snippet
+        $this->webCrawler->addUniqueSnippet($metaSnippet);
 
-        $this->webCrawler->addSnippet($metaSnippet);
+        // Save crawled uri
+        $this->webCrawler->addToCrawledUris($url);
     }
 
     /**
@@ -87,4 +106,21 @@ public function crawlFailed(UriInterface $url, RequestException $requestExceptio
     {
         $this->webCrawler->logError($requestException);
     }
+
+    /**
+     * @param Symfony\Component\DomCrawler\Crawler  $crawler
+     * @return bool
+     */
+    private function hasLang(Crawler $crawler): bool
+    {
+        $has = false;
+
+        foreach ($this->webCrawler->getConfig()['crawler']['langs'] as $value) {
+            if (false !== stripos($crawler->filter('html')->attr('lang'), $value)) {
+                $has = true;
+            }
+        }
+
+        return $has;
+    }
 }

src/Profiles/CrawlSubdomainsAndUniqueUri.php

@@ -0,0 +1,27 @@
+<?php
+
+namespace Snippetify\SnippetSniffer\Profiles;
+
+use Psr\Http\Message\UriInterface;
+use Spatie\Crawler\CrawlSubdomains;
+use Snippetify\SnippetSniffer\WebCrawler;
+
+class CrawlSubdomainsAndUniqueUri extends CrawlSubdomains
+{
+    /**
+     * @var Snippetify\SnippetSniffer\WebCrawler
+     */
+    private $webCrawler;
+
+    public function __construct($baseUrl, WebCrawler $webCrawler)
+    {
+        parent::__construct($baseUrl);
+
+        $this->webCrawler = $webCrawler;
+    }
+
+    public function shouldCrawl(UriInterface $url): bool
+    {
+        return $this->isSubdomainOfHost($url) && !$this->webCrawler->isCrawled($url);
+    }
+}

src/Scrapers/AbstractScraper.php

@@ -185,11 +185,11 @@ protected function fetchSnippet(Crawler $node, Crawler $crawler, array $meta = [
         return new Snippet([
             'tags'          => $tags,
             'code'          => $node->text(),
-            'type'          => Snippet::WIKI_TYPE,
+            'type'          => Snippet::ROBOT_TYPE,
             'title'         => $crawler->filter('title')->text(),
             'description'   => $desc,
             'meta'          => [
-                'url'       => $node->getUri(),
+                'url'       => $crawler->getUri(),
                 'target'    => $this->config['app'],
                 'website'   => $this->fetchWebsiteMetadata($crawler)
             ]

src/Scrapers/StackoverflowScraper.php

@@ -86,12 +86,12 @@ protected function fetchSnippet(Crawler $node, Crawler $crawler, array $meta = [
         return new Snippet([
             'tags'          => $tags,
             'title'         => $title,
-            'type'          => Snippet::WIKI_TYPE,
+            'type'          => Snippet::ROBOT_TYPE,
             'code'          => $node->filter('code')->text(),
             'description'   => $desc,
             'meta'          => [
                 'accepted'  => $meta['accepted'],
-                'url'       => $node->getUri(),
+                'url'       => $crawler->getUri(),
                 'target'    => $this->config['app'],
                 'website'   => $this->fetchWebsiteMetadata($crawler)
             ]

src/SnippetSniffer.php

@@ -39,6 +39,7 @@ final class SnippetSniffer
     /**
      * @param  array  $config
      * @return void
+     * @throws InvalidArgumentException
      */
     public function __construct(array $config)
     {
@@ -78,6 +79,7 @@ public static function create(array $config): self
      * @param  string  $name
      * @param  string  $class
      * @return self
+     * @throws InvalidArgumentException
      */
     public function addScraper(string $name, string $class): self
     {
@@ -96,6 +98,7 @@ public function addScraper(string $name, string $class): self
      * @param  string  $name
      * @param  string  $class
      * @return self
+     * @throws InvalidArgumentException
      */
     public function addProvider(string $name, string $class): self
     {
@@ -123,14 +126,15 @@ public function fetch(string $query, array $meta = []): array
         foreach ($urls as $url) {
             $snippets = array_merge($snippets, $this->scraper($url->getHost())->fetch($url));
         }
-        \Snippetify\SnippetSniffer\Common\Logger::create()->log(json_encode($snippets));
+
         return $snippets;
     }
 
     /**
      * Get provider.
      *
-     * @return  Snippetify\SnippetSniffer\Providers\ProviderInterface
+     * @return Snippetify\SnippetSniffer\Providers\ProviderInterface
+     * @throws RuntimeException
      */
     private function provider(): ProviderInterface
     {
@@ -160,7 +164,9 @@ private function provider(): ProviderInterface
     /**
      * Get scraper.
      *
-     * @return  Snippetify\SnippetSniffer\Scrapers\ScraperInterface
+     * @param  string  $name
+     * @return Snippetify\SnippetSniffer\Scrapers\ScraperInterface
+     * @throws RuntimeException
      */
     private function scraper(string $name): ScraperInterface
     {