Implement parsing for BC Liquors

.gitignore

@@ -1,3 +1,7 @@
 Gemfile.lock
 pkg
+.DS_Store
+.rvmrc
+/spec/support
 
+scrape.rb

CHANGELOG.md

@@ -1,3 +1,9 @@
+1.4.0
+* added support for saq.com
+
+1.3.0
+* added support for bcliquorstores.com
+
 Version 1.2.3
 
   * Updated `ProductPage` to return `RedirectedError` for gift cards which now

Gemfile

@@ -1,2 +1,5 @@
-source :rubygems
+source 'http://rubygems.org'
+
+gem 'graphql-client'
+
 gemspec

Rakefile

@@ -12,7 +12,7 @@ task :default => :spec
 
 desc 'Start an irb console'
 task :console do
-  system 'irb -I lib -r lcbo'
+  system 'irb -I lib -r lcbo -r bcl -r saq'
 end
 
 desc 'Validates the gemspec'
@@ -43,19 +43,28 @@ task :package => :gemspec
 
 Rake::TestTask.new(:spec) do |t|
   t.libs += %w[lcbo spec]
-  t.test_files = FileList['spec/**/*.rb']
+  # t.test_files = FileList['spec/**/*.rb']
+  # t.test_files = FileList[ENV['SPEC'] || 'spec/**/*.rb']
+  t.test_files = [
+    FileList['spec/lcbo_spec.rb'],
+    FileList['spec/bcl_spec.rb'],
+    FileList['spec/saq_spec.rb'],
+  ]
   t.verbose = true
 end
 
-desc 'Download all HTML indicated in YAML assertion files'
-task :download_support do
-  require 'yaml'
-  require 'open-uri'
-  product_pages = YAML.load_file('./spec/support/product_pages.yml')
-  product_pages.each do |spec|
-    html = open(spec[:uri]).read
-    File.open("./spec/support/product_pages/#{spec[:file]}", ?w) { |file|
-      file.print(html)
-    }
-  end
-end
+# desc 'Download all HTML indicated in YAML assertion files'
+# task :download_support do
+#   require 'yaml'
+#   require 'open-uri'
+#   pages = YAML.load_file('./spec/support/pages.yml')
+#   pages.each do |type, uris|
+#     uris.each_with_index do |uri, i|
+#       html = open(uri).read
+#       File.open("./spec/support/#{type}/#{i}.html", ?w) { |file|
+#         file.print(html)
+#       }
+#     end
+#   end
+# end
+

lcbo.gemspec

@@ -1,22 +1,20 @@
 # coding: utf-8
-require File.expand_path("../lib/lcbo/version", __FILE__)
+require File.expand_path("../lib/version", __FILE__)
 
 Gem::Specification.new do |s|
   s.name        = 'lcbo'
-  s.version     = LCBO::VERSION
+  s.version     = VERSION
   s.platform    = Gem::Platform::RUBY
-  s.authors     = ['Carsten Nielsen']
-  s.email       = ['heycarsten@gmail.com']
-  s.homepage    = 'http://github.com/heycarsten/lcbo'
-  s.summary     = %q{A library for parsing HTML pages from http://lcbo.com}
+  s.authors     = ['Carsten Nielsen', 'Lenard Andal', 'Ahmed El-Daly']
+  s.email       = ['lenard@winealign.com']
+  s.summary     = %q{A library for parsing HTML pages from http://lcbo.com, http://bcliquorstores.com, http://saq.com}
   s.description = %q{Request and parse product, store, inventory, and product search pages directly from the official LCBO website.}
 
-  s.rubyforge_project = 'lcbo'
-
-  s.add_dependency 'typhoeus',      '~> 0.3.3'
-  s.add_dependency 'nokogiri',      '~> 1.5.0'
-  s.add_dependency 'unicode_utils', '~> 1.2.2'
-  s.add_dependency 'stringex',      '~> 1.3.0'
+  s.add_dependency 'typhoeus'      
+  s.add_dependency 'nokogiri'      
+  s.add_dependency 'unicode_utils'
+  s.add_dependency 'stringex'
+  s.add_dependency 'graphql-client'
 
   s.files         = `git ls-files`.split(?\n)
   s.test_files    = `git ls-files -- {test,spec}/*`.split(?\n)

lib/bcl.rb

@@ -0,0 +1,23 @@
+module BCL
+
+  DEFAULT_CONFIG = {
+    :user_agent  => nil, # Use the default User-Agent by default
+    :max_retries => 3,   # Number of times to retry a request that fails
+    :timeout     => 8    # Seconds to wait for a request before timing out
+  }.freeze
+
+  def self.config
+    reset_config! unless @config
+    @config
+  end
+
+  def self.reset_config!
+    @config = DEFAULT_CONFIG.dup
+  end
+
+end
+
+require 'ext'
+require 'bcl/helpers'
+require 'crawlkit'
+require 'bcl/pages'

lib/bcl/helpers.rb

@@ -0,0 +1,39 @@
+module BCL
+
+  PAGE_TYPES = {
+    :product      => 'ProductPage',
+    :product_list => 'ProductListPage',
+    :store_list   => 'StoreListPage',
+    :store        => 'StorePage',
+    :inventory    => 'InventoryPage'
+  }
+
+  def self.page(type)
+    Object.const_get(PAGE_TYPES[type.to_sym])
+  end
+
+  def self.parse(page_type, response)
+    page[page_type].parse(response)
+  end
+
+  def self.product(id)
+    ProductPage.process(:id => id).as_hash
+  end
+
+  def self.store(id)
+    StorePage.process(:id => id).as_hash
+  end
+
+  def self.inventory(product_id)
+    InventoryPage.process(:product_id => product_id).as_hash
+  end
+
+  def self.product_list(beginIndex=0)
+    ProductListPage.process(:beginIndex => beginIndex*BCL::ProductListPage::PER_PAGE).as_hash
+  end
+
+  def self.store_list
+    StoreListPage.process({}, {}).as_hash
+  end
+
+end

lib/bcl/pages.rb

@@ -0,0 +1,6 @@
+require 'bcl/pages/inventory_page'
+require 'bcl/pages/product_list_page'
+require 'bcl/pages/product_page'
+require 'bcl/pages/store_list_page'
+require 'bcl/pages/store_page'
+require 'bcl/pages/cities_list_page'

lib/bcl/pages/cities_list_page.rb

@@ -0,0 +1,99 @@
+module BCL
+  class CitiesListPage
+
+    include CrawlKit::Page
+
+    uri 'http://www.bcliquorstores.com/store/locator/location_data_js'
+
+    # default_body_params \
+    #   :STOCK_TYPE_NAME    => 'All',
+    #   :ITEM_NAME          => '',
+    #   :KEYWORDS           => '',
+    #   :ITEM_NUMBER        => '',
+    #   :productListingType => '',
+    #   :LIQUOR_TYPE_SHORT_ => '*',
+    #   :CATEGORY_NAME      => '*',
+    #   :SUB_CATEGORY_NAME  => '*',
+    #   :PRODUCING_CNAME    => '*',
+    #   :PRODUCING_REGION_N => '*',
+    #   :UNIT_VOLUME        => '*',
+    #   :SELLING_PRICE      => '*',
+    #   :LTO_SALES_CODE     => 'N',
+    #   :VQA_CODE           => 'N',
+    #   :KOSHER_CODE        => 'N',
+    #   :VINTAGES_CODE      => 'N',
+    #   :VALUE_ADD_SALES_CO => 'N',
+    #   :AIR_MILES_SALES_CO => 'N',
+    #   :language           => 'EN',
+    #   :style              => 'LCBO.css',
+    #   :sort               => 'sortedProduct',
+    #   :order              => '1',
+    #   :resultsPerPage     => PER_PAGE.to_s,
+    #   :page               => '1',
+    #   :action             => 'result',
+    #   :sortby             => 'sortedProduct',
+    #   :orderby            => '',
+    #   :numPerPage         => PER_PAGE.to_s
+
+    # emits :page do
+    #   body_params[:page].to_i
+    # end
+
+    # emits :final_page do
+    #   @final_page ||= begin
+    #     count = total_products / PER_PAGE
+    #     0 == (total_products % PER_PAGE) ? count : count + 1
+    #   end
+    # end
+
+    # emits :next_page do
+    #   @next_page ||= begin
+    #     page < final_page ? page + 1 : nil
+    #   end
+    # end
+
+    # emits :total_products do
+    #   @total_products ||= begin
+    #     doc.css('td[width="58%"] font.main_font b')[0].
+    #     text.
+    #     gsub(/\s+/, ' ').
+    #     strip.
+    #     to_i
+    #   end
+    # end
+
+    emits :city_ids do
+      city_hash.keys.sort
+    end
+
+    emits :city_hash do
+      result = {}
+      cities do |city_hash|
+        city_hash.delete('stores')
+        result[city_hash['city_id']] = city_hash
+      end
+      result
+    end
+    # alias_method :as_array, :product_ids
+
+    def data
+      @data ||= JSON.parse(doc.content)
+    end
+
+    def regions
+      data.each do |region_id, region_hash|
+        yield region_hash
+      end
+    end
+
+    def cities
+      regions do |region_hash|
+        region_hash['cities'].each do |city_id, city_hash|
+          yield city_hash
+        end
+      end
+    end
+
+
+  end
+end

lib/bcl/pages/inventory_page.rb

@@ -0,0 +1,80 @@
+require 'json'
+require 'open-uri'
+
+module BCL
+  class InventoryPage
+
+    include CrawlKit::Page
+
+    # FIXME: Duplicated BELOW!!!
+    # uri 'http://m.bcliquorstores.com/m/where_to_buy/{product_id}'
+    uri 'http://m.bcliquorstores.com/m/api_stores/{product_id}/9999/'
+
+    emits :product_id do
+      query_params[:product_id].to_i
+    end
+
+    emits :inventory_count do
+      @product_page[:total_units]
+    end
+
+    emits :nid do
+      @nid
+    end
+
+    emits :board do
+      "BCL"
+    end
+
+    emits :inventories do
+      results = []
+      # doc.css("#listStores li").each do |store_element|
+      #   store_id = store_element.css('a.arrow').attribute('href').value.match(/\/m\/stores\/view\/(\d+)/)[1].to_i
+      #   stock = store_element.css('a.arrow div:nth-of-type(1)')[0].content.strip.match(/Quantity: (\d+)/)[1].to_i
+      #   results << {store_id: store_id, quantity: stock}
+      # end
+
+      json.each do |store_element|
+        store_id = store_element['serial']
+        stock = store_element['quantity'].to_i
+        results << {store_id: store_id, quantity: stock}
+      end
+
+      results
+    end
+
+    def json
+      @data ||= JSON.parse(doc.content)
+    end
+
+    # emits :xdoc do
+    #   @doc
+    # end
+
+    def parse
+      return if is_parsed?
+      return unless @html
+      fire :before_parse
+      inv_uri = "http://m.bcliquorstores.com/m/api_stores/#{product_id}/9999/"
+      @doc = Nokogiri::HTML(CrawlKit::RequestPrototype.new(inv_uri).request().body)
+      fire :after_parse
+      self
+    end
+
+    def request
+      return if @html
+      fire :before_request
+
+      @product_page = BCL.product(product_id)
+      @nid = @product_page[:nid] rescue product_id
+      @query_params[:nid] = @nid
+
+      @response = request_prototype.request(query_params, body_params)
+      @html     = @response.body
+
+      fire :after_request
+      self
+    end
+
+  end
+end