diff --git a/contacts.gemspec b/contacts.gemspec index 13ef4e7..288869b 100644 --- a/contacts.gemspec +++ b/contacts.gemspec @@ -1,6 +1,6 @@ Gem::Specification.new do |s| s.name = "contacts" - s.version = "1.2.4" + s.version = "1.2.6" s.date = "2010-07-06" s.summary = "A universal interface to grab contact list information from various providers including Yahoo, AOL, Gmail, Hotmail, and Plaxo." s.email = "lucas@rufy.com" diff --git a/joshuaknox-contacts.gemspec b/joshuaknox-contacts.gemspec new file mode 100644 index 0000000..ff33141 --- /dev/null +++ b/joshuaknox-contacts.gemspec @@ -0,0 +1,14 @@ +Gem::Specification.new do |s| + s.name = "joshuaknox-contacts" + s.version = "1.2.7" + s.date = "2011-07-14" + s.summary = "A universal interface to grab contact list information from various providers including Yahoo, AOL, Gmail, Hotmail, and Plaxo." + s.email = "joshuaknox@gmail.com" + s.homepage = "http://github.com/joshuaknox/contacts" + s.description = "A universal interface to grab contact list information from various providers including Yahoo, AOL, Gmail, Hotmail, and Plaxo." + s.has_rdoc = false + s.authors = ["Joshua Knox", "Glenn Sidney", "Lucas Carlson"] + s.files = ["LICENSE", "Rakefile", "README", "examples/grab_contacts.rb", "lib/contacts.rb", "lib/contacts/base.rb", "lib/contacts/json_picker.rb", "lib/contacts/gmail.rb", "lib/contacts/aol.rb", "lib/contacts/hotmail.rb", "lib/contacts/plaxo.rb", "lib/contacts/yahoo.rb"] + s.add_dependency("json", ">= 1.1.1") + s.add_dependency('gdata', '>= 1.1.1') +end \ No newline at end of file diff --git a/lib/contacts/aol.rb b/lib/contacts/aol.rb index 5336ae6..df19528 100644 --- a/lib/contacts/aol.rb +++ b/lib/contacts/aol.rb @@ -6,10 +6,6 @@ class Aol < Base LOGIN_URL = "https://my.screenname.aol.com/_cqr/login/login.psp" LOGIN_REFERER_URL = "http://webmail.aol.com/" LOGIN_REFERER_PATH = "sitedomain=sns.webmail.aol.com&lang=en&locale=us&authLev=0&uitype=mini&loginId=&redirType=js&xchk=false" - AOL_NUM = "29970-343" # this seems to change each time they change the protocol - - CONTACT_LIST_URL = "http://webmail.aol.com/#{AOL_NUM}/aim-2/en-us/Lite/ContactList.aspx?folder=Inbox&showUserFolders=False" - CONTACT_LIST_CSV_URL = "http://webmail.aol.com/#{AOL_NUM}/aim-2/en-us/Lite/ABExport.aspx?command=all" PROTOCOL_ERROR = "AOL has changed its protocols, please upgrade this library first. If that does not work, dive into the code and submit a patch at http://github.com/cardmagic/contacts" def real_connect @@ -80,7 +76,7 @@ def real_connect data, resp, cookies, forward, old_url = get(forward, cookies, old_url) + [forward] end - if data.index("Invalid Username or Password. Please try again.") + if data.index("Incorrect Username or Password.") raise AuthenticationError, "Username and password do not match" elsif data.index("Required field must not be blank") raise AuthenticationError, "Login and password must not be blank" @@ -91,7 +87,11 @@ def real_connect elsif cookies == "" raise ConnectionError, PROTOCOL_ERROR end - + + @aol_num = data.match(/URL=\"http:\/\/mail.aol.com\/(.*)\/aol-6\/en-us\/common\/error.aspx\?/)[1] + @contact_list_url = "http://mail.aol.com/#{@aol_num}/aol-6/en-us/Lite/ContactList.aspx?folder=Inbox&showUserFolders=False" + @contact_list_csv_url = "http://mail.aol.com/#{@aol_num}/aol-6/en-us/Lite/ABExport.aspx?command=all" + @cookies = cookies end @@ -103,7 +103,7 @@ def contacts return @contacts if @contacts if connected? - data, resp, cookies, forward, old_url = get(CONTACT_LIST_URL, @cookies, CONTACT_LIST_URL) + [CONTACT_LIST_URL] + data, resp, cookies, forward, old_url = get(@contact_list_url, @cookies, @contact_list_url) + [@contact_list_url] until forward.nil? data, resp, cookies, forward, old_url = get(forward, cookies, old_url) + [forward] @@ -119,7 +119,7 @@ def contacts postdata["user"] = input.attributes["value"] if input.attributes["name"] == "user" end - data, resp, cookies, forward, old_url = get(CONTACT_LIST_CSV_URL, @cookies, CONTACT_LIST_URL) + [CONTACT_LIST_URL] + data, resp, cookies, forward, old_url = get(@contact_list_csv_url, @cookies, @contact_list_url) + [@contact_list_url] until forward.nil? data, resp, cookies, forward, old_url = get(forward, cookies, old_url) + [forward] @@ -132,14 +132,67 @@ def contacts parse data end end + + def contacts_data + postdata = { + "file" => 'contacts', + "fileType" => 'csv' + } + + if connected? + data, resp, cookies, forward, old_url = get(@contact_list_url, @cookies, @contact_list_url) + [@contact_list_url] + + until forward.nil? + data, resp, cookies, forward, old_url = get(forward, cookies, old_url) + [forward] + end + + if resp.code_type != Net::HTTPOK + raise ConnectionError, self.class.const_get(:PROTOCOL_ERROR) + end + + # parse data and grab + doc = Hpricot(data) + (doc/:input).each do |input| + postdata["user"] = input.attributes["value"] if input.attributes["name"] == "user" + end + + data, resp, cookies, forward, old_url = get(@contact_list_csv_url, @cookies, @contact_list_url) + [@contact_list_url] + + until forward.nil? + data, resp, cookies, forward, old_url = get(forward, cookies, old_url) + [forward] + end + + if data.include?("error.gif") + raise AuthenticationError, "Account invalid" + end + + data + end + end + private def parse(data, options={}) - data = CSV::Reader.parse(data) + if CSV.const_defined? :Reader + data = CSV::Reader.parse(data) + else + if data.is_a?(String) + data = CSV.parse(data) + else + data = CSV.read(data) + end + end col_names = data.shift - @contacts = data.map do |person| - ["#{person[0]} #{person[1]}", person[4]] if person[4] && !person[4].empty? - end.compact + @contacts = [] + data.each do |person| + if person[4] && !person[4].empty? + @contacts << ["#{person[0]} #{person[1]}", person[4]] + end + if person[5] && !person[5].empty? + @contacts << ["#{person[0]} #{person[1]}", person[5]] + end + end + @contacts end def h_to_query_string(hash) @@ -151,4 +204,4 @@ def h_to_query_string(hash) end TYPES[:aol] = Aol -end \ No newline at end of file +end diff --git a/lib/contacts/base.rb b/lib/contacts/base.rb index 3089dc2..6e5b638 100644 --- a/lib/contacts/base.rb +++ b/lib/contacts/base.rb @@ -18,7 +18,7 @@ def initialize(login, password, options={}) @captcha_token = options[:captcha_token] @captcha_response = options[:captcha_response] @connections = {} - connect + connect unless options[:no_connect] end def connect @@ -166,9 +166,9 @@ def get(url, cookies="", referer="") data = uncompress(resp, data) cookies = parse_cookies(resp.response['set-cookie'], cookies) forward = resp.response['Location'] - if (not forward.nil?) && URI.parse(forward).host.nil? - forward = url.scheme.to_s + "://" + url.host.to_s + forward - end + if (not forward.nil?) && URI.parse(forward).host.nil? + forward = url.scheme.to_s + "://" + url.host.to_s + forward + end return data, resp, cookies, forward end diff --git a/lib/contacts/hotmail.rb b/lib/contacts/hotmail.rb index 827c570..626de78 100644 --- a/lib/contacts/hotmail.rb +++ b/lib/contacts/hotmail.rb @@ -9,6 +9,52 @@ class Hotmail < Base PWDPAD = "IfYouAreReadingThisYouHaveTooMuchFreeTime" MAX_HTTP_THREADS = 8 + def real_connect_debug + data, resp, cookies, forward = get(URL) + old_url = URL + until forward.nil? + data, resp, cookies, forward, old_url = get(forward, cookies, old_url) + [forward] + end + + postdata = "PPSX=%s&PwdPad=%s&login=%s&passwd=%s&LoginOptions=2&PPFT=%s" % [ + CGI.escape(data.split("><").grep(/PPSX/).first[/=\S+$/][2..-3]), + PWDPAD[0...(PWDPAD.length-@password.length)], + CGI.escape(login), + CGI.escape(password), + CGI.escape(data.split("><").grep(/PPFT/).first[/=\S+$/][2..-3]) + ] + + form_url = data.split("><").grep(/form/).first.split[5][8..-2] + data, resp, cookies, forward = post(form_url, postdata, cookies) + + old_url = form_url + until cookies =~ /; PPAuth=/ || forward.nil? + data, resp, cookies, forward, old_url = get(forward, cookies, old_url) + [forward] + end + data + + if data.index("The e-mail address or password is incorrect") + raise AuthenticationError, "Username and password do not match" + elsif cookies == "" + raise ConnectionError, PROTOCOL_ERROR + end + + data, resp, cookies, forward = get("http://mail.live.com/mail", cookies) + until forward.nil? + data, resp, cookies, forward, old_url = get(forward, cookies, old_url) + [forward] + end + + + @domain = URI.parse(old_url).host + @cookies = cookies + rescue AuthenticationError => m + if @attempt == 1 + retry + else + raise m + end + end + def real_connect data, resp, cookies, forward = get(URL) old_url = URL @@ -45,6 +91,7 @@ def real_connect data, resp, cookies, forward, old_url = get(forward, cookies, old_url) + [forward] end + @domain = URI.parse(old_url).host @cookies = cookies rescue AuthenticationError => m @@ -55,6 +102,13 @@ def real_connect end end + def debug_contacts + if connected? + url = URI.parse(contact_list_url) + data, resp, cookies, forward = get( contact_list_url, @cookies ) + end + end + def contacts(options = {}) if connected? url = URI.parse(contact_list_url) @@ -76,7 +130,7 @@ def contacts(options = {}) resp, data = http.get(get_contact_list_url(index), "Cookie" => @cookies) email_match_text_beginning = Regexp.escape("http://m.mail.live.com/?rru=compose&to=") - email_match_text_end = Regexp.escape("&") + email_match_text_end = Regexp.escape("&ru=") raw_html = resp.body.split(" ").grep(/(?:e|dn)lk[0-9]+/) @@ -89,7 +143,7 @@ def contacts(options = {}) # Grab info case c_info[1] when "e" # Email - build_contacts.last[1] = row.match(/#{email_match_text_beginning}(.*)#{email_match_text_end}/)[1] + build_contacts.last[1] = row.match(/#{email_match_text_beginning}(.*?)#{email_match_text_end}/)[1] when "dn" # Name build_contacts.last[0] = row.match(/]*>(.+)<\/a>/)[1] end @@ -105,7 +159,7 @@ def contacts(options = {}) build_contacts.each do |contact| unless contact[1].nil? # Only return contacts with email addresses - contact[1] = CGI::unescape(contact[1]) + contact[1] = CGI::unescape CGI::unescape(contact[1]) @contacts << contact end end @@ -122,4 +176,4 @@ def get_contact_list_url(index) TYPES[:hotmail] = Hotmail end -end \ No newline at end of file +end diff --git a/lib/contacts/json_picker.rb b/lib/contacts/json_picker.rb index 6149b62..c935f61 100644 --- a/lib/contacts/json_picker.rb +++ b/lib/contacts/json_picker.rb @@ -4,8 +4,7 @@ class Contacts def self.parse_json( string ) - if Object.const_defined?('ActiveSupport') and - ActiveSupport.const_defined?('JSON') + if Object.const_defined?('ActiveSupport') and ActiveSupport.const_defined?('JSON') ActiveSupport::JSON.decode( string ) elsif Object.const_defined?('JSON') JSON.parse( string ) @@ -13,4 +12,4 @@ def self.parse_json( string ) raise 'Contacts requires JSON or Rails (with ActiveSupport::JSON)' end end -end \ No newline at end of file +end diff --git a/lib/contacts/plaxo.rb b/lib/contacts/plaxo.rb index e86e51a..8fa6df5 100644 --- a/lib/contacts/plaxo.rb +++ b/lib/contacts/plaxo.rb @@ -24,6 +24,18 @@ def contacts parse data end # contacts + def contacts_data + getdata = "&authInfo.authByEmail.email=%s" % CGI.escape(login) + getdata += "&authInfo.authByEmail.password=%s" % CGI.escape(password) + data, resp, cookies, forward = get(CONTACT_LIST_URL + getdata) + + if resp.code_type != Net::HTTPOK + raise ConnectionError, PROTOCOL_ERROR + end + + data + end + private def parse(data, options={}) doc = REXML::Document.new(data) @@ -39,14 +51,16 @@ def parse(data, options={}) elsif cont.elements['displayName'] cont.elements['displayName'].text end - email = if cont.elements['email1'] - cont.elements['email1'].text - end - if name || email - @contacts << [name, email] + i = 1 + emails = [] + while (cont.elements["email#{i}"] || cont.elements["workEmail#{i}"]) + emails << cont.elements["email#{i}"].text if cont.elements["email#{i}"] + emails << cont.elements["workEmail#{i}"].text if cont.elements["workEmail#{i}"] + i += 1 end + emails.each {|email| @contacts << [name, email] } end - @contacts + @contacts.uniq {|a,b| b} else raise ConnectionError, PROTOCOL_ERROR end @@ -127,4 +141,4 @@ def parse(data, options={}) 3 -=end \ No newline at end of file +=end diff --git a/lib/contacts/yahoo.rb b/lib/contacts/yahoo.rb index 5b27d36..22cac0b 100644 --- a/lib/contacts/yahoo.rb +++ b/lib/contacts/yahoo.rb @@ -5,17 +5,24 @@ class Yahoo < Base ADDRESS_BOOK_URL = "http://address.mail.yahoo.com/?.rand=430244936" CONTACT_LIST_URL = "http://address.mail.yahoo.com/?_src=&_crumb=crumb&sortfield=3&bucket=1&scroll=1&VPC=social_list&.r=time" PROTOCOL_ERROR = "Yahoo has changed its protocols, please upgrade this library first. If that does not work, dive into the code and submit a patch at http://github.com/cardmagic/contacts" - - def real_connect + INVALID_PASS = '
Invalid ID or password.
Please try again using your full Yahoo! ID.
' + NOT_YET_TAKEN = '
This ID is not yet taken.
Are you trying to ' + + def real_connect(attempt_count=0) postdata = ".tries=2&.src=ym&.md5=&.hash=&.js=&.last=&promo=&.intl=us&.bypass=" postdata += "&.partner=&.u=4eo6isd23l8r3&.v=0&.challenge=gsMsEcoZP7km3N3NeI4mX" postdata += "kGB7zMV&.yplus=&.emailCode=&pkg=&stepid=&.ev=&hasMsgr=1&.chkP=Y&." postdata += "done=#{CGI.escape(URL)}&login=#{CGI.escape(login)}&passwd=#{CGI.escape(password)}" - + data, resp, cookies, forward = post(LOGIN_URL, postdata) - - if data.index("Invalid ID or password") || data.index("This ID is not yet taken") - raise AuthenticationError, "Username and password do not match" + + if data.index(INVALID_PASS) || data.index(NOT_YET_TAKEN) + if attempt_count < 1 + sleep(5) + return real_connect(attempt_count + 1) + else + raise AuthenticationError, "Username and password do not match" + end elsif data.index("Sign in") && data.index("to Yahoo!") raise AuthenticationError, "Required field must not be blank" elsif !data.match(/uncompressed\/chunked/) @@ -23,20 +30,50 @@ def real_connect elsif cookies == "" raise ConnectionError, PROTOCOL_ERROR end - + data, resp, cookies, forward = get(forward, cookies, LOGIN_URL) - + if resp.code_type != Net::HTTPOK raise ConnectionError, PROTOCOL_ERROR end - + @cookies = cookies end + + def contacts_data + if connected? + # first, get the addressbook site with the new crumb parameter + url = URI.parse(address_book_url) + http = open_http(url) + resp, data = http.get("#{url.path}?#{url.query}", + "Cookie" => @cookies + ) - def contacts + if resp.code_type != Net::HTTPOK + raise ConnectionError, self.class.const_get(:PROTOCOL_ERROR) + end + + crumb = data.to_s[/dotCrumb: '(.*?)'/][13...-1] + + # now proceed with the new ".crumb" parameter to get the csv data + url = URI.parse(contact_list_url.sub("_crumb=crumb","_crumb=#{crumb}").sub("time", Time.now.to_f.to_s.sub(".","")[0...-2])) + http = open_http(url) + resp, more_data = http.get("#{url.path}?#{url.query}", + "Cookie" => @cookies, + "X-Requested-With" => "XMLHttpRequest", + "Referer" => address_book_url + ) + + if resp.code_type != Net::HTTPOK + raise ConnectionError, self.class.const_get(:PROTOCOL_ERROR) + end + + [data, more_data] + end + end + + def contacts return @contacts if @contacts - @contacts = [] - if connected? # first, get the addressbook site with the new crumb parameter url = URI.parse(address_book_url) @@ -63,12 +100,16 @@ def contacts if resp.code_type != Net::HTTPOK raise ConnectionError, self.class.const_get(:PROTOCOL_ERROR) end - + + parse data + + parse more_data + if more_data =~ /"TotalABContacts":(\d+)/ total = $1.to_i - ((total / 50.0).ceil).times do |i| + ((total / 50)).times do |i| # now proceed with the new ".crumb" parameter to get the csv data - url = URI.parse(contact_list_url.sub("bucket=1","bucket=#{i}").sub("_crumb=crumb","_crumb=#{crumb}").sub("time", Time.now.to_f.to_s.sub(".","")[0...-2])) + url = URI.parse(contact_list_url.sub("bucket=1","bucket=#{i+1}").sub("_crumb=crumb","_crumb=#{crumb}").sub("time", Time.now.to_f.to_s.sub(".","")[0...-2])) http = open_http(url) resp, more_data = http.get("#{url.path}?#{url.query}", "Cookie" => @cookies, @@ -79,26 +120,28 @@ def contacts if resp.code_type != Net::HTTPOK raise ConnectionError, self.class.const_get(:PROTOCOL_ERROR) end - + parse more_data end end - - @contacts + # eliminate dupe emails + @contacts = @contacts.uniq {|a,b| b} end end private - + def parse(data, options={}) @contacts ||= [] - @contacts += Contacts.parse_json(data)["response"]["ResultSet"]["Contacts"].to_a.select{|contact|!contact["email"].to_s.empty?}.map do |contact| - name = contact["contactName"].split(",") - [[name.pop, name.join(",")].join(" ").strip, contact["email"]] - end if data =~ /^\{"response":/ - @contacts + if data =~ /var InitialContacts = (\[.*?\]);/ + @contacts += Contacts.parse_json($1).select{|contact|!contact["email"].to_s.empty?}.map{|contact|[contact["contactName"], contact["email"]]} + elsif data =~ /^\{"response":/ + @contacts += Contacts.parse_json(data)["response"]["ResultSet"]["Contacts"].to_a.select{|contact|!contact["email"].to_s.empty?}.map{|contact|[contact["contactName"], contact["email"]]} + else + @contacts + end end - + end TYPES[:yahoo] = Yahoo