From 0ccc4b45256a994b3eab1c477d6280e4748708e2 Mon Sep 17 00:00:00 2001 From: Thilo Utke Date: Wed, 20 Oct 2010 17:07:32 +0200 Subject: [PATCH 1/7] add gmx as provider, scanning the emails from inbox --- lib/contacts.rb | 1 + lib/contacts/gmx.rb | 65 ++++++++++++++++++++++++++ test/unit/gmx_contact_importer_test.rb | 39 ++++++++++++++++ 3 files changed, 105 insertions(+) create mode 100644 lib/contacts/gmx.rb create mode 100644 test/unit/gmx_contact_importer_test.rb diff --git a/lib/contacts.rb b/lib/contacts.rb index 974b4ff..f45e360 100644 --- a/lib/contacts.rb +++ b/lib/contacts.rb @@ -10,3 +10,4 @@ require 'plaxo' require 'aol' require 'mailru' +require 'gmx' diff --git a/lib/contacts/gmx.rb b/lib/contacts/gmx.rb new file mode 100644 index 0000000..eed45f6 --- /dev/null +++ b/lib/contacts/gmx.rb @@ -0,0 +1,65 @@ +class Contacts + class Gmx < Base + LOGIN_URL = 'https://service.gmx.net/de/cgi/login' + JUMP_URL = 'https://uas2.uilogin.de/intern/jump' #used for unified address service auth + MAIL_FOLDER_URL = 'http://service.gmx.net/de/cgi/g.fcgi/mail/index' + MAIL_FOLDER_PARAMS = "folder=%s&allfolders=false&first=%s&CUSTOMERNO=%s&t=%s" + + def real_connect + postdata = "AREA=1&id=%s&p=%s" % [ + CGI.escape(login), + CGI.escape(password) + ] + + data, resp, @cookies, forward = post(LOGIN_URL, postdata) + + if data.include?("/lose/password") || data.include?("login-failed") + raise AuthenticationError, "Username and password do not match" + end + + @customerno = forward.match(/CUSTOMERNO=(\d+)/)[1] + @t = forward.match(/t=([^&]+)&/)[1] + end + + def contacts + @contacts = {} + + if connected? + folders.each do |folder| + contacts_for_folder(folder) + end + end + + @contacts.to_a + end + + private + + def scan_contacts(html) + html.scan(/"\"?([\w\.\s-]+)\"?\s<([\w\.-]+@[\w\.-]+\.\w+)>"/) do |match| + @contacts[match[0]] = match[1] + end + end + + def contacts_for_folder(folder) + count = 0 + while count < 100 do + data, resp, cookies, forward = get MAIL_FOLDER_URL + "?" + MAIL_FOLDER_PARAMS % [ + folder, + count, + @customerno, + @t + ], @cookies + scan_contacts(data) + count += 10 + end + end + + def folders + %w(inbox sent) + end + + end + + TYPES[:gmx] = Gmx +end \ No newline at end of file diff --git a/test/unit/gmx_contact_importer_test.rb b/test/unit/gmx_contact_importer_test.rb new file mode 100644 index 0000000..4498d57 --- /dev/null +++ b/test/unit/gmx_contact_importer_test.rb @@ -0,0 +1,39 @@ +dir = File.dirname(__FILE__) +require "#{dir}/../test_helper" +require 'contacts' + +class GmxContactImporterTest < ContactImporterTestCase + def setup + super + @account = TestAccounts[:gmx] + end + + def test_successful_login + assert Contacts.new(:gmx, @account.username, @account.password).connected? + end + + def test_importer_fails_with_invalid_password + assert_raise(Contacts::AuthenticationError) do + Contacts.new(:gmx, @account.username, "wrong_password") + end + end + + def test_importer_fails_with_blank_password + assert_raise(Contacts::AuthenticationError) do + Contacts.new(:gmx, @account.username, "") + end + end + + def test_importer_fails_with_blank_username + assert_raise(Contacts::AuthenticationError) do + Contacts.new(:gmx, "", @account.password) + end + end + + def test_fetch_contacts + contacts = Contacts.new(:gmx, @account.username, @account.password).contacts + @account.contacts.each do |contact| + assert contacts.include?(contact), "Could not find: #{contact.inspect} in #{contacts.inspect}" + end + end +end \ No newline at end of file From a67338b0b7cb68eb5acc0aa9410d2da094ae85ef Mon Sep 17 00:00:00 2001 From: Thilo Utke Date: Wed, 20 Oct 2010 18:44:08 +0200 Subject: [PATCH 2/7] work with web.de address book --- lib/contacts.rb | 1 + lib/contacts/webde.rb | 65 ++++++++++++++++++++++++ test/unit/webde_contact_importer_test.rb | 39 ++++++++++++++ 3 files changed, 105 insertions(+) create mode 100644 lib/contacts/webde.rb create mode 100644 test/unit/webde_contact_importer_test.rb diff --git a/lib/contacts.rb b/lib/contacts.rb index f45e360..f43bcfc 100644 --- a/lib/contacts.rb +++ b/lib/contacts.rb @@ -11,3 +11,4 @@ require 'aol' require 'mailru' require 'gmx' +require 'webde' diff --git a/lib/contacts/webde.rb b/lib/contacts/webde.rb new file mode 100644 index 0000000..2062435 --- /dev/null +++ b/lib/contacts/webde.rb @@ -0,0 +1,65 @@ +class Contacts + class Webde < Base + LOGIN_URL = "https://login.web.de/intern/login/" + JUMP_URL = "https://uas2.uilogin.de/intern/jump/" + ADDRESSBOOK_URL = "https://adressbuch.web.de/exportcontacts" + + def real_connect + postdata = "service=freemail&server=%s&password=%s&username=%s" % [ + CGI.escape('https://freemail.web.de'), + CGI.escape(password), + CGI.escape(login) + ] + + # send login + data, resp, cookies, forward = post(LOGIN_URL, postdata) + + if forward.include?("logonfailed") + raise AuthenticationError, "Username and password do not match" + end + + # request session from login service + data, resp, cookies, forward = get forward + + # start mail app session + data, resp, cookies, forward = get forward + + @si = forward.match(/si=([^&]+)/)[1] + end + + def connected? + @si && @si.length > 0 + end + + def contacts + conect_to_addressbook + @contacts = [] + if @sessionid + CSV.parse(get_entries_from_addressbook) do |row| + @contacts << ["#{row[2]} #{row[0]}", row[9]] unless header_row?(row) + end + end + + @contacts + end + + private + + def header_row?(row) + row[0] == 'Nachname' + end + + def conect_to_addressbook + data, resp, cookies, forward = get JUMP_URL + "?serviceID=comsaddressbook-live.webde&session=#{@si}&server=https://freemailng2901.web.de&partnerdata=" + @sessionid = forward.match(/session=([^&]+)/)[1] + end + + def get_entries_from_addressbook + postdata = "language=de&raw_format=csv_Outlook2003&session=#{@sessionid}&what=PERSON" + data, resp, cookies, forward = post ADDRESSBOOK_URL, postdata + data + end + end + + TYPES[:webde] = Webde +end \ No newline at end of file diff --git a/test/unit/webde_contact_importer_test.rb b/test/unit/webde_contact_importer_test.rb new file mode 100644 index 0000000..9c1eacf --- /dev/null +++ b/test/unit/webde_contact_importer_test.rb @@ -0,0 +1,39 @@ +dir = File.dirname(__FILE__) +require "#{dir}/../test_helper" +require 'contacts' + +class WebdeContactImporterTest < ContactImporterTestCase + def setup + super + @account = TestAccounts[:webde] + end + + def test_successful_login + assert Contacts.new(:webde, @account.username, @account.password).connected? + end + + def test_importer_fails_with_invalid_password + assert_raise(Contacts::AuthenticationError) do + Contacts.new(:webde, @account.username, "wrong_password") + end + end + + def test_importer_fails_with_blank_password + assert_raise(Contacts::AuthenticationError) do + Contacts.new(:webde, @account.username, "") + end + end + + def test_importer_fails_with_blank_username + assert_raise(Contacts::AuthenticationError) do + Contacts.new(:webde, "", @account.password) + end + end + + def test_fetch_contacts + contacts = Contacts.new(:webde, @account.username, @account.password).contacts + @account.contacts.each do |contact| + assert contacts.include?(contact), "Could not find: #{contact.inspect} in #{contacts.inspect}" + end + end +end \ No newline at end of file From d7271506e950e13682604e8b78fa4bcea19590f9 Mon Sep 17 00:00:00 2001 From: Thilo Utke Date: Wed, 20 Oct 2010 18:45:50 +0200 Subject: [PATCH 3/7] ignore .rvmrc --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 75d9612..6d33611 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +.rvmrc .DS_Store pkg/* test/accounts.yml From 6e9c7e61ac336a5c8fffabd08870b4d6abbb84b3 Mon Sep 17 00:00:00 2001 From: Thilo Utke Date: Thu, 21 Oct 2010 12:02:10 +0200 Subject: [PATCH 4/7] fixed hotmail email parsing, as email params are now url escaped --- lib/contacts/hotmail.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/contacts/hotmail.rb b/lib/contacts/hotmail.rb index 827c570..853702d 100644 --- a/lib/contacts/hotmail.rb +++ b/lib/contacts/hotmail.rb @@ -89,7 +89,7 @@ def contacts(options = {}) # Grab info case c_info[1] when "e" # Email - build_contacts.last[1] = row.match(/#{email_match_text_beginning}(.*)#{email_match_text_end}/)[1] + build_contacts.last[1] = CGI.unescape(row.match(/#{email_match_text_beginning}([^&]*)#{email_match_text_end}/)[1]) when "dn" # Name build_contacts.last[0] = row.match(/]*>(.+)<\/a>/)[1] end From 18ae2c60d76f881b316990a4f0ec90f381a63071 Mon Sep 17 00:00:00 2001 From: Thilo Utke Date: Fri, 22 Oct 2010 15:06:40 +0200 Subject: [PATCH 5/7] make gmx login less picky about order of login params --- lib/contacts/gmx.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/contacts/gmx.rb b/lib/contacts/gmx.rb index eed45f6..2222f39 100644 --- a/lib/contacts/gmx.rb +++ b/lib/contacts/gmx.rb @@ -18,7 +18,7 @@ def real_connect end @customerno = forward.match(/CUSTOMERNO=(\d+)/)[1] - @t = forward.match(/t=([^&]+)&/)[1] + @t = forward.match(/t=([^&]+)/)[1] end def contacts From 0d421480bee1d90e2d09ee819f92108c8298fd0a Mon Sep 17 00:00:00 2001 From: endor Date: Mon, 1 Nov 2010 12:45:26 +0100 Subject: [PATCH 6/7] for web.de convert contact names and emails from latin-1 to utf-8 --- lib/contacts/webde.rb | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/lib/contacts/webde.rb b/lib/contacts/webde.rb index 2062435..ebf8a7d 100644 --- a/lib/contacts/webde.rb +++ b/lib/contacts/webde.rb @@ -1,3 +1,5 @@ +require "iconv" + class Contacts class Webde < Base LOGIN_URL = "https://login.web.de/intern/login/" @@ -32,11 +34,11 @@ def connected? end def contacts - conect_to_addressbook + connect_to_addressbook @contacts = [] if @sessionid CSV.parse(get_entries_from_addressbook) do |row| - @contacts << ["#{row[2]} #{row[0]}", row[9]] unless header_row?(row) + @contacts << [latin1_to_utf8("#{row[2]} #{row[0]}"), latin1_to_utf8(row[9])] unless header_row?(row) end end @@ -44,12 +46,16 @@ def contacts end private + + def latin1_to_utf8(string) + Iconv.conv("utf-8", "ISO-8859-1", string) + end def header_row?(row) row[0] == 'Nachname' end - def conect_to_addressbook + def connect_to_addressbook data, resp, cookies, forward = get JUMP_URL + "?serviceID=comsaddressbook-live.webde&session=#{@si}&server=https://freemailng2901.web.de&partnerdata=" @sessionid = forward.match(/session=([^&]+)/)[1] end From 51e56d2381b95bea93a42630ed6c43bc05d3631e Mon Sep 17 00:00:00 2001 From: endor Date: Mon, 1 Nov 2010 12:45:53 +0100 Subject: [PATCH 7/7] for gmx, handle malformed title attribute --- lib/contacts/gmx.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/contacts/gmx.rb b/lib/contacts/gmx.rb index 2222f39..684daa9 100644 --- a/lib/contacts/gmx.rb +++ b/lib/contacts/gmx.rb @@ -36,7 +36,7 @@ def contacts private def scan_contacts(html) - html.scan(/"\"?([\w\.\s-]+)\"?\s<([\w\.-]+@[\w\.-]+\.\w+)>"/) do |match| + html.scan(/"\"?([\w\.\s-]+)\"?\s<([\w\.-]+@[\w\.-]+\.\w+)(?:>)?"/) do |match| @contacts[match[0]] = match[1] end end