From c35219088752093dfcfa3e2a44ce06fc38f917cb Mon Sep 17 00:00:00 2001 From: Eugene Lisitsky Date: Mon, 2 Apr 2018 17:48:23 +0300 Subject: [PATCH 1/3] Added simple Go client --- go/simple.go | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100755 go/simple.go diff --git a/go/simple.go b/go/simple.go new file mode 100755 index 0000000..89ea4fd --- /dev/null +++ b/go/simple.go @@ -0,0 +1,40 @@ +//usr/bin/env go run $0 $@; exit $? +package main + +import ( + "fmt" + "io/ioutil" + "log" + "math/rand" + "net/http" + "net/url" + "time" +) + +const ( + username = "lum-customer-CUSTOMER-zone-ZONE" + password = "PASSWORD" + port = 22225 + proxyTemplateURL = "http://%v-session-%v:%v@zproxy.luminati.io:%v" +) + +func main() { + rand.Seed(time.Now().UnixNano()) + session_id := rand.Intn(10000) + proxyUrlStr := fmt.Sprintf(proxyTemplateURL, username, session_id, password, port) + proxyUrl, err := url.Parse(proxyUrlStr) + if err != nil { + log.Fatalf("Error parsing proxy URL: %v", err) + } + httpClient := &http.Client{ + Transport: &http.Transport{ + Proxy: http.ProxyURL(proxyUrl), + }, + } + req, err := http.NewRequest("GET", "http://lumtest.com/myip.json", nil) + req.Header.Add("Accept", "text/plain") + resp, err := httpClient.Do(req) + log.Printf("Resp: %+v, err %v", resp, err) + body, err := ioutil.ReadAll(resp.Body) + log.Printf("Got data: %s", body) +} From 8577b39f223c25225226ff68bc021d9947d895f4 Mon Sep 17 00:00:00 2001 From: Eugene Lisitsky Date: Tue, 3 Apr 2018 11:17:07 +0300 Subject: [PATCH 2/3] Take CUSTOMER, ZONE, PASSWORD, USERNAME from environment if set. Check error and add defer. --- go/simple.go | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/go/simple.go b/go/simple.go index 89ea4fd..bf4cd0d 100755 --- a/go/simple.go +++ b/go/simple.go @@ -8,20 +8,36 @@ import ( "math/rand" "net/http" "net/url" + "os" "time" ) -const ( - username = "lum-customer-CUSTOMER-zone-ZONE" +var ( + customer = "CUSTOMER" + zone = "ZONE" password = "PASSWORD" port = 22225 + username = "lum-customer-%v-zone-%v" // "lum-customer-CUSTOMER-zone-ZONE" proxyTemplateURL = "http://%v-session-%v:%v@zproxy.luminati.io:%v" ) func main() { + if c := os.Getenv("CUSTOMER"); c != "" { + customer = c + } + if z := os.Getenv("ZONE"); z != "" { + zone = z + } + if p := os.Getenv("PASSWORD"); p != "" { + password = p + } + username = fmt.Sprintf(username, customer, zone) + if u := os.Getenv("USERNAME"); u != "" { + username = u + } rand.Seed(time.Now().UnixNano()) - session_id := rand.Intn(10000) - proxyUrlStr := fmt.Sprintf(proxyTemplateURL, username, session_id, password, port) + sessionId := rand.Intn(10000) + proxyUrlStr := fmt.Sprintf(proxyTemplateURL, username, sessionId, password, port) proxyUrl, err := url.Parse(proxyUrlStr) if err != nil { log.Fatalf("Error parsing proxy URL: %v", err) @@ -34,7 +50,11 @@ func main() { req, err := http.NewRequest("GET", "http://lumtest.com/myip.json", nil) req.Header.Add("Accept", "text/plain") resp, err := httpClient.Do(req) - log.Printf("Resp: %+v, err %v", resp, err) + if err != nil { + log.Fatalf("Cannot get data: %v", err) + } + defer resp.Body.Close() + log.Printf("Response: %+v, err %v", resp, err) body, err := ioutil.ReadAll(resp.Body) log.Printf("Got data: %s", body) } From 2a0e7953a2fd021cbbcee9f0f61414a86a0a01cc Mon Sep 17 00:00:00 2001 From: Eugene Lisitsky Date: Wed, 4 Apr 2018 15:30:04 +0300 Subject: [PATCH 3/3] Multithreaded version of proxychecker --- go/multi.go | 194 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 194 insertions(+) create mode 100755 go/multi.go diff --git a/go/multi.go b/go/multi.go new file mode 100755 index 0000000..cff00d1 --- /dev/null +++ b/go/multi.go @@ -0,0 +1,194 @@ +//usr/bin/env go run $0 $@; exit $? +// +// Multithreaded parallel version of proxychecker +// +// To run set env variables: +// CUSTOMER - your customer name +// ZONE - zone id +// PASSWORD - your zone password + +package main + +import ( + "fmt" + "io/ioutil" + "log" + "net/http" + "net/url" + "os" + "sync" + "time" +) + +func getProxyUrl(task Task) (*url.URL, error) { + var ( + customer = getEnvOrDefault("CUSTOMER", "customer") + zone = getEnvOrDefault("ZONE", "zone") + password = getEnvOrDefault("PASSWORD", "password") + port = 22225 + username = "lum-customer-%v-zone-%v" // "lum-customer-CUSTOMER-zone-ZONE" + proxyTemplateURL = "http://%v-session-%v:%v@zproxy.luminati.io:%v" + ) + username = fmt.Sprintf(username, customer, zone) + proxyStringUrl := fmt.Sprintf(proxyTemplateURL, username, task.session, password, port) + proxyUrl, err := url.Parse(proxyStringUrl) + if err != nil { + return nil, err + } + return proxyUrl, nil +} + +func main() { + var startAt = time.Now() + // maximal number of parallel workers + var maxInParallel = 10 + + var taskChannel = make(ChanTask) + var resultChannel = make(ChanResult) + var quitChanel = make(chan struct{}) + + // start result reader and printer to stdout + go resultPrinter(resultChannel, quitChanel) + + // start page processor + go startProcessors(taskChannel, resultChannel, maxInParallel) + + for _, taskInfo := range taskInfos { + taskChannel <- Task{TaskInfo: taskInfo, startAt: time.Now()} + } + close(taskChannel) + <-quitChanel + log.Printf("Total time: %v", time.Since(startAt)) +} + +type TaskInfo struct { + session string + href string +} +type Task struct { + TaskInfo + startAt, endAt time.Time +} +type ChanTask chan Task + +type Result struct { + session string + startAt time.Time + body []byte +} +type ChanResult chan Result + +func startProcessors(inCh ChanTask, outCh ChanResult, number int) { + var wg sync.WaitGroup + for i := 0; i < number; i++ { + go processor(inCh, outCh, &wg) + wg.Add(1) + } + wg.Wait() + close(outCh) +} + +func processor(inCh ChanTask, outCh ChanResult, wg *sync.WaitGroup) { + for task := range inCh { + proxyUrl, err := getProxyUrl(task) + if err != nil { + log.Printf("Error getting proxyURL: %v", err) + continue // ignore this task + } + result, err := processPage(task, proxyUrl) + if err != nil { + log.Printf("Error processing page url %v: %v", task.href, err) + continue // ignore this task + } + outCh <- *result + } + wg.Done() +} + +func processPage(task Task, proxyUrl *url.URL) (result *Result, err error) { + httpClient := &http.Client{ + Transport: &http.Transport{ + Proxy: http.ProxyURL(proxyUrl), + }, + } + request, err := http.NewRequest("GET", task.href, nil) + if err != nil { + log.Printf("Error forming request: %v", err) + return nil, err + } + request.Header.Add("Accept", "text/plain") + resp, err := httpClient.Do(request) + if err != nil { + log.Printf("Cannot get data: %v", err) + return nil, err + } + defer resp.Body.Close() + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + return &Result{session: task.session, startAt: task.startAt, body: body}, nil +} + +// prints results, +// to stop printing close incoming channel inCh +// before exit, closes quitChat to inform about it +func resultPrinter(inCh ChanResult, quitChan chan struct{}) { + var totalDuration time.Duration + var i = 0 + for result := range inCh { + var dur = time.Since(result.startAt) + log.Printf("%d\tStartTime: %v\tDuration: %v\tSession: %- 8s\tBody: %s", + i, result.startAt, dur, result.session, result.body) + i++ + totalDuration += dur + } + close(quitChan) + log.Printf("Total duration: %v", totalDuration) +} + +func getEnvOrDefault(key, defaultValue string) string { + value := os.Getenv(key) + if value == "" { + return defaultValue + } + return value +} + +var ( + taskInfos = []TaskInfo{ + {"12313", "http://lumtest.com/myip.json"}, + {"12313", "http://lumtest.com/myip.json"}, + {"11", "http://lumtest.com/myip.json"}, + {"2323", "http://lumtest.com/myip.json"}, + {"rer", "http://lumtest.com/myip.json"}, + {"sdf", "http://lumtest.com/myip.json"}, + {"44", "http://lumtest.com/myip.json"}, + {"343", "http://lumtest.com/myip.json"}, + {"gegd", "http://lumtest.com/myip.json"}, + {"gegd", "http://lumtest.com/myip.json"}, + {"gegd", "http://lumtest.com/myip.json"}, + {"gegd", "http://lumtest.com/myip.json"}, + {"ett4", "http://lumtest.com/myip.json"}, + {"3r3r3r", "http://lumtest.com/myip.json"}, + {"34r3", "http://lumtest.com/myip.json"}, + {"fffd", "http://lumtest.com/myip.json"}, + {"12313", "http://lumtest.com/myip.json"}, + {"sdff", "http://lumtest.com/myip.json"}, + {"12313", "http://lumtest.com/myip.json"}, + {"sfsf", "http://lumtest.com/myip.json"}, + {"12313", "http://lumtest.com/myip.json"}, + {"12313", "http://lumtest.com/myip.json"}, + {"12313", "http://lumtest.com/myip.json"}, + {"sfs", "http://lumtest.com/myip.json"}, + {"12313", "http://lumtest.com/myip.json"}, + {"12313", "http://lumtest.com/myip.json"}, + {"12313", "http://lumtest.com/myip.json"}, + {"12313", "http://lumtest.com/myip.json"}, + {"gegd", "http://lumtest.com/myip.json"}, + {"12313", "http://lumtest.com/myip.json"}, + {"00000", "http://lumtest.com/myip.json"}, + } +)