This repository was archived by the owner on Nov 7, 2023. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathScraper.cs
163 lines (133 loc) · 4.97 KB
/
Scraper.cs
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
using System;
using System.IO;
using System.Collections.Generic;
using System.Net;
using System.Linq;
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using ScrapySharp.Extensions;
using ScrapySharp.Html.Dom;
using MongoDB.Driver;
using MongoDB.Bson;
using Discord;
using Discord.Webhook;
namespace MembershipScraperNG
{
public class Scraper
{
public ICookieStore cookieStore;
string heartbeatUrl;
string failUrl;
string mongoUrl;
string mongoCollection;
string sumsUrl;
MongoClient client;
IMongoDatabase database;
IMongoCollection<BsonDocument> collection;
public Scraper(ICookieStore c)
{
this.cookieStore = c;
heartbeatUrl = Environment.GetEnvironmentVariable("heartbeatUrl");
failUrl = Environment.GetEnvironmentVariable("failUrl");
mongoUrl = Environment.GetEnvironmentVariable("mongoUrl");
mongoCollection = Environment.GetEnvironmentVariable("mongoCollection");
sumsUrl = Environment.GetEnvironmentVariable("sumsUrl");
client = new MongoClient(new MongoUrl(mongoUrl));
database = client.GetDatabase(mongoCollection);
collection = database.GetCollection<BsonDocument>("members");
}
public async Task GetMembers()
{
#region make_request
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(sumsUrl);
request.CookieContainer = new CookieContainer();
request.CookieContainer.Add(new Cookie("su_session", cookieStore.Cookie, "/", "student-dashboard.sums.su"));
HttpWebResponse response = (HttpWebResponse)(await request.GetResponseAsync());
StreamReader readStream = new StreamReader(response.GetResponseStream(), System.Text.Encoding.UTF8);
#endregion
var members = ScrapeHtml(readStream);
Console.WriteLine("Scraped {0} Members", members.Count);
await updateMembers(members);
#region update_cookie
string s = response.Headers["Set-Cookie"];
Console.WriteLine(s);
if (s != null)
{
var val = CookieValue(s, "su_session");
Console.WriteLine(val);
cookieStore.Cookie = val.Remove(val.Length - 1, 1); ;
}
#endregion
#region send heartbeat
if(members.Count > 0)
{
HttpWebRequest heartbeat = (HttpWebRequest)WebRequest.Create(heartbeatUrl);
await heartbeat.GetResponseAsync();
} else {
DiscordWebhook hook = new DiscordWebhook();
hook.Url = failUrl;
DiscordMessage message = new DiscordMessage();
message.Content = "Scraping Returned 0 members :(";
hook.Send(message);
}
#endregion
#region cleanup
response.Close();
readStream.Close();
#endregion
}
protected List<Member> ScrapeHtml(StreamReader s)
{
var src = s.ReadToEnd();
var html = HDocument.Parse(src);
var rs = html.CssSelect("#group-member-list-datatable > tbody > tr");
var ds = rs.Select(r => r.Children.Select(vs => vs.InnerText));
var ms = new List<Member>();
foreach (var r in ds)
{
var x = r.ToList();
if (x.Count != 11)
{
foreach (var z in x)
{
Console.WriteLine(z.Trim());
}
Console.WriteLine();
continue;
}
var m = new Member(Convert.ToUInt32(x[1].Trim()), x[3].Trim(), x[5].Trim(), x[9].Trim());
ms.Add(m);
}
return ms;
}
protected async Task updateMembers(List<Member> ms)
{
var xs = new List<Task>();
foreach (var m in ms)
{
xs.Add(AddOrUpdate(m));
}
foreach (var x in xs)
{
await x;
}
}
public async Task AddOrUpdate(Member m)
{
var document = BsonDocument.Parse(JsonSerializer.Serialize(m));
var filter = Builders<BsonDocument>.Filter.Eq("ID", m.ID);
var nMatches = await collection.Find(filter).CountDocumentsAsync();
if (nMatches == 0)
{
await collection.InsertOneAsync(document);
}
}
private string CookieValue(string header, string name)
{
MatchCollection Ms = Regex.Matches(header, string.Format("{0}=(?<value>.*?);", name));
return Ms.Select(M => M.ToString().Split('=')[1]).Where(x => x != "deleted;").First();
}
}
}