Skip to content

Commit

Permalink
update gql endpoints
Browse files Browse the repository at this point in the history
  • Loading branch information
@vladkens
vladkens committed Sep 8, 2023
1 parent a742fe6 commit d53535f
Show file tree
Hide file tree
Showing 17 changed files with 4,400 additions and 5,686 deletions.
8 changes: 0 additions & 8 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,18 +47,12 @@ changelog:

test34:
@# https://www.sqlite.org/chronology.html
# @$(SQTEST) --build-arg SQLY=2017 --build-arg SQLV=3210000 -t twscrape_sq21 .
# @$(SQTEST) --build-arg SQLY=2018 --build-arg SQLV=3220000 -t twscrape_sq22 .
@$(SQTEST) --build-arg SQLY=2018 --build-arg SQLV=3230000 -t twscrape_sq23 .
@$(SQTEST) --build-arg SQLY=2018 --build-arg SQLV=3240000 -t twscrape_sq24 .
@$(SQTEST) --build-arg SQLY=2019 --build-arg SQLV=3270200 -t twscrape_sq27 .
@$(SQTEST) --build-arg SQLY=2019 --build-arg SQLV=3300100 -t twscrape_sq30 .
@$(SQTEST) --build-arg SQLY=2020 --build-arg SQLV=3330000 -t twscrape_sq33 .
@$(SQTEST) --build-arg SQLY=2021 --build-arg SQLV=3340100 -t twscrape_sq34 .
@$(SQTEST) --build-arg SQLY=2023 --build-arg SQLV=3430000 -t twscrape_sq43 .
# @docker run twscrape_sq21
# @docker run twscrape_sq22
# @docker run twscrape_sq23
@docker run twscrape_sq24
@docker run twscrape_sq27
@docker run twscrape_sq30
Expand All @@ -77,5 +71,3 @@ update-mocks:
twscrape user_tweets --raw --limit 10 2244994945 | jq > ./tests/mocked-data/user_tweets_raw.json
twscrape user_tweets_and_replies --raw --limit 10 2244994945 | jq > ./tests/mocked-data/user_tweets_and_replies_raw.json
twscrape search --raw --limit 10 "elon musk lang:en" | jq > ./tests/mocked-data/search_raw.json

twscrape tweet_details --raw 1665951747842641921 | jq > ./tests/mocked-data/_issue_42_regr.json
3 changes: 2 additions & 1 deletion _get_gql_ops.py
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
import httpx

# update this url on next run
url = "https://abs.twimg.com/responsive-web/client-web/api.f4ff3bfa.js"
# url = "https://abs.twimg.com/responsive-web/client-web/api.f4ff3bfa.js"
url = "https://abs.twimg.com/responsive-web/client-web/api.bb81931a.js"
script = httpx.get(url).text

ops = """
Expand Down
4 changes: 2 additions & 2 deletions readme.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,8 @@
<a href="https://github.com/vladkens/twscrape/actions">
<img src="https://github.com/vladkens/twscrape/workflows/test/badge.svg" alt="test status" />
</a>
<!-- <a href="https://npmjs.org/package/array-utils-ts">
<img src="https://badgen.net/npm/dm/array-utils-ts" alt="downloads" />
<!-- <a href="https://pypi.org/project/twscrape">
<img src="https://badgen.net/pypi/dm/twscrape" alt="downloads" />
</a> -->
<a href="https://github.com/vladkens/twscrape/blob/main/LICENSE">
<img src="https://badgen.net/github/license/vladkens/twscrape" alt="license" />
Expand Down
316 changes: 112 additions & 204 deletions tests/mocked-data/_issue_42_regr.json
View file

Large diffs are not rendered by default.

68 changes: 34 additions & 34 deletions tests/mocked-data/favoriters_raw.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,23 +37,23 @@
}
},
"fast_followers_count": 0,
"favourites_count": 742,
"followers_count": 29,
"favourites_count": 745,
"followers_count": 28,
"friends_count": 355,
"has_custom_timelines": false,
"is_translator": false,
"listed_count": 0,
"location": "",
"media_count": 1,
"name": "Jimmy🇪🇬🇹🇷🇹🇼🇮🇹🇲🇾🇯🇵🇹🇭🇳🇿🇪🇸🇲🇺",
"normal_followers_count": 29,
"normal_followers_count": 28,
"pinned_tweet_ids_str": [],
"possibly_sensitive": false,
"profile_banner_url": "https://pbs.twimg.com/profile_banners/1694091590065864704/1693328949",
"profile_image_url_https": "https://pbs.twimg.com/profile_images/1696570691267813377/rbACa8Ds_normal.jpg",
"profile_interstitial_type": "",
"screen_name": "lumnlg021091",
"statuses_count": 84,
"statuses_count": 85,
"translator_type": "none",
"verified": false,
"want_retweets": false,
Expand Down Expand Up @@ -109,16 +109,16 @@
}
},
"fast_followers_count": 0,
"favourites_count": 27915,
"followers_count": 555,
"favourites_count": 27934,
"followers_count": 557,
"friends_count": 335,
"has_custom_timelines": false,
"is_translator": false,
"listed_count": 4,
"location": "",
"media_count": 3629,
"media_count": 3639,
"name": "WaifuCheckPoint",
"normal_followers_count": 555,
"normal_followers_count": 557,
"pinned_tweet_ids_str": [
"1676868435333054465"
],
Expand All @@ -127,7 +127,7 @@
"profile_image_url_https": "https://pbs.twimg.com/profile_images/1699590247972052992/NS_TwAfZ_normal.jpg",
"profile_interstitial_type": "",
"screen_name": "WaifuCheckPoint",
"statuses_count": 8173,
"statuses_count": 8191,
"translator_type": "none",
"url": "https://t.co/KSBXe1cMMf",
"verified": false,
Expand Down Expand Up @@ -171,7 +171,7 @@
}
},
"fast_followers_count": 0,
"favourites_count": 4552,
"favourites_count": 4555,
"followers_count": 29,
"friends_count": 50,
"has_custom_timelines": false,
Expand Down Expand Up @@ -245,7 +245,7 @@
}
},
"fast_followers_count": 0,
"favourites_count": 17832,
"favourites_count": 17839,
"followers_count": 1024,
"friends_count": 4991,
"has_custom_timelines": true,
Expand All @@ -263,7 +263,7 @@
"profile_image_url_https": "https://pbs.twimg.com/profile_images/3151864799/a120a6c9fc91333cf88f233fb3879bff_normal.jpeg",
"profile_interstitial_type": "",
"screen_name": "MadhuAnantharaj",
"statuses_count": 2899,
"statuses_count": 2900,
"translator_type": "regular",
"url": "https://t.co/nT5eAU6kTA",
"verified": false,
Expand Down Expand Up @@ -352,7 +352,7 @@
"fast_followers_count": 0,
"favourites_count": 182555,
"followers_count": 282,
"friends_count": 5001,
"friends_count": 5002,
"has_custom_timelines": false,
"is_translator": false,
"listed_count": 7,
Expand Down Expand Up @@ -523,16 +523,16 @@
}
},
"fast_followers_count": 0,
"favourites_count": 46734,
"followers_count": 2799,
"friends_count": 1609,
"favourites_count": 46748,
"followers_count": 2800,
"friends_count": 1611,
"has_custom_timelines": true,
"is_translator": false,
"listed_count": 52,
"location": "price discovery",
"media_count": 150,
"name": "natalie 💫",
"normal_followers_count": 2799,
"normal_followers_count": 2800,
"pinned_tweet_ids_str": [
"1577388145032585216"
],
Expand Down Expand Up @@ -650,15 +650,15 @@
},
"fast_followers_count": 0,
"favourites_count": 1944,
"followers_count": 20459,
"followers_count": 20488,
"friends_count": 19038,
"has_custom_timelines": true,
"is_translator": false,
"listed_count": 10,
"location": "Tangail, Dhaka.",
"media_count": 87,
"name": "❤️ Follow back 💥💥💥",
"normal_followers_count": 20459,
"normal_followers_count": 20488,
"pinned_tweet_ids_str": [
"1681770881658556416"
],
Expand Down Expand Up @@ -770,15 +770,15 @@
},
"fast_followers_count": 0,
"favourites_count": 131262,
"followers_count": 3319,
"friends_count": 7,
"followers_count": 3318,
"friends_count": 8,
"has_custom_timelines": true,
"is_translator": false,
"listed_count": 1,
"location": "",
"media_count": 0,
"name": "imn",
"normal_followers_count": 3319,
"normal_followers_count": 3318,
"pinned_tweet_ids_str": [
"1557924779130232832"
],
Expand Down Expand Up @@ -843,7 +843,7 @@
}
},
"fast_followers_count": 0,
"favourites_count": 17623,
"favourites_count": 17629,
"followers_count": 71,
"friends_count": 910,
"has_custom_timelines": false,
Expand All @@ -854,14 +854,14 @@
"name": "PatrickWallaceEbertz🌈",
"normal_followers_count": 71,
"pinned_tweet_ids_str": [
"1699954759233568877"
"1700162661818777954"
],
"possibly_sensitive": false,
"profile_banner_url": "https://pbs.twimg.com/profile_banners/3163262162/1657398100",
"profile_image_url_https": "https://pbs.twimg.com/profile_images/1545865702967431168/aq7Hwqee_normal.jpg",
"profile_interstitial_type": "",
"screen_name": "PatchyPatchy17",
"statuses_count": 3439,
"statuses_count": 3445,
"translator_type": "none",
"url": "https://t.co/GRUwBnZnQD",
"verified": false,
Expand Down Expand Up @@ -918,7 +918,7 @@
"fast_followers_count": 0,
"favourites_count": 15042,
"followers_count": 1030,
"friends_count": 2067,
"friends_count": 2068,
"has_custom_timelines": true,
"is_translator": false,
"listed_count": 17,
Expand Down Expand Up @@ -975,7 +975,7 @@
}
},
"fast_followers_count": 0,
"favourites_count": 14074,
"favourites_count": 14080,
"followers_count": 93,
"friends_count": 258,
"has_custom_timelines": true,
Expand Down Expand Up @@ -1046,7 +1046,7 @@
}
},
"fast_followers_count": 0,
"favourites_count": 20341,
"favourites_count": 20353,
"followers_count": 221,
"friends_count": 512,
"has_custom_timelines": true,
Expand Down Expand Up @@ -1250,16 +1250,16 @@
}
},
"fast_followers_count": 0,
"favourites_count": 4486,
"followers_count": 873,
"friends_count": 4064,
"favourites_count": 4488,
"followers_count": 874,
"friends_count": 4081,
"has_custom_timelines": false,
"is_translator": false,
"listed_count": 3,
"location": "",
"media_count": 1,
"name": "Gemma Charley",
"normal_followers_count": 873,
"normal_followers_count": 874,
"pinned_tweet_ids_str": [],
"possibly_sensitive": false,
"profile_banner_url": "https://pbs.twimg.com/profile_banners/1602930202002456577/1671216177",
Expand Down Expand Up @@ -1324,15 +1324,15 @@
},
"fast_followers_count": 0,
"favourites_count": 3060,
"followers_count": 38,
"followers_count": 39,
"friends_count": 206,
"has_custom_timelines": false,
"is_translator": false,
"listed_count": 2,
"location": "Faro",
"media_count": 4,
"name": "Alan Martins",
"normal_followers_count": 38,
"normal_followers_count": 39,
"pinned_tweet_ids_str": [],
"possibly_sensitive": false,
"profile_banner_url": "https://pbs.twimg.com/profile_banners/1046279048610156544/1688091041",
Expand Down
Loading

0 comments on commit d53535f

Please sign in to comment.