From f22aa20d292461c2a4de3d39a3dea8bee2c0f062 Mon Sep 17 00:00:00 2001 From: Max Treskin Date: Thu, 4 Feb 2021 01:20:57 +0300 Subject: [PATCH 1/4] Add ability to query extended tweet --- twitter_dl/downloader.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/twitter_dl/downloader.py b/twitter_dl/downloader.py index dde79b8..d11ec32 100644 --- a/twitter_dl/downloader.py +++ b/twitter_dl/downloader.py @@ -55,11 +55,11 @@ class Downloader: raise RuntimeError("Bearer TokenNot Fetched") def download_media_of_tweet(self, tid, save_dest, size="large", include_video=False, - include_photo=True): + include_photo=True, is_extended=False): ''' ''' save_dest = ensure_dir(save_dest) - tweet = self.get_tweet(tid) + tweet = self.get_tweet(tid, is_extended) self.process_tweet(tweet, save_dest, size, include_video, include_photo) def download_media_of_user(self, user, save_dest, size="large", limit=3200, rts=False, @@ -165,25 +165,28 @@ class Downloader: return self.api_fetch_tweets(apiurl, payload, start, count, rts, since_id) - def get_tweet(self, id): + def get_tweet(self, tid, is_extended=False): """Download single tweet Args: - id: Tweet ID. + tid: Tweet ID. + is_extended: extended tweet mode """ bearer_token = self.bearer_token url = "https://api.twitter.com/1.1/statuses/show.json" headers = {"Authorization": f"Bearer {bearer_token}"} - payload = {"id": id, "include_entities": "true"} - + payload = {"id": tid, "include_entities": "true"} + if is_extended: + self.log.info("Extended mode") + payload["tweet_mode"] = "extended" # get the request r = requests.get(url, headers=headers, params=payload) # check the response if r.status_code == 200: tweet = r.json() - self.log.info(f"Got tweet with id {id} of user @{tweet['user']['name']}") + self.log.info(f"Got tweet with id {tid} of user @{tweet['user']['name']}") return tweet else: self.log.error(f"{url} error, code was {r.status_code}") From af196d7b48140adb8f3bab7e781c1044e1c7a068 Mon Sep 17 00:00:00 2001 From: Max Treskin Date: Thu, 4 Feb 2021 01:21:43 +0300 Subject: [PATCH 2/4] version 0.2.1 --- twitter_dl/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/twitter_dl/__init__.py b/twitter_dl/__init__.py index 915d3ed..78a9568 100644 --- a/twitter_dl/__init__.py +++ b/twitter_dl/__init__.py @@ -1,4 +1,4 @@ from .downloader import Downloader from .threaded_aio_dlder import AioDownloader -version = "0.2.0" +version = "0.2.1" From 1210088d9d7b16221dcea967c14cf26a0da7a863 Mon Sep 17 00:00:00 2001 From: Max Treskin Date: Wed, 3 Mar 2021 16:01:13 +0300 Subject: [PATCH 3/4] Allow switch off go to quoted tweet --- twitter_dl/downloader.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/twitter_dl/downloader.py b/twitter_dl/downloader.py index d11ec32..7269c15 100644 --- a/twitter_dl/downloader.py +++ b/twitter_dl/downloader.py @@ -55,12 +55,12 @@ class Downloader: raise RuntimeError("Bearer TokenNot Fetched") def download_media_of_tweet(self, tid, save_dest, size="large", include_video=False, - include_photo=True, is_extended=False): + include_photo=True, is_extended=False, goto_quoted=False): ''' ''' save_dest = ensure_dir(save_dest) tweet = self.get_tweet(tid, is_extended) - self.process_tweet(tweet, save_dest, size, include_video, include_photo) + self.process_tweet(tweet, save_dest, size, include_video, include_photo, goto_quoted) def download_media_of_user(self, user, save_dest, size="large", limit=3200, rts=False, include_video=False, include_photo=True, since_id=0): @@ -192,11 +192,11 @@ class Downloader: self.log.error(f"{url} error, code was {r.status_code}") return None - def process_tweet(self, tweet, save_dest, size="large", include_video=False, include_photo=True): + def process_tweet(self, tweet, save_dest, size="large", include_video=False, include_photo=True, goto_quoted=False): if 'retweeted_status' in tweet: tweet = tweet['retweeted_status'] self.log.debug('this is a retweet, turn to orignal tweet') - elif ("quoted_status" in tweet): + elif ("quoted_status" in tweet) and goto_quoted: tweet = tweet['quoted_status'] self.log.debug('this is a quoted tweet, turn to orignal tweet') @@ -214,10 +214,12 @@ class Downloader: Args: tweet: A dict object representing a tweet. """ - extended = tweet.get("extended_entities") + extended = tweet.get("entities") + if "media" not in extended: + extended = tweet.get("extended_entities") if not extended: return [] - + rv = [] if "media" in extended: for x in extended["media"]: From 9caf0669a7ecd903be8a381a830bc9773b803d78 Mon Sep 17 00:00:00 2001 From: Max Treskin Date: Wed, 3 Mar 2021 16:25:23 +0300 Subject: [PATCH 4/4] Use extended entities --- twitter_dl/downloader.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/twitter_dl/downloader.py b/twitter_dl/downloader.py index 7269c15..b0c8edb 100644 --- a/twitter_dl/downloader.py +++ b/twitter_dl/downloader.py @@ -214,9 +214,7 @@ class Downloader: Args: tweet: A dict object representing a tweet. """ - extended = tweet.get("entities") - if "media" not in extended: - extended = tweet.get("extended_entities") + extended = tweet.get("extended_entities") if not extended: return []