Adding support for "iframe" source_version in normalize_tweet

Author: evancheval

README.md

@@ -265,6 +265,7 @@ When setting `extract_referenced_posts` to `True` it will instead return a list
 * **locale** *(pytz.timezone as str, optional)*: timezone used to convert dates. If not given, will default to UTC.
 * **extract_referenced_posts** *(bool, optional)*: whether to return in the output, in addition to the tweet to be normalized, also normalized data for each other referenced tweets found in the payload data (including retweeted and quoted tweets). If `False`, the function will return a `dict`, if `True` a `list` of `dict`. Defaults to `False`.
 * **collection_source** *(string, optional)*: An optional information to add within the `collected_via` field of the normalized tweet to indicate whence it was collected.
+* **source_version** *(str, optional)*: version of the Twitter payload. Must be either "v1" or "iframe". Default to "v1".
 
 ### normalize_tweets_payload_v2
 

test/anonymizers_test.py

@@ -77,6 +77,8 @@ def test_anonymize_normalized_tweet(self):
                 "user_id",
                 "user_name",
                 "user_verified",
+                "user_verified_type",
+                "user_is_blue_verified",
                 "user_description",
                 "user_url",
                 "user_image",
@@ -139,6 +141,8 @@ def test_anonymize_normalized_tweet(self):
                 "",
                 "",
                 "",
+                "",
+                "",
                 "2787",
                 "1663",
                 "286",
@@ -198,6 +202,8 @@ def test_anonymize_normalized_tweet(self):
                 "",
                 "",
                 "",
+                "",
+                "",
                 "20948",
                 "462",
                 "213",
@@ -257,6 +263,8 @@ def test_anonymize_normalized_tweet(self):
                 "",
                 "",
                 "",
+                "",
+                "",
                 "6082",
                 "4554",
                 "372",
@@ -316,6 +324,8 @@ def test_anonymize_normalized_tweet(self):
                 "",
                 "",
                 "",
+                "",
+                "",
                 "5098",
                 "133",
                 "1931",

test/normalizers_test.py

@@ -230,3 +230,48 @@ def test_incomplete_includes(self):
 
         assert exc_info.value.kind == "user"
         assert exc_info.value.key == "1217864994852941825"
+
+    def test_normalize_tweet_without_api_key(self):
+        # tz = timezone("Europe/Paris")
+
+        tests = get_json_resource("normalization-no-api-key.json")
+        fn = partial(normalize_tweet, source_version="iframe")
+
+        # from test.utils import dump_json_resource
+        # for idx, test in enumerate(tests):
+        #     tests[idx]['normalized'] = fn(test['source'], extract_referenced_tweets=True)
+        # dump_json_resource(tests, 'normalization.json')
+
+        # With referenced tweets
+        for test in tests:
+            result = fn(test["source"], extract_referenced_tweets=True)
+
+            assert isinstance(result, list)
+            assert set(t["id"] for t in result) == set(
+                t["id"] for t in test["normalized"]
+            )
+
+            for tweet in result:
+                assert "collection_time" in tweet and isinstance(
+                    tweet["collection_time"], str
+                )
+
+            for t1, t2 in zip(result, test["normalized"]):
+                compare_tweets(test["source"]["id_str"], t1, t2)
+
+        # With single output
+        for test in tests:
+            tweet = fn(test["source"])
+
+            assert isinstance(tweet, dict)
+
+            _id = test["source"]["id_str"]
+            compare_tweets(
+                _id, tweet, next(t for t in test["normalized"] if t["id"] == _id)
+            )
+
+        # With custom collection_source
+        for test in tests:
+            tweet = fn(test["source"], collection_source="unit_test")
+
+            assert tweet["collected_via"] == ["unit_test"]