Terms of Service rollout (#17629)

* allow marking a flash message as safe to render HTML

* terms of use -> terms of service

* track user and organization tos engagements

* implement a task to notify users of ToS update via email

* draft of blog post

* add pricing/payment terms

* Apply suggestions from code review

Co-authored-by: Mike Fiedler <[email protected]>

* clarify that ToU is being superseded

* set effective date for existing users

* add a batch size tunable to notify_users_of_tos_update task

* Apply suggestions from code review

Co-authored-by: Mike Fiedler <[email protected]>

* suggested in code review

* refactor needs_tos_update -> needs_tos_flash, reduce interface

* rename trash field name

* lint

* comments

* clarifying

* punt

* refactor record_tos_engagement

* update factory to use enum rather than string

* translations

* dev: don't flash ToS banner to admin users

* Apply suggestions from code review

Co-authored-by: Mike Fiedler <[email protected]>

---------

Co-authored-by: Mike Fiedler <[email protected]>

Author: ewdurbin

dev/db/post-migrations.sql

@@ -50,6 +50,9 @@ VALUES
 -- Set TOTP secret to IU7UP3EMIPI7EBPQUUSEHEJUFNBIWOYG for select users
 UPDATE users SET totp_secret = '\x453f47ec8c43d1f205f0a5244391342b428b3b06' WHERE username IN ('ewdurbin', 'di', 'dstufft', 'miketheman');
 
+-- Create Terms of Service Engagements for select users to keep from flashing banner on login
+INSERT INTO user_terms_of_service_engagements (user_id, revision, created, engagement) (SELECT id, 'initial', NOW(), 'agreed' from users where username IN ('ewdurbin', 'di', 'dstufft', 'miketheman'));
+
 -- Make select users owners of 'sampleproject'
 INSERT INTO roles (role_name, user_id, project_id)
   SELECT 'Owner', id, '4587cc12-e342-4880-9f61-ea4990fb81ea'

dev/environment

@@ -5,6 +5,8 @@ WAREHOUSE_ENV=development
 WAREHOUSE_TOKEN=insecuretoken
 WAREHOUSE_IP_SALT="insecure himalayan pink salt"
 
+TERMS_NOTIFICATION_BATCH_SIZE=0
+
 AWS_ACCESS_KEY_ID=foo
 AWS_SECRET_ACCESS_KEY=foo
 

docs/blog/posts/2025-02-25-terms-of-service.md

@@ -0,0 +1,58 @@
+---
+title: Introducing our new Terms of Service
+description: PyPI is formalizing our policies to help us move forward with new services.
+authors:
+  - ewdurbin
+date: 2025-02-25
+tags:
+  - policies
+  - transparency
+---
+
+We're introducing a new
+[Terms of Service](https://policies.python.org/pypi.org/Terms-of-Service/)
+to formalize our relationship to users
+and enable us to move forward with providing new features and services,
+specifically
+[Organization Accounts](https://docs.pypi.org/organization-accounts/).
+
+<!-- more -->
+
+PyPI has had some form of [Terms of Use](https://policies.python.org/pypi.org/Terms-of-Use/)
+document for users since it
+[began accepting uploads in 2005](https://github.com/pypi/legacy/commit/b139c00cfc5794159afb1fc185d77dbc5fc1a2a4#diff-a67499b048e6bb6ef08d44c7a3c541199615b68e3bd153eb0ccedc492e3dec9dR7-R13)
+and has only been updated twice[^1] since.
+These terms have primarily served to protect PyPI
+and the Python Software Foundation (PSF) who operates it.
+
+Over time we have introduced additional policies to protect our users and community
+such as our
+[Code of Conduct](https://policies.python.org/python.org/code-of-conduct/)
+[Privacy Notice](https://policies.python.org/pypi.org/Privacy-Notice/)
+and
+[Acceptable Use Policy](https://policies.python.org/pypi.org/Acceptable-Use-Policy/).
+
+Our new
+[Terms of Service](https://policies.python.org/pypi.org/Terms-of-Service/)
+formalizes our relationship to PyPI users,
+makes protections for the PSF and PyPI users more explicit,
+and establishes terms we need to provide
+[Organization Accounts](https://docs.pypi.org/organization-accounts/)
+to paid
+[Corporate Organizations](https://docs.pypi.org/organization-accounts/pricing-and-payments/#corporate-organizations).
+
+We have worked with our legal team to retain compatibility with the superseded
+[Terms of Use](https://policies.python.org/pypi.org/Terms-of-Use/)
+while adding as permissive a set of new terms as possible to ensure that PyPI users 
+and the PSF are protected.
+
+You will notice a banner on login reminding you of these updated terms,
+as well as an email notification to your primary email address if it has been verified.
+These terms will take effect for existing users March 27, 2025 and
+your continued use of PyPI after that date constitutes agreement to these new terms.
+
+[^1]: 
+  See these commits for substantive changes since the Terms of Use was introduced:
+  [2009-11-29](https://github.com/pypi/legacy/commit/ddbd32a78a431ab46cad912046c2492998edc618#diff-a6e30135c956f467cffa36eb37a756a53921754d55ddd6ea80d2a0b4c3f4abfaR16-R33)
+  and
+  [2016-12-16](https://github.com/pypi/legacy/commit/f645942c65a372fdacd4d48ffb4afed4502632e8#diff-bbf95bcc6416475537256acea89690f7c6b1f965c0306e9b883813bd3e4f6c10R15-R98).

docs/user/organization-accounts/pricing-and-payments.md

@@ -10,11 +10,11 @@ title: Pricing and Payments
 
 ### Pricing
 
-Pricing for Corporate Organizations will be published at launch.
+* $5 per member User, per month
 
 ### Payment Terms
 
-Payment terms for Corporate Organizations will be published at launch.
+Invoiced monthly, based on usage. Payment is due upon reciept and will be charged to the billing information on file.
 
 ## Community Organizations
 

tests/common/db/accounts.py

@@ -21,7 +21,9 @@
     Email,
     ProhibitedEmailDomain,
     ProhibitedUserName,
+    TermsOfServiceEngagement,
     User,
+    UserTermsOfServiceEngagement,
 )
 
 from .base import WarehouseFactory
@@ -43,6 +45,13 @@ class Params:
                 verified=True,
             )
         )
+        # Shortcut to create a user with a ToS Agreement
+        with_terms_of_service_agreement = factory.Trait(
+            terms_of_service_engagements=factory.RelatedFactory(
+                "tests.common.db.accounts.UserTermsOfServiceEngagementFactory",
+                factory_related_name="user",
+            )
+        )
         # Allow passing a cleartext password to the factory
         # This will be hashed before saving the user.
         # Usage: UserFactory(clear_pwd="password")
@@ -85,6 +94,19 @@ class Meta:
     source = factory.SubFactory(User)
 
 
+class UserTermsOfServiceEngagementFactory(WarehouseFactory):
+    class Meta:
+        model = UserTermsOfServiceEngagement
+
+    revision = "initial"
+    engagement = TermsOfServiceEngagement.Agreed
+    created = factory.Faker(
+        "date_time_between_dates",
+        datetime_start=datetime.datetime(2025, 1, 1),
+        datetime_end=datetime.datetime(2025, 2, 19),
+    )
+
+
 class EmailFactory(WarehouseFactory):
     class Meta:
         model = Email

tests/conftest.py

@@ -332,6 +332,7 @@ def get_app_config(database, nondefaults=None):
         "sessions.url": "redis://localhost:0/",
         "statuspage.url": "https://2p66nmmycsj3.statuspage.io",
         "warehouse.xmlrpc.cache.url": "redis://localhost:0/",
+        "terms.revision": "initial",
     }
 
     if nondefaults:

tests/functional/manage/test_views.py

@@ -57,7 +57,9 @@ def test_changing_password_succeeds(self, webtest, socket_enabled):
         """A user can log in, and change their password."""
         # create a User
         user = UserFactory.create(
-            with_verified_primary_email=True, clear_pwd="password"
+            with_verified_primary_email=True,
+            with_terms_of_service_agreement=True,
+            clear_pwd="password",
         )
 
         # visit login page

tests/unit/accounts/test_services.py

@@ -42,13 +42,22 @@
     TooManyEmailsAdded,
     TooManyFailedLogins,
 )
-from warehouse.accounts.models import DisableReason, ProhibitedUserName
+from warehouse.accounts.models import (
+    DisableReason,
+    ProhibitedUserName,
+    TermsOfServiceEngagement,
+    UserTermsOfServiceEngagement,
+)
 from warehouse.events.tags import EventTag
 from warehouse.metrics import IMetricsService, NullMetrics
 from warehouse.rate_limiting.interfaces import IRateLimiter
 
 from ...common.constants import REMOTE_ADDR
-from ...common.db.accounts import EmailFactory, UserFactory
+from ...common.db.accounts import (
+    EmailFactory,
+    UserFactory,
+    UserTermsOfServiceEngagementFactory,
+)
 from ...common.db.ip_addresses import IpAddressFactory
 
 
@@ -1049,6 +1058,88 @@ def test_get_password_timestamp_no_value(self, user_service):
 
         assert user_service.get_password_timestamp(user.id) == 0
 
+    def test_needs_tos_flash_no_engagements(self, user_service):
+        user = UserFactory.create()
+        assert user_service.needs_tos_flash(user.id, "initial") is True
+
+    def test_needs_tos_flash_with_passive_engagements(self, user_service):
+        user = UserFactory.create()
+        assert user_service.needs_tos_flash(user.id, "initial") is True
+
+        user_service.record_tos_engagement(
+            user.id, "initial", TermsOfServiceEngagement.Notified
+        )
+        assert user_service.needs_tos_flash(user.id, "initial") is True
+
+        user_service.record_tos_engagement(
+            user.id, "initial", TermsOfServiceEngagement.Flashed
+        )
+        assert user_service.needs_tos_flash(user.id, "initial") is True
+
+    def test_needs_tos_flash_with_viewed_engagement(self, user_service):
+        user = UserFactory.create()
+        assert user_service.needs_tos_flash(user.id, "initial") is True
+
+        user_service.record_tos_engagement(
+            user.id, "initial", TermsOfServiceEngagement.Viewed
+        )
+        assert user_service.needs_tos_flash(user.id, "initial") is False
+
+    def test_needs_tos_flash_with_agreed_engagement(self, user_service):
+        user = UserFactory.create()
+        assert user_service.needs_tos_flash(user.id, "initial") is True
+
+        user_service.record_tos_engagement(
+            user.id, "initial", TermsOfServiceEngagement.Agreed
+        )
+        assert user_service.needs_tos_flash(user.id, "initial") is False
+
+    def test_needs_tos_flash_if_engaged_more_than_30_days_ago(self, user_service):
+        user = UserFactory.create()
+        UserTermsOfServiceEngagementFactory.create(
+            user=user,
+            created=(datetime.datetime.now(datetime.UTC) - datetime.timedelta(days=31)),
+            engagement=TermsOfServiceEngagement.Notified,
+        )
+        assert user_service.needs_tos_flash(user.id, "initial") is False
+
+    def test_record_tos_engagement_invalid_engagement(self, user_service):
+        user = UserFactory.create()
+        assert user.terms_of_service_engagements == []
+        with pytest.raises(ValueError):  # noqa: PT011
+            user_service.record_tos_engagement(
+                user.id,
+                "initial",
+                None,
+            )
+
+    @pytest.mark.parametrize(
+        "engagement",
+        [
+            TermsOfServiceEngagement.Flashed,
+            TermsOfServiceEngagement.Notified,
+            TermsOfServiceEngagement.Viewed,
+            TermsOfServiceEngagement.Agreed,
+        ],
+    )
+    def test_record_tos_engagement(self, user_service, db_request, engagement):
+        user = UserFactory.create()
+        assert user.terms_of_service_engagements == []
+        user_service.record_tos_engagement(
+            user.id,
+            "initial",
+            engagement,
+        )
+        assert (
+            db_request.db.query(UserTermsOfServiceEngagement)
+            .filter(
+                UserTermsOfServiceEngagement.user_id == user.id,
+                UserTermsOfServiceEngagement.revision == "initial",
+                UserTermsOfServiceEngagement.engagement == engagement,
+            )
+            .count()
+        ) == 1
+
 
 class TestTokenService:
     def test_verify_service(self):

tests/unit/accounts/test_tasks.py

@@ -13,13 +13,108 @@
 from datetime import datetime, timedelta, timezone
 
 import pretend
+import pytest
 
-from warehouse.accounts.tasks import compute_user_metrics
+from warehouse.accounts import tasks
+from warehouse.accounts.models import TermsOfServiceEngagement
+from warehouse.accounts.tasks import compute_user_metrics, notify_users_of_tos_update
 
 from ...common.db.accounts import EmailFactory, UserFactory
 from ...common.db.packaging import ProjectFactory, ReleaseFactory
 
 
+def test_notify_users_of_tos_update(db_request, user_service, monkeypatch):
+    db_request.registry.settings = {"terms.revision": "initial"}
+    users_to_notify = UserFactory.create_batch(3, with_verified_primary_email=True)
+    # Users we should not notify because they have already agreed to ToS
+    UserFactory.create_batch(
+        5, with_verified_primary_email=True, with_terms_of_service_agreement=True
+    )
+    # Users we should not notify because they don't have a primary/verified email
+    UserFactory.create_batch(7)
+
+    send_email = pretend.call_recorder(lambda request, user: None)
+    monkeypatch.setattr(tasks, "send_user_terms_of_service_updated", send_email)
+
+    user_service.record_tos_engagement = pretend.call_recorder(
+        lambda user_id, revision, engagement: None
+    )
+
+    notify_users_of_tos_update(db_request)
+
+    assert send_email.calls == [pretend.call(db_request, u) for u in users_to_notify]
+    assert user_service.record_tos_engagement.calls == [
+        pretend.call(u.id, "initial", TermsOfServiceEngagement.Notified)
+        for u in users_to_notify
+    ]
+
+
+@pytest.mark.parametrize("batch_size", [0, 10])
+def test_notify_users_of_tos_update_respects_batch_size(
+    db_request, batch_size, user_service, monkeypatch
+):
+    db_request.registry.settings = {
+        "terms.revision": "initial",
+        "terms.notification_batch_size": batch_size,
+    }
+    users_to_notify = UserFactory.create_batch(20, with_verified_primary_email=True)
+
+    send_email = pretend.call_recorder(lambda request, user: None)
+    monkeypatch.setattr(tasks, "send_user_terms_of_service_updated", send_email)
+
+    user_service.record_tos_engagement = pretend.call_recorder(
+        lambda user_id, revision, engagement: None
+    )
+
+    notify_users_of_tos_update(db_request)
+
+    assert (
+        send_email.calls
+        == [pretend.call(db_request, u) for u in users_to_notify][:batch_size]
+    )
+    assert (
+        user_service.record_tos_engagement.calls
+        == [
+            pretend.call(u.id, "initial", TermsOfServiceEngagement.Notified)
+            for u in users_to_notify
+        ][:batch_size]
+    )
+
+
+def test_notify_users_of_tos_update_does_not_renotify(
+    db_request, user_service, monkeypatch
+):
+    db_request.registry.settings = {"terms.revision": "initial"}
+    users_to_notify = UserFactory.create_batch(3, with_verified_primary_email=True)
+    # Users we should not notify because they have already agreed to ToS
+    UserFactory.create_batch(
+        5, with_verified_primary_email=True, with_terms_of_service_agreement=True
+    )
+    # Users we should not notify because they don't have a primary/verified email
+    UserFactory.create_batch(7)
+
+    send_email = pretend.call_recorder(lambda request, user: None)
+    monkeypatch.setattr(tasks, "send_user_terms_of_service_updated", send_email)
+
+    user_service.record_tos_engagement(
+        users_to_notify[-1].id, "initial", TermsOfServiceEngagement.Notified
+    )
+
+    user_service.record_tos_engagement = pretend.call_recorder(
+        lambda user_id, revision, engagement: None
+    )
+
+    notify_users_of_tos_update(db_request)
+
+    assert send_email.calls == [
+        pretend.call(db_request, u) for u in users_to_notify[:-1]
+    ]
+    assert user_service.record_tos_engagement.calls == [
+        pretend.call(u.id, "initial", TermsOfServiceEngagement.Notified)
+        for u in users_to_notify[:-1]
+    ]
+
+
 def _create_old_users_and_releases():
     users = UserFactory.create_batch(3, is_active=True)
     for user in users: