Sometimes include deletion times in domain-list exports (#2602)

We only include the deletion time if the domain is in the 5-day
PENDING_DELETE period after the 30 day REDEMPTION period. For all other
domains, we just have an empty string as that field.

This is behind a feature flag so that we can control when it is enabled

Author: gbrodman

core/src/main/java/google/registry/export/ExportDomainListsAction.java

@@ -17,6 +17,7 @@
 import static com.google.common.base.Verify.verifyNotNull;
 import static google.registry.model.tld.Tlds.getTldsOfType;
 import static google.registry.persistence.PersistenceModule.TransactionIsolationLevel.TRANSACTION_REPEATABLE_READ;
+import static google.registry.persistence.transaction.TransactionManagerFactory.replicaTm;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.request.Action.Method.POST;
 import static java.nio.charset.StandardCharsets.UTF_8;
@@ -28,6 +29,9 @@
 import com.google.common.net.MediaType;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.gcs.GcsUtils;
+import google.registry.model.common.FeatureFlag;
+import google.registry.model.domain.rgp.GracePeriodStatus;
+import google.registry.model.eppcommon.StatusValue;
 import google.registry.model.tld.Tld;
 import google.registry.model.tld.Tld.TldType;
 import google.registry.request.Action;
@@ -38,8 +42,13 @@
 import java.io.OutputStream;
 import java.io.OutputStreamWriter;
 import java.io.Writer;
+import java.time.Instant;
 import java.util.List;
 import javax.inject.Inject;
+import org.hibernate.query.NativeQuery;
+import org.hibernate.query.TupleTransformer;
+import org.joda.time.DateTime;
+import org.joda.time.DateTimeZone;
 
 /**
  * An action that exports the list of active domains on all real TLDs to Google Drive and GCS.
@@ -55,7 +64,21 @@
 public class ExportDomainListsAction implements Runnable {
 
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
-  public static final String REGISTERED_DOMAINS_FILENAME = "registered_domains.txt";
+  private static final String SELECT_DOMAINS_STATEMENT =
+      "SELECT domainName FROM Domain WHERE tld = :tld AND deletionTime > :now ORDER by domainName";
+  private static final String SELECT_DOMAINS_AND_DELETION_TIMES_STATEMENT =
+      """
+      SELECT d.domain_name, d.deletion_time, d.statuses, gp.type FROM "Domain" d
+        LEFT JOIN (SELECT type, domain_repo_id FROM "GracePeriod"
+          WHERE type = 'REDEMPTION'
+          AND expiration_time > CAST(:now AS timestamptz)) AS gp
+        ON d.repo_id = gp.domain_repo_id
+        WHERE d.tld = :tld
+        AND d.deletion_time > CAST(:now AS timestamptz)
+        ORDER BY d.domain_name""";
+
+  // This may be a CSV, but it is uses a .txt file extension for back-compatibility
+  static final String REGISTERED_DOMAINS_FILENAME = "registered_domains.txt";
 
   @Inject Clock clock;
   @Inject DriveConnection driveConnection;
@@ -68,47 +91,50 @@ public class ExportDomainListsAction implements Runnable {
   public void run() {
     ImmutableSet<String> realTlds = getTldsOfType(TldType.REAL);
     logger.atInfo().log("Exporting domain lists for TLDs %s.", realTlds);
+
+    boolean includeDeletionTimes =
+        tm().transact(
+                () ->
+                    FeatureFlag.isActiveNowOrElse(
+                        FeatureFlag.FeatureName.INCLUDE_PENDING_DELETE_DATE_FOR_DOMAINS, false));
     realTlds.forEach(
         tld -> {
-          List<String> domains =
-              tm().transact(
+          List<String> domainsList =
+              replicaTm()
+                  .transact(
                       TRANSACTION_REPEATABLE_READ,
-                      () ->
-                          // Note that if we had "creationTime <= :now" in the condition (not
-                          // necessary as there is no pending creation, the order of deletionTime
-                          // and creationTime in the query would have been significant and it
-                          // should come after deletionTime. When Hibernate substitutes "now" it
-                          // will first validate that the **first** field that is to be compared
-                          // with it (deletionTime) is assignable from the substituted Java object
-                          // (click.nowUtc()). Since creationTime is a CreateAutoTimestamp, if it
-                          // comes first, we will need to substitute "now" with
-                          // CreateAutoTimestamp.create(clock.nowUtc()). This might look a bit
-                          // strange as the Java object type is clearly incompatible between the
-                          // two fields deletionTime (DateTime) and creationTime, yet they are
-                          // compared with the same "now". It is actually OK because in the end
-                          // Hibernate converts everything to SQL types (and Java field names to
-                          // SQL column names) to run the query. Both CreateAutoTimestamp and
-                          // DateTime are persisted as timestamp_z in SQL. It is only the
-                          // validation that compares the Java types, and only with the first
-                          // field that compares with the substituted value.
-                          tm().query(
-                                  "SELECT domainName FROM Domain "
-                                      + "WHERE tld = :tld "
-                                      + "AND deletionTime > :now "
-                                      + "ORDER by domainName ASC",
-                                  String.class)
+                      () -> {
+                        if (includeDeletionTimes) {
+                          // We want to include deletion times, but only for domains in the 5-day
+                          // PENDING_DELETE period after the REDEMPTION grace period. In order to
+                          // accomplish this without loading the entire list of domains, we use a
+                          // native query to join against the GracePeriod table to find
+                          // PENDING_DELETE domains that don't have a REDEMPTION grace period.
+                          return replicaTm()
+                              .getEntityManager()
+                              .createNativeQuery(SELECT_DOMAINS_AND_DELETION_TIMES_STATEMENT)
+                              .unwrap(NativeQuery.class)
+                              .setTupleTransformer(new DomainResultTransformer())
+                              .setParameter("tld", tld)
+                              .setParameter("now", replicaTm().getTransactionTime().toString())
+                              .getResultList();
+                        } else {
+                          return replicaTm()
+                              .query(SELECT_DOMAINS_STATEMENT, String.class)
                               .setParameter("tld", tld)
-                              .setParameter("now", clock.nowUtc())
-                              .getResultList());
-          String domainsList = Joiner.on("\n").join(domains);
+                              .setParameter("now", replicaTm().getTransactionTime())
+                              .getResultList();
+                        }
+                      });
           logger.atInfo().log(
-              "Exporting %d domains for TLD %s to GCS and Drive.", domains.size(), tld);
-          exportToGcs(tld, domainsList, gcsBucket, gcsUtils);
-          exportToDrive(tld, domainsList, driveConnection);
+              "Exporting %d domains for TLD %s to GCS and Drive.", domainsList.size(), tld);
+          String domainsListOutput = Joiner.on('\n').join(domainsList);
+          exportToGcs(tld, domainsListOutput, gcsBucket, gcsUtils);
+          exportToDrive(tld, domainsListOutput, driveConnection);
         });
   }
 
-  protected static boolean exportToDrive(
+  protected static void exportToDrive(
       String tldStr, String domains, DriveConnection driveConnection) {
     verifyNotNull(driveConnection, "Expecting non-null driveConnection");
     try {
@@ -131,12 +157,10 @@ protected static boolean exportToDrive(
     } catch (Throwable e) {
       logger.atSevere().withCause(e).log(
           "Error exporting registered domains for TLD %s to Drive, skipping...", tldStr);
-      return false;
     }
-    return true;
   }
 
-  protected static boolean exportToGcs(
+  protected static void exportToGcs(
       String tld, String domains, String gcsBucket, GcsUtils gcsUtils) {
     BlobId blobId = BlobId.of(gcsBucket, tld + ".txt");
     try (OutputStream gcsOutput = gcsUtils.openOutputStream(blobId);
@@ -145,8 +169,22 @@ protected static boolean exportToGcs(
     } catch (Throwable e) {
       logger.atSevere().withCause(e).log(
           "Error exporting registered domains for TLD %s to GCS, skipping...", tld);
-      return false;
     }
-    return true;
+  }
+
+  /** Transforms the multiple columns selected from SQL into the output line. */
+  private static class DomainResultTransformer implements TupleTransformer<String> {
+    @Override
+    public String transformTuple(Object[] domainResult, String[] strings) {
+      String domainName = (String) domainResult[0];
+      Instant deletionInstant = (Instant) domainResult[1];
+      DateTime deletionTime = new DateTime(deletionInstant.toEpochMilli(), DateTimeZone.UTC);
+      String[] domainStatuses = (String[]) domainResult[2];
+      String gracePeriodType = (String) domainResult[3];
+      boolean inPendingDelete =
+          ImmutableSet.copyOf(domainStatuses).contains(StatusValue.PENDING_DELETE.toString())
+              && !GracePeriodStatus.REDEMPTION.toString().equals(gracePeriodType);
+      return String.format("%s,%s", domainName, inPendingDelete ? deletionTime : "");
+    }
   }
 }

core/src/main/java/google/registry/model/common/FeatureFlag.java

@@ -66,6 +66,7 @@ public enum FeatureName {
     TEST_FEATURE,
     MINIMUM_DATASET_CONTACTS_OPTIONAL,
     MINIMUM_DATASET_CONTACTS_PROHIBITED,
+    INCLUDE_PENDING_DELETE_DATE_FOR_DOMAINS
   }
 
   /** The name of the flag/feature. */
@@ -154,6 +155,15 @@ public FeatureStatus getStatus(DateTime time) {
     return status.getValueAtTime(time);
   }
 
+  /** Returns if the flag is active, or the default value if the flag does not exist. */
+  public static boolean isActiveNowOrElse(FeatureName featureName, boolean defaultValue) {
+    tm().assertInTransaction();
+    return CACHE
+        .get(featureName)
+        .map(flag -> flag.getStatus(tm().getTransactionTime()).equals(ACTIVE))
+        .orElse(defaultValue);
+  }
+
   /** Returns if the FeatureFlag with the given FeatureName is active now. */
   public static boolean isActiveNow(FeatureName featureName) {
     tm().assertInTransaction();

core/src/test/java/google/registry/export/ExportDomainListsActionTest.java

@@ -16,10 +16,14 @@
 
 import static com.google.common.truth.Truth.assertThat;
 import static google.registry.export.ExportDomainListsAction.REGISTERED_DOMAINS_FILENAME;
+import static google.registry.model.common.FeatureFlag.FeatureName.INCLUDE_PENDING_DELETE_DATE_FOR_DOMAINS;
+import static google.registry.model.common.FeatureFlag.FeatureStatus.ACTIVE;
+import static google.registry.model.common.FeatureFlag.FeatureStatus.INACTIVE;
 import static google.registry.testing.DatabaseHelper.createTld;
 import static google.registry.testing.DatabaseHelper.persistActiveDomain;
 import static google.registry.testing.DatabaseHelper.persistDeletedDomain;
 import static google.registry.testing.DatabaseHelper.persistResource;
+import static google.registry.util.DateTimeUtils.START_OF_TIME;
 import static java.nio.charset.StandardCharsets.UTF_8;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.mockito.ArgumentMatchers.eq;
@@ -31,8 +35,14 @@
 import com.google.cloud.storage.StorageException;
 import com.google.cloud.storage.contrib.nio.testing.LocalStorageHelper;
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSortedMap;
 import com.google.common.net.MediaType;
 import google.registry.gcs.GcsUtils;
+import google.registry.model.common.FeatureFlag;
+import google.registry.model.domain.Domain;
+import google.registry.model.domain.GracePeriod;
+import google.registry.model.domain.rgp.GracePeriodStatus;
+import google.registry.model.eppcommon.StatusValue;
 import google.registry.model.tld.Tld;
 import google.registry.model.tld.Tld.TldType;
 import google.registry.persistence.transaction.JpaTestExtensions;
@@ -56,7 +66,7 @@ class ExportDomainListsActionTest {
 
   @RegisterExtension
   final JpaIntegrationTestExtension jpa =
-      new JpaTestExtensions.Builder().buildIntegrationTestExtension();
+      new JpaTestExtensions.Builder().withClock(clock).buildIntegrationTestExtension();
 
   @BeforeEach
   void beforeEach() {
@@ -70,6 +80,7 @@ void beforeEach() {
     action.gcsUtils = gcsUtils;
     action.clock = clock;
     action.driveConnection = driveConnection;
+    persistFeatureFlag(INACTIVE);
   }
 
   private void verifyExportedToDrive(String folderId, String domains) throws Exception {
@@ -83,7 +94,7 @@ private void verifyExportedToDrive(String folderId, String domains) throws Excep
   }
 
   @Test
-  void test_outputsOnlyActiveDomains() throws Exception {
+  void test_outputsOnlyActiveDomains_txt() throws Exception {
     persistActiveDomain("onetwo.tld");
     persistActiveDomain("rudnitzky.tld");
     persistDeletedDomain("mortuary.tld", DateTime.parse("2001-03-14T10:11:12Z"));
@@ -97,7 +108,22 @@ void test_outputsOnlyActiveDomains() throws Exception {
   }
 
   @Test
-  void test_outputsOnlyDomainsOnRealTlds() throws Exception {
+  void test_outputsOnlyActiveDomains_csv() throws Exception {
+    persistFeatureFlag(ACTIVE);
+    persistActiveDomain("onetwo.tld");
+    persistActiveDomain("rudnitzky.tld");
+    persistDeletedDomain("mortuary.tld", DateTime.parse("2001-03-14T10:11:12Z"));
+    action.run();
+    BlobId existingFile = BlobId.of("outputbucket", "tld.txt");
+    String tlds = new String(gcsUtils.readBytesFrom(existingFile), UTF_8);
+    // Check that it only contains the active domains, not the dead one.
+    assertThat(tlds).isEqualTo("onetwo.tld,\nrudnitzky.tld,");
+    verifyExportedToDrive("brouhaha", "onetwo.tld,\nrudnitzky.tld,");
+    verifyNoMoreInteractions(driveConnection);
+  }
+
+  @Test
+  void test_outputsOnlyDomainsOnRealTlds_txt() throws Exception {
     persistActiveDomain("onetwo.tld");
     persistActiveDomain("rudnitzky.tld");
     persistActiveDomain("wontgo.testtld");
@@ -116,7 +142,58 @@ void test_outputsOnlyDomainsOnRealTlds() throws Exception {
   }
 
   @Test
-  void test_outputsDomainsFromDifferentTldsToMultipleFiles() throws Exception {
+  void test_outputsOnlyDomainsOnRealTlds_csv() throws Exception {
+    persistFeatureFlag(ACTIVE);
+    persistActiveDomain("onetwo.tld");
+    persistActiveDomain("rudnitzky.tld");
+    persistActiveDomain("wontgo.testtld");
+    action.run();
+    BlobId existingFile = BlobId.of("outputbucket", "tld.txt");
+    String tlds = new String(gcsUtils.readBytesFrom(existingFile), UTF_8).trim();
+    // Check that it only contains the domains on the real TLD, and not the test one.
+    assertThat(tlds).isEqualTo("onetwo.tld,\nrudnitzky.tld,");
+    // Make sure that the test TLD file wasn't written out.
+    BlobId nonexistentFile = BlobId.of("outputbucket", "testtld.txt");
+    assertThrows(StorageException.class, () -> gcsUtils.readBytesFrom(nonexistentFile));
+    ImmutableList<String> ls = gcsUtils.listFolderObjects("outputbucket", "");
+    assertThat(ls).containsExactly("tld.txt");
+    verifyExportedToDrive("brouhaha", "onetwo.tld,\nrudnitzky.tld,");
+    verifyNoMoreInteractions(driveConnection);
+  }
+
+  @Test
+  void test_outputIncludesDeletionTimes_forPendingDeletes_notRdemption() throws Exception {
+    persistFeatureFlag(ACTIVE);
+    // Domains pending delete (meaning the 5 day period, not counting the 30 day redemption period)
+    // should include their pending deletion date
+    persistActiveDomain("active.tld");
+    Domain redemption = persistActiveDomain("redemption.tld");
+    persistResource(
+        redemption
+            .asBuilder()
+            .addStatusValue(StatusValue.PENDING_DELETE)
+            .addGracePeriod(
+                GracePeriod.createWithoutBillingEvent(
+                    GracePeriodStatus.REDEMPTION,
+                    redemption.getRepoId(),
+                    clock.nowUtc().plusDays(20),
+                    redemption.getCurrentSponsorRegistrarId()))
+            .build());
+    persistResource(
+        persistActiveDomain("pendingdelete.tld")
+            .asBuilder()
+            .addStatusValue(StatusValue.PENDING_DELETE)
+            .setDeletionTime(clock.nowUtc().plusDays(3))
+            .build());
+
+    action.run();
+
+    verifyExportedToDrive(
+        "brouhaha", "active.tld,\npendingdelete.tld,2020-02-05T02:02:02.000Z\nredemption.tld,");
+  }
+
+  @Test
+  void test_outputsDomainsFromDifferentTldsToMultipleFiles_txt() throws Exception {
     createTld("tldtwo");
     persistResource(Tld.get("tldtwo").asBuilder().setDriveFolderId("hooray").build());
 
@@ -143,4 +220,43 @@ void test_outputsDomainsFromDifferentTldsToMultipleFiles() throws Exception {
     // tldthree does not have a drive id, so no export to drive is performed.
     verifyNoMoreInteractions(driveConnection);
   }
+
+  @Test
+  void test_outputsDomainsFromDifferentTldsToMultipleFiles_csv() throws Exception {
+    persistFeatureFlag(ACTIVE);
+    createTld("tldtwo");
+    persistResource(Tld.get("tldtwo").asBuilder().setDriveFolderId("hooray").build());
+
+    createTld("tldthree");
+    // You'd think this test was written around Christmas, but it wasn't.
+    persistActiveDomain("dasher.tld");
+    persistActiveDomain("prancer.tld");
+    persistActiveDomain("rudolph.tldtwo");
+    persistActiveDomain("santa.tldtwo");
+    persistActiveDomain("buddy.tldtwo");
+    persistActiveDomain("cupid.tldthree");
+    action.run();
+    BlobId firstTldFile = BlobId.of("outputbucket", "tld.txt");
+    String tlds = new String(gcsUtils.readBytesFrom(firstTldFile), UTF_8).trim();
+    assertThat(tlds).isEqualTo("dasher.tld,\nprancer.tld,");
+    BlobId secondTldFile = BlobId.of("outputbucket", "tldtwo.txt");
+    String moreTlds = new String(gcsUtils.readBytesFrom(secondTldFile), UTF_8).trim();
+    assertThat(moreTlds).isEqualTo("buddy.tldtwo,\nrudolph.tldtwo,\nsanta.tldtwo,");
+    BlobId thirdTldFile = BlobId.of("outputbucket", "tldthree.txt");
+    String evenMoreTlds = new String(gcsUtils.readBytesFrom(thirdTldFile), UTF_8).trim();
+    assertThat(evenMoreTlds).isEqualTo("cupid.tldthree,");
+    verifyExportedToDrive("brouhaha", "dasher.tld,\nprancer.tld,");
+    verifyExportedToDrive("hooray", "buddy.tldtwo,\nrudolph.tldtwo,\nsanta.tldtwo,");
+    // tldthree does not have a drive id, so no export to drive is performed.
+    verifyNoMoreInteractions(driveConnection);
+  }
+
+  private void persistFeatureFlag(FeatureFlag.FeatureStatus status) {
+    persistResource(
+        new FeatureFlag()
+            .asBuilder()
+            .setFeatureName(INCLUDE_PENDING_DELETE_DATE_FOR_DOMAINS)
+            .setStatusMap(ImmutableSortedMap.of(START_OF_TIME, status))
+            .build());
+  }
 }