RA/loglist: Allow submissions to test logs from all_log_list.json (#8157)

Allow use of test ("type": "test") CT logs for Informational and Final
submissions.

Add new RA configuration field "SubmitToTestLogs" to optionally include
test CT logs for SCT submissions.

Co-authored-by: beautifulentropy <[email protected]>

---------

Co-authored-by: Samantha <[email protected]>

Author: mcpherrinm

cmd/boulder-ra/main.go

@@ -227,13 +227,13 @@ func main() {
 	allLogs, err := loglist.New(c.RA.CTLogs.LogListFile)
 	cmd.FailOnError(err, "Failed to parse log list")
 
-	sctLogs, err := allLogs.SubsetForPurpose(c.RA.CTLogs.SCTLogs, loglist.Issuance)
+	sctLogs, err := allLogs.SubsetForPurpose(c.RA.CTLogs.SCTLogs, loglist.Issuance, c.RA.CTLogs.SubmitToTestLogs)
 	cmd.FailOnError(err, "Failed to load SCT logs")
 
-	infoLogs, err := allLogs.SubsetForPurpose(c.RA.CTLogs.InfoLogs, loglist.Informational)
+	infoLogs, err := allLogs.SubsetForPurpose(c.RA.CTLogs.InfoLogs, loglist.Informational, true)
 	cmd.FailOnError(err, "Failed to load informational logs")
 
-	finalLogs, err := allLogs.SubsetForPurpose(c.RA.CTLogs.FinalLogs, loglist.Informational)
+	finalLogs, err := allLogs.SubsetForPurpose(c.RA.CTLogs.FinalLogs, loglist.Informational, true)
 	cmd.FailOnError(err, "Failed to load final logs")
 
 	ctp = ctpolicy.New(pubc, sctLogs, infoLogs, finalLogs, c.RA.CTLogs.Stagger.Duration, logger, scope)

ctpolicy/ctconfig/ctconfig.go

@@ -24,4 +24,7 @@ type CTConfig struct {
 	// This may include duplicates from the lists above, to submit both precerts
 	// and final certs to the same log.
 	FinalLogs []string
+	// SubmitToTestLogs enables inclusion of "test" logs when obtaining SCTs.
+	// This should only be used in test environments.
+	SubmitToTestLogs bool
 }

ctpolicy/loglist/lintlist.go

@@ -22,7 +22,7 @@ func InitLintList(path string) error {
 			return
 		}
 
-		l, err = l.forPurpose(Validation)
+		l, err = l.forPurpose(Validation, false)
 		if err != nil {
 			lintlist.err = err
 			return

ctpolicy/loglist/loglist.go

@@ -48,6 +48,7 @@ type Log struct {
 	EndExclusive   time.Time
 	State          loglist3.LogStatus
 	Tiled          bool
+	Type           string
 }
 
 // usableForPurpose returns true if the log state is acceptable for the given
@@ -95,6 +96,7 @@ func newHelper(file []byte) (List, error) {
 				Url:      log.URL,
 				State:    log.State.LogStatus(),
 				Tiled:    false,
+				Type:     log.Type,
 			}
 
 			if log.TemporalInterval != nil {
@@ -114,6 +116,7 @@ func newHelper(file []byte) (List, error) {
 				Url:      log.SubmissionURL,
 				State:    log.State.LogStatus(),
 				Tiled:    true,
+				Type:     log.Type,
 			}
 
 			if log.TemporalInterval != nil {
@@ -133,13 +136,13 @@ func newHelper(file []byte) (List, error) {
 // given purpose. It returns an error if any of the given names are not found
 // in the starting list, or if the resulting list is too small to satisfy the
 // Chrome "two operators" policy.
-func (ll List) SubsetForPurpose(names []string, p purpose) (List, error) {
+func (ll List) SubsetForPurpose(names []string, p purpose, submitToTestLogs bool) (List, error) {
 	sub, err := ll.subset(names)
 	if err != nil {
 		return nil, err
 	}
 
-	res, err := sub.forPurpose(p)
+	res, err := sub.forPurpose(p, submitToTestLogs)
 	if err != nil {
 		return nil, err
 	}
@@ -171,17 +174,24 @@ func (ll List) subset(names []string) (List, error) {
 }
 
 // forPurpose returns a new log list containing only those logs whose states are
-// acceptable for the given purpose. It returns an error if the purpose is
-// Issuance or Validation and the set of remaining logs is too small to satisfy
-// the Google "two operators" log policy.
-func (ll List) forPurpose(p purpose) (List, error) {
+// acceptable for the given purpose. Test logs are included only when
+// submitToTestLogs is true. It returns an error if the purpose is Issuance or
+// Validation and the set of remaining logs is too small to satisfy the Google
+// "two operators" log policy.
+func (ll List) forPurpose(p purpose, submitToTestLogs bool) (List, error) {
 	res := make(List, 0)
 	operators := make(map[string]struct{})
+
+	// Test logs in Chrome's all_logs_list.json omit the "state" field. loglist3
+	// interprets this as "UndefinedLogStatus", which causes usableForPurpose()
+	// to return false. To account for this, we skip this check for test logs.
 	for _, log := range ll {
-		if !usableForPurpose(log.State, p) {
+		if log.Type == "test" && !submitToTestLogs {
+			continue
+		}
+		if log.Type != "test" && !usableForPurpose(log.State, p) {
 			continue
 		}
-
 		res = append(res, log)
 		operators[log.Operator] = struct{}{}
 	}

ctpolicy/loglist/loglist_test.go

@@ -59,7 +59,7 @@ func TestForPurpose(t *testing.T) {
 		Log{Name: "Log A1", Operator: "A", State: loglist3.UsableLogStatus},
 		Log{Name: "Log B1", Operator: "B", State: loglist3.UsableLogStatus},
 	}
-	actual, err := input.forPurpose(Issuance)
+	actual, err := input.forPurpose(Issuance, false)
 	test.AssertNotError(t, err, "should have two acceptable logs")
 	test.AssertDeepEquals(t, actual, expected)
 
@@ -71,14 +71,14 @@ func TestForPurpose(t *testing.T) {
 		Log{Name: "Log C1", Operator: "C", State: loglist3.PendingLogStatus},
 		Log{Name: "Log C2", Operator: "C", State: loglist3.ReadOnlyLogStatus},
 	}
-	_, err = input.forPurpose(Issuance)
+	_, err = input.forPurpose(Issuance, false)
 	test.AssertError(t, err, "should only have one acceptable log")
 
 	expected = List{
 		Log{Name: "Log A1", Operator: "A", State: loglist3.UsableLogStatus},
 		Log{Name: "Log C2", Operator: "C", State: loglist3.ReadOnlyLogStatus},
 	}
-	actual, err = input.forPurpose(Validation)
+	actual, err = input.forPurpose(Validation, false)
 	test.AssertNotError(t, err, "should have two acceptable logs")
 	test.AssertDeepEquals(t, actual, expected)
 
@@ -87,9 +87,31 @@ func TestForPurpose(t *testing.T) {
 		Log{Name: "Log B1", Operator: "B", State: loglist3.QualifiedLogStatus},
 		Log{Name: "Log C1", Operator: "C", State: loglist3.PendingLogStatus},
 	}
-	actual, err = input.forPurpose(Informational)
+	actual, err = input.forPurpose(Informational, false)
 	test.AssertNotError(t, err, "should have three acceptable logs")
 	test.AssertDeepEquals(t, actual, expected)
+
+	input = List{
+		Log{Name: "Log A1", Operator: "A", State: loglist3.UsableLogStatus},
+		Log{Name: "Log B1", Operator: "B", State: loglist3.UsableLogStatus},
+		Log{Name: "Log T1", Operator: "T", Type: "test", State: loglist3.UndefinedLogStatus},
+	}
+	expected = List{
+		Log{Name: "Log A1", Operator: "A", State: loglist3.UsableLogStatus},
+		Log{Name: "Log B1", Operator: "B", State: loglist3.UsableLogStatus},
+	}
+	actual, err = input.forPurpose(Issuance, false)
+	test.AssertNotError(t, err, "should have two acceptable logs with submitToTestLogs=[false]")
+	test.AssertDeepEquals(t, actual, expected)
+
+	expected = List{
+		Log{Name: "Log A1", Operator: "A", State: loglist3.UsableLogStatus},
+		Log{Name: "Log B1", Operator: "B", State: loglist3.UsableLogStatus},
+		Log{Name: "Log T1", Operator: "T", Type: "test", State: loglist3.UndefinedLogStatus},
+	}
+	actual, err = input.forPurpose(Issuance, true)
+	test.AssertNotError(t, err, "should have two acceptable logs with submitToTestLogs=[true]")
+	test.AssertDeepEquals(t, actual, expected)
 }
 
 func TestForTime(t *testing.T) {