43
43
public class AdaptiveBigQueryWriter extends BigQueryWriter {
44
44
private static final Logger logger = LoggerFactory .getLogger (AdaptiveBigQueryWriter .class );
45
45
46
- // The maximum number of retries we will attempt to write rows after updating a BQ table schema.
47
- private static final int AFTER_UPDATE_RETRY_LIMIT = 5 ;
48
- // Wait for about 30s between each retry since both creating table and updating schema take up to 2 minutes to take effect.
46
+ // The maximum number of retries we will attempt to write rows after creating a table or updating a BQ table schema.
47
+ private static final int RETRY_LIMIT = 5 ;
48
+ // Wait for about 30s between each retry since both creating table and updating schema take up to 2~3 minutes to take effect.
49
49
private static final int RETRY_WAIT_TIME = 30000 ;
50
50
51
51
private final BigQuery bigQuery ;
@@ -101,8 +101,7 @@ public Map<Long, List<BigQueryError>> performWriteRequest(
101
101
try {
102
102
request = createInsertAllRequest (tableId , rows );
103
103
writeResponse = bigQuery .insertAll (request );
104
- // Should only perform one schema update attempt; may have to continue insert attempts due to
105
- // BigQuery schema updates taking up to two minutes to take effect
104
+ // Should only perform one schema update attempt.
106
105
if (writeResponse .hasErrors ()
107
106
&& onlyContainsInvalidSchemaErrors (writeResponse .getInsertErrors ()) && autoUpdateSchemas ) {
108
107
attemptSchemaUpdate (tableId , topic );
@@ -118,7 +117,8 @@ && onlyContainsInvalidSchemaErrors(writeResponse.getInsertErrors()) && autoUpdat
118
117
}
119
118
}
120
119
121
- // Schema update or table creation might be delayed, so multiple insertion attempts may be necessary
120
+ // Creating tables or updating table schemas in BigQuery takes up to 2~3 minutes to take affect,
121
+ // so multiple insertion attempts may be necessary.
122
122
int attemptCount = 0 ;
123
123
while (writeResponse == null || writeResponse .hasErrors ()) {
124
124
logger .trace ("insertion failed" );
@@ -135,10 +135,10 @@ && onlyContainsInvalidSchemaErrors(writeResponse.getInsertErrors()) && autoUpdat
135
135
return writeResponse .getInsertErrors ();
136
136
}
137
137
attemptCount ++;
138
- if (attemptCount >= AFTER_UPDATE_RETRY_LIMIT ) {
138
+ if (attemptCount >= RETRY_LIMIT ) {
139
139
throw new BigQueryConnectException (
140
140
"Failed to write rows after BQ schema update within "
141
- + AFTER_UPDATE_RETRY_LIMIT + " attempts for: " + tableId .getBaseTableId ());
141
+ + RETRY_LIMIT + " attempts for: " + tableId .getBaseTableId ());
142
142
}
143
143
try {
144
144
Thread .sleep (RETRY_WAIT_TIME );
0 commit comments