Skip to content

Commit bd24fd8

Browse files
whuffman36whuffman36
authored
feat: Enable maxTimeTravelHours in BigQuery java client library (#3555)
* Enable maxTimeTravelHours in BigQuery java client library The BigQuery API contains a field called maxTimeTravelHours that refers to the Time Travel Window feature. This feature allows users to query deleted or updated data for an amount of time specified in the maxTimeTravelHours parameter. This change exposes the maxTimeTravelHours to users of the client library so they can set and update the amount of time that deleted or updated data is stored for a given dataset. * Update maxTimeTravelHours description to match discovery doc. Fix formatting errors. * Add DatasetInfo.Builder.setMaxTimeTravelHours() to clirr-ignored file.
1 parent 7a03899 commit bd24fd8

File tree

6 files changed

+109
-0
lines changed

6 files changed

+109
-0
lines changed

‎google-cloud-bigquery/clirr-ignored-differences.xml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -129,4 +129,9 @@
129129
<className>com/google/cloud/bigquery/StandardTableDefinition*</className>
130130
<method>*BigLakeConfiguration(*)</method>
131131
</difference>
132+
<difference>
133+
<differenceType>7013</differenceType>
134+
<className>com/google/cloud/bigquery/DatasetInfo*</className>
135+
<method>*setMaxTimeTravelHours(*)</method>
136+
</difference>
132137
</differences>

‎google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Dataset.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -164,6 +164,12 @@ public Builder setStorageBillingModel(String storageBillingModel) {
164164
return this;
165165
}
166166

167+
@Override
168+
public Builder setMaxTimeTravelHours(Long maxTimeTravelHours) {
169+
infoBuilder.setMaxTimeTravelHours(maxTimeTravelHours);
170+
return this;
171+
}
172+
167173
@Override
168174
public Dataset build() {
169175
return new Dataset(bigquery, infoBuilder);

‎google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/DatasetInfo.java‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,7 @@ public Dataset apply(DatasetInfo datasetInfo) {
7575
private final String defaultCollation;
7676
private final ExternalDatasetReference externalDatasetReference;
7777
private final String storageBillingModel;
78+
private final Long maxTimeTravelHours;
7879

7980
/** A builder for {@code DatasetInfo} objects. */
8081
public abstract static class Builder {
@@ -142,6 +143,12 @@ public abstract Builder setExternalDatasetReference(
142143
*/
143144
public abstract Builder setStorageBillingModel(String storageBillingModel);
144145

146+
/**
147+
* Optional. Defines the time travel window in hours. The value can be from 48 to 168 hours (2
148+
* to 7 days). The default value is 168 hours if this is not set. The value may be {@code null}.
149+
*/
150+
public abstract Builder setMaxTimeTravelHours(Long maxTimeTravelHours);
151+
145152
/**
146153
* The default encryption key for all tables in the dataset. Once this property is set, all
147154
* newly-created partitioned tables in the dataset will have encryption key set to this value,
@@ -200,6 +207,7 @@ static final class BuilderImpl extends Builder {
200207
private String defaultCollation;
201208
private ExternalDatasetReference externalDatasetReference;
202209
private String storageBillingModel;
210+
private Long maxTimeTravelHours;
203211

204212
BuilderImpl() {}
205213

@@ -221,6 +229,7 @@ static final class BuilderImpl extends Builder {
221229
this.defaultCollation = datasetInfo.defaultCollation;
222230
this.externalDatasetReference = datasetInfo.externalDatasetReference;
223231
this.storageBillingModel = datasetInfo.storageBillingModel;
232+
this.maxTimeTravelHours = datasetInfo.maxTimeTravelHours;
224233
}
225234

226235
BuilderImpl(com.google.api.services.bigquery.model.Dataset datasetPb) {
@@ -260,6 +269,7 @@ public Acl apply(Dataset.Access accessPb) {
260269
ExternalDatasetReference.fromPb(datasetPb.getExternalDatasetReference());
261270
}
262271
this.storageBillingModel = datasetPb.getStorageBillingModel();
272+
this.maxTimeTravelHours = datasetPb.getMaxTimeTravelHours();
263273
}
264274

265275
@Override
@@ -372,6 +382,12 @@ public Builder setStorageBillingModel(String storageBillingModel) {
372382
return this;
373383
}
374384

385+
@Override
386+
public Builder setMaxTimeTravelHours(Long maxTimeTravelHours) {
387+
this.maxTimeTravelHours = maxTimeTravelHours;
388+
return this;
389+
}
390+
375391
@Override
376392
public DatasetInfo build() {
377393
return new DatasetInfo(this);
@@ -396,6 +412,7 @@ public DatasetInfo build() {
396412
defaultCollation = builder.defaultCollation;
397413
externalDatasetReference = builder.externalDatasetReference;
398414
storageBillingModel = builder.storageBillingModel;
415+
maxTimeTravelHours = builder.maxTimeTravelHours;
399416
}
400417

401418
/** Returns the dataset identity. */
@@ -529,6 +546,14 @@ public String getStorageBillingModel() {
529546
return storageBillingModel;
530547
}
531548

549+
/**
550+
* Returns the number of hours that deleted or updated data will be available to be queried for
551+
* all tables in the dataset.
552+
*/
553+
public Long getMaxTimeTravelHours() {
554+
return maxTimeTravelHours;
555+
}
556+
532557
/**
533558
* Returns information about the external metadata storage where the dataset is defined. Filled
534559
* out when the dataset type is EXTERNAL.
@@ -562,6 +587,7 @@ public String toString() {
562587
.add("defaultCollation", defaultCollation)
563588
.add("externalDatasetReference", externalDatasetReference)
564589
.add("storageBillingModel", storageBillingModel)
590+
.add("maxTimeTravelHours", maxTimeTravelHours)
565591
.toString();
566592
}
567593

@@ -646,6 +672,9 @@ public Dataset.Access apply(Acl acl) {
646672
if (storageBillingModel != null) {
647673
datasetPb.setStorageBillingModel(storageBillingModel);
648674
}
675+
if (maxTimeTravelHours != null) {
676+
datasetPb.setMaxTimeTravelHours(maxTimeTravelHours);
677+
}
649678
return datasetPb;
650679
}
651680

‎google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/DatasetInfoTest.java‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
package com.google.cloud.bigquery;
1818

1919
import static org.junit.Assert.assertEquals;
20+
import static org.junit.Assert.assertNotEquals;
2021
import static org.junit.Assert.assertNull;
2122
import static org.junit.Assert.assertTrue;
2223

@@ -59,6 +60,8 @@ public class DatasetInfoTest {
5960
private static final EncryptionConfiguration DATASET_ENCRYPTION_CONFIGURATION =
6061
EncryptionConfiguration.newBuilder().setKmsKeyName("KMS_KEY_1").build();
6162
private static final String STORAGE_BILLING_MODEL = "LOGICAL";
63+
private static final Long MAX_TIME_TRAVEL_HOURS_5_DAYS = 120L;
64+
private static final Long MAX_TIME_TRAVEL_HOURS_7_DAYS = 168L;
6265

6366
private static final ExternalDatasetReference EXTERNAL_DATASET_REFERENCE =
6467
ExternalDatasetReference.newBuilder()
@@ -81,6 +84,7 @@ public class DatasetInfoTest {
8184
.setDefaultEncryptionConfiguration(DATASET_ENCRYPTION_CONFIGURATION)
8285
.setDefaultPartitionExpirationMs(DEFAULT_PARTITION__EXPIRATION)
8386
.setStorageBillingModel(STORAGE_BILLING_MODEL)
87+
.setMaxTimeTravelHours(MAX_TIME_TRAVEL_HOURS_7_DAYS)
8488
.build();
8589
private static final DatasetInfo DATASET_INFO_COMPLETE =
8690
DATASET_INFO
@@ -92,6 +96,8 @@ public class DatasetInfoTest {
9296
DATASET_INFO.toBuilder().setAcl(ACCESS_RULES_IAM_MEMBER).build();
9397
private static final DatasetInfo DATASET_INFO_COMPLETE_WITH_EXTERNAL_DATASET_REFERENCE =
9498
DATASET_INFO.toBuilder().setExternalDatasetReference(EXTERNAL_DATASET_REFERENCE).build();
99+
private static final DatasetInfo DATASET_INFO_WITH_MAX_TIME_TRAVEL_5_DAYS =
100+
DATASET_INFO.toBuilder().setMaxTimeTravelHours(MAX_TIME_TRAVEL_HOURS_5_DAYS).build();
95101

96102
@Test
97103
public void testToBuilder() {
@@ -173,6 +179,10 @@ public void testBuilder() {
173179
EXTERNAL_DATASET_REFERENCE,
174180
DATASET_INFO_COMPLETE_WITH_EXTERNAL_DATASET_REFERENCE.getExternalDatasetReference());
175181
assertEquals(STORAGE_BILLING_MODEL, DATASET_INFO_COMPLETE.getStorageBillingModel());
182+
assertEquals(MAX_TIME_TRAVEL_HOURS_7_DAYS, DATASET_INFO.getMaxTimeTravelHours());
183+
assertEquals(
184+
MAX_TIME_TRAVEL_HOURS_5_DAYS,
185+
DATASET_INFO_WITH_MAX_TIME_TRAVEL_5_DAYS.getMaxTimeTravelHours());
176186
}
177187

178188
@Test
@@ -194,6 +204,7 @@ public void testOf() {
194204
assertTrue(datasetInfo.getLabels().isEmpty());
195205
assertNull(datasetInfo.getExternalDatasetReference());
196206
assertNull(datasetInfo.getStorageBillingModel());
207+
assertNull(datasetInfo.getMaxTimeTravelHours());
197208

198209
datasetInfo = DatasetInfo.of(DATASET_ID);
199210
assertEquals(DATASET_ID, datasetInfo.getDatasetId());
@@ -212,6 +223,7 @@ public void testOf() {
212223
assertTrue(datasetInfo.getLabels().isEmpty());
213224
assertNull(datasetInfo.getExternalDatasetReference());
214225
assertNull(datasetInfo.getStorageBillingModel());
226+
assertNull(datasetInfo.getMaxTimeTravelHours());
215227
}
216228

217229
@Test
@@ -229,6 +241,16 @@ public void testSetProjectId() {
229241
assertEquals(DATASET_INFO_COMPLETE, DATASET_INFO.setProjectId("project"));
230242
}
231243

244+
@Test
245+
public void testSetMaxTimeTravelHours() {
246+
assertNotEquals(
247+
DATASET_INFO_WITH_MAX_TIME_TRAVEL_5_DAYS.getMaxTimeTravelHours(),
248+
DATASET_INFO.getMaxTimeTravelHours());
249+
assertEquals(
250+
DATASET_INFO_WITH_MAX_TIME_TRAVEL_5_DAYS,
251+
DATASET_INFO.toBuilder().setMaxTimeTravelHours(MAX_TIME_TRAVEL_HOURS_5_DAYS).build());
252+
}
253+
232254
private void compareDatasets(DatasetInfo expected, DatasetInfo value) {
233255
assertEquals(expected, value);
234256
assertEquals(expected.getDatasetId(), value.getDatasetId());
@@ -249,5 +271,6 @@ private void compareDatasets(DatasetInfo expected, DatasetInfo value) {
249271
expected.getDefaultPartitionExpirationMs(), value.getDefaultPartitionExpirationMs());
250272
assertEquals(expected.getExternalDatasetReference(), value.getExternalDatasetReference());
251273
assertEquals(expected.getStorageBillingModel(), value.getStorageBillingModel());
274+
assertEquals(expected.getMaxTimeTravelHours(), value.getMaxTimeTravelHours());
252275
}
253276
}

‎google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/DatasetTest.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,7 @@ public class DatasetTest {
6767
private static final DatasetInfo DATASET_INFO = DatasetInfo.newBuilder(DATASET_ID).build();
6868
private static final Field FIELD = Field.of("FieldName", LegacySQLTypeName.INTEGER);
6969
private static final String STORAGE_BILLING_MODEL = "LOGICAL";
70+
private static final Long MAX_TIME_TRAVEL_HOURS = 168L;
7071
private static final StandardTableDefinition TABLE_DEFINITION =
7172
StandardTableDefinition.of(Schema.of(FIELD));
7273
private static final ViewDefinition VIEW_DEFINITION = ViewDefinition.of("QUERY");
@@ -122,6 +123,7 @@ public void testBuilder() {
122123
.setSelfLink(SELF_LINK)
123124
.setLabels(LABELS)
124125
.setStorageBillingModel(STORAGE_BILLING_MODEL)
126+
.setMaxTimeTravelHours(MAX_TIME_TRAVEL_HOURS)
125127
.build();
126128
assertEquals(DATASET_ID, builtDataset.getDatasetId());
127129
assertEquals(ACCESS_RULES, builtDataset.getAcl());
@@ -136,6 +138,7 @@ public void testBuilder() {
136138
assertEquals(SELF_LINK, builtDataset.getSelfLink());
137139
assertEquals(LABELS, builtDataset.getLabels());
138140
assertEquals(STORAGE_BILLING_MODEL, builtDataset.getStorageBillingModel());
141+
assertEquals(MAX_TIME_TRAVEL_HOURS, builtDataset.getMaxTimeTravelHours());
139142
}
140143

141144
@Test
@@ -344,6 +347,7 @@ public void testExternalDatasetReference() {
344347
.setLabels(LABELS)
345348
.setExternalDatasetReference(EXTERNAL_DATASET_REFERENCE)
346349
.setStorageBillingModel(STORAGE_BILLING_MODEL)
350+
.setMaxTimeTravelHours(MAX_TIME_TRAVEL_HOURS)
347351
.build();
348352
assertEquals(
349353
EXTERNAL_DATASET_REFERENCE,
@@ -374,5 +378,6 @@ private void compareDatasetInfo(DatasetInfo expected, DatasetInfo value) {
374378
assertEquals(expected.getLastModified(), value.getLastModified());
375379
assertEquals(expected.getExternalDatasetReference(), value.getExternalDatasetReference());
376380
assertEquals(expected.getStorageBillingModel(), value.getStorageBillingModel());
381+
assertEquals(expected.getMaxTimeTravelHours(), value.getMaxTimeTravelHours());
377382
}
378383
}

‎google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -213,6 +213,8 @@ public class ITBigQueryTest {
213213
private static final String PROJECT_ID = ServiceOptions.getDefaultProjectId();
214214
private static final String RANDOM_ID = UUID.randomUUID().toString().substring(0, 8);
215215
private static final String STORAGE_BILLING_MODEL = "LOGICAL";
216+
private static final Long MAX_TIME_TRAVEL_HOURS = 120L;
217+
private static final Long MAX_TIME_TRAVEL_HOURS_DEFAULT = 168L;
216218
private static final String CLOUD_SAMPLES_DATA =
217219
Optional.fromNullable(System.getenv("CLOUD_SAMPLES_DATA_BUCKET")).or("cloud-samples-data");
218220
private static final Map<String, String> LABELS =
@@ -1214,6 +1216,7 @@ public void testGetDatasetWithSelectedFields() {
12141216
assertNull(dataset.getLocation());
12151217
assertNull(dataset.getSelfLink());
12161218
assertNull(dataset.getStorageBillingModel());
1219+
assertNull(dataset.getMaxTimeTravelHours());
12171220
}
12181221

12191222
@Test
@@ -1230,6 +1233,7 @@ public void testUpdateDataset() {
12301233
assertThat(dataset.getDescription()).isEqualTo("Some Description");
12311234
assertThat(dataset.getLabels()).containsExactly("a", "b");
12321235
assertThat(dataset.getStorageBillingModel()).isNull();
1236+
assertThat(dataset.getMaxTimeTravelHours()).isNull();
12331237

12341238
Map<String, String> updateLabels = new HashMap<>();
12351239
updateLabels.put("x", "y");
@@ -1241,10 +1245,12 @@ public void testUpdateDataset() {
12411245
.setDescription("Updated Description")
12421246
.setLabels(updateLabels)
12431247
.setStorageBillingModel("LOGICAL")
1248+
.setMaxTimeTravelHours(MAX_TIME_TRAVEL_HOURS)
12441249
.build());
12451250
assertThat(updatedDataset.getDescription()).isEqualTo("Updated Description");
12461251
assertThat(updatedDataset.getLabels()).containsExactly("x", "y");
12471252
assertThat(updatedDataset.getStorageBillingModel()).isEqualTo("LOGICAL");
1253+
assertThat(updatedDataset.getMaxTimeTravelHours()).isEqualTo(MAX_TIME_TRAVEL_HOURS);
12481254

12491255
updatedDataset = bigquery.update(updatedDataset.toBuilder().setLabels(null).build());
12501256
assertThat(updatedDataset.getLabels()).isEmpty();
@@ -1275,6 +1281,7 @@ public void testUpdateDatasetWithSelectedFields() {
12751281
assertNull(updatedDataset.getLocation());
12761282
assertNull(updatedDataset.getSelfLink());
12771283
assertNull(updatedDataset.getStorageBillingModel());
1284+
assertNull(updatedDataset.getMaxTimeTravelHours());
12781285
assertTrue(dataset.delete());
12791286
}
12801287

@@ -1630,6 +1637,40 @@ public void testCreateDatasetWithSpecifiedStorageBillingModel() {
16301637
RemoteBigQueryHelper.forceDelete(bigquery, billingModelDataset);
16311638
}
16321639

1640+
@Test
1641+
public void testCreateDatasetWithSpecificMaxTimeTravelHours() {
1642+
String timeTravelDataset = RemoteBigQueryHelper.generateDatasetName();
1643+
DatasetInfo info =
1644+
DatasetInfo.newBuilder(timeTravelDataset)
1645+
.setDescription(DESCRIPTION)
1646+
.setMaxTimeTravelHours(MAX_TIME_TRAVEL_HOURS)
1647+
.setLabels(LABELS)
1648+
.build();
1649+
bigquery.create(info);
1650+
1651+
Dataset dataset = bigquery.getDataset(DatasetId.of(timeTravelDataset));
1652+
assertEquals(MAX_TIME_TRAVEL_HOURS, dataset.getMaxTimeTravelHours());
1653+
1654+
RemoteBigQueryHelper.forceDelete(bigquery, timeTravelDataset);
1655+
}
1656+
1657+
@Test
1658+
public void testCreateDatasetWithDefaultMaxTimeTravelHours() {
1659+
String timeTravelDataset = RemoteBigQueryHelper.generateDatasetName();
1660+
DatasetInfo info =
1661+
DatasetInfo.newBuilder(timeTravelDataset)
1662+
.setDescription(DESCRIPTION)
1663+
.setLabels(LABELS)
1664+
.build();
1665+
bigquery.create(info);
1666+
1667+
Dataset dataset = bigquery.getDataset(DatasetId.of(timeTravelDataset));
1668+
// In the backend, BigQuery sets the default Time Travel Window to be 168 hours (7 days).
1669+
assertEquals(MAX_TIME_TRAVEL_HOURS_DEFAULT, dataset.getMaxTimeTravelHours());
1670+
1671+
RemoteBigQueryHelper.forceDelete(bigquery, timeTravelDataset);
1672+
}
1673+
16331674
@Test
16341675
public void testCreateDatasetWithDefaultCollation() {
16351676
String collationDataset = RemoteBigQueryHelper.generateDatasetName();

0 commit comments

Comments
 (0)