Append buffered records

Use the JSON stream writer to append records in buffered mode

Code sample

Java

Before trying this sample, follow theJava setup instructions in theBigQuery quickstart using client libraries. For more information, see theBigQueryJava API reference documentation.

To authenticate to BigQuery, set up Application Default Credentials. For more information, seeSet up authentication for client libraries.

importcom.google.api.core.ApiFuture;importcom.google.api.gax.retrying.RetrySettings;importcom.google.cloud.bigquery.storage.v1.AppendRowsResponse;importcom.google.cloud.bigquery.storage.v1.BigQueryWriteClient;importcom.google.cloud.bigquery.storage.v1.CreateWriteStreamRequest;importcom.google.cloud.bigquery.storage.v1.FinalizeWriteStreamRequest;importcom.google.cloud.bigquery.storage.v1.FlushRowsRequest;importcom.google.cloud.bigquery.storage.v1.FlushRowsResponse;importcom.google.cloud.bigquery.storage.v1.JsonStreamWriter;importcom.google.cloud.bigquery.storage.v1.TableName;importcom.google.cloud.bigquery.storage.v1.WriteStream;importcom.google.protobuf.Descriptors.DescriptorValidationException;importcom.google.protobuf.Int64Value;importjava.io.IOException;importjava.util.concurrent.ExecutionException;importorg.json.JSONArray;importorg.json.JSONObject;importorg.threeten.bp.Duration;publicclassWriteBufferedStream{publicstaticvoidrunWriteBufferedStream()throwsDescriptorValidationException,InterruptedException,IOException{// TODO(developer): Replace these variables before running the sample.StringprojectId="MY_PROJECT_ID";StringdatasetName="MY_DATASET_NAME";StringtableName="MY_TABLE_NAME";writeBufferedStream(projectId,datasetName,tableName);}publicstaticvoidwriteBufferedStream(StringprojectId,StringdatasetName,StringtableName)throwsDescriptorValidationException,InterruptedException,IOException{try(BigQueryWriteClientclient=BigQueryWriteClient.create()){// Initialize a write stream for the specified table.// For more information on WriteStream.Type, see:// https://googleapis.dev/java/google-cloud-bigquerystorage/latest/com/google/cloud/bigquery/storage/v1/WriteStream.Type.htmlWriteStreamstream=WriteStream.newBuilder().setType(WriteStream.Type.BUFFERED).build();TableNameparentTable=TableName.of(projectId,datasetName,tableName);CreateWriteStreamRequestcreateWriteStreamRequest=CreateWriteStreamRequest.newBuilder().setParent(parentTable.toString()).setWriteStream(stream).build();WriteStreamwriteStream=client.createWriteStream(createWriteStreamRequest);// Configure in-stream automatic retry settings.// Error codes that are immediately retried:// * ABORTED, UNAVAILABLE, CANCELLED, INTERNAL, DEADLINE_EXCEEDED// Error codes that are retried with exponential backoff:// * RESOURCE_EXHAUSTEDRetrySettingsretrySettings=RetrySettings.newBuilder().setInitialRetryDelay(Duration.ofMillis(500)).setRetryDelayMultiplier(1.1).setMaxAttempts(5).setMaxRetryDelay(Duration.ofMinutes(1)).build();// Use the JSON stream writer to send records in JSON format.// For more information about JsonStreamWriter, see:// https://cloud.google.com/java/docs/reference/google-cloud-bigquerystorage/latest/com.google.cloud.bigquery.storage.v1.JsonStreamWritertry(JsonStreamWriterwriter=JsonStreamWriter.newBuilder(writeStream.getName(),writeStream.getTableSchema()).setRetrySettings(retrySettings).build()){// Write two batches to the stream, each with 10 JSON records.for(inti=0;i <2;i++){JSONArrayjsonArr=newJSONArray();for(intj=0;j <10;j++){// Create a JSON object that is compatible with the table schema.JSONObjectrecord=newJSONObject();record.put("col1",String.format("buffered-record %03d",i));jsonArr.put(record);}ApiFuture<AppendRowsResponse>future=writer.append(jsonArr);AppendRowsResponseresponse=future.get();}// Flush the buffer.FlushRowsRequestflushRowsRequest=FlushRowsRequest.newBuilder().setWriteStream(writeStream.getName()).setOffset(Int64Value.of(10*2-1))// Advance the cursor to the latest record..build();FlushRowsResponseflushRowsResponse=client.flushRows(flushRowsRequest);// You can continue to write to the stream after flushing the buffer.}// Finalize the stream after use.FinalizeWriteStreamRequestfinalizeWriteStreamRequest=FinalizeWriteStreamRequest.newBuilder().setName(writeStream.getName()).build();client.finalizeWriteStream(finalizeWriteStreamRequest);System.out.println("Appended and committed records successfully.");}catch(ExecutionExceptione){// If the wrapped exception is a StatusRuntimeException, check the state of the operation.// If the state is INTERNAL, CANCELLED, or ABORTED, you can retry. For more information, see:// https://grpc.github.io/grpc-java/javadoc/io/grpc/StatusRuntimeException.htmlSystem.out.println(e);}}}

What's next

To search and filter code samples for other Google Cloud products, see theGoogle Cloud sample browser.

Except as otherwise noted, the content of this page is licensed under theCreative Commons Attribution 4.0 License, and code samples are licensed under theApache 2.0 License. For details, see theGoogle Developers Site Policies. Java is a registered trademark of Oracle and/or its affiliates.