Amazon Simple Storage Service
Developer Guide (API Version 2006-03-01)

Selecting Content from Objects Using the SDK for Java

You use Amazon S3 Select to select contents of an object with Java using the selectObjectContent method, which on success returns the results of the SQL expression. The specified bucket and object key must exist, or an error results.


The following Java code returns the value of the first column for each record that is stored in an object that contains data stored in CSV format. It also requests Progress and Stats messages to be returned. You must provide a valid bucket name and an object that contains data in CSV format.

For instructions on creating and testing a working sample, see Testing the Amazon S3 Java Code Examples.

package com.amazonaws; import; import; import; import; import; import; import; import; import; import; import; import; import; import; import; import; import java.util.concurrent.atomic.AtomicBoolean; import static com.amazonaws.util.IOUtils.copy; /** * This example shows how to query data from S3Select and consume the response in the form of an * InputStream of records and write it to a file. */ public class RecordInputStreamExample { private static final String BUCKET_NAME = "${my-s3-bucket}"; private static final String CSV_OBJECT_KEY = "${my-csv-object-key}"; private static final String S3_SELECT_RESULTS_PATH = "${my-s3-select-results-path}"; private static final String QUERY = "select s._1 from S3Object s"; public static void main(String[] args) throws Exception { final AmazonS3 s3Client = AmazonS3ClientBuilder.defaultClient(); SelectObjectContentRequest request = generateBaseCSVRequest(BUCKET_NAME, CSV_OBJECT_KEY, QUERY); final AtomicBoolean isResultComplete = new AtomicBoolean(false); try (OutputStream fileOutputStream = new FileOutputStream(new File (S3_SELECT_RESULTS_PATH)); SelectObjectContentResult result = s3Client.selectObjectContent(request)) { InputStream resultInputStream = result.getPayload().getRecordsInputStream( new SelectObjectContentEventVisitor() { @Override public void visit(SelectObjectContentEvent.StatsEvent event) { System.out.println( "Received Stats, Bytes Scanned: " + event.getDetails().getBytesScanned() + " Bytes Processed: " + event.getDetails().getBytesProcessed()); } /* * An End Event informs that the request has finished successfully. */ @Override public void visit(SelectObjectContentEvent.EndEvent event) { isResultComplete.set(true); System.out.println("Received End Event. Result is complete."); } } ); copy(resultInputStream, fileOutputStream); } /* * The End Event indicates all matching records have been transmitted. * If the End Event is not received, the results may be incomplete. */ if (!isResultComplete.get()) { throw new Exception("S3 Select request was incomplete as End Event was not received."); } } private static SelectObjectContentRequest generateBaseCSVRequest(String bucket, String key, String query) { SelectObjectContentRequest request = new SelectObjectContentRequest(); request.setBucketName(bucket); request.setKey(key); request.setExpression(query); request.setExpressionType(ExpressionType.SQL); InputSerialization inputSerialization = new InputSerialization(); inputSerialization.setCsv(new CSVInput()); inputSerialization.setCompressionType(CompressionType.NONE); request.setInputSerialization(inputSerialization); OutputSerialization outputSerialization = new OutputSerialization(); outputSerialization.setCsv(new CSVOutput()); request.setOutputSerialization(outputSerialization); return request; } }