CreateCrawler.java - AWS Code Sample

This version of the AWS Code Sample Catalog has been replaced by the AWS Code Library, which contains new and updated code examples.

CreateCrawler.java

CreateCrawler.java demonstrates how to create an AWS Glue crawler.

Note

To run this Java (SDK V2) code example, ensure that you have set up your development environment, including your credentials. For more information, see AWS SDK for Java 2.x Developer Guide.

/* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. SPDX-License-Identifier: Apache-2.0 */ package com.example.glue; import software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.glue.GlueClient; import software.amazon.awssdk.services.glue.model.CreateCrawlerRequest; import software.amazon.awssdk.services.glue.model.CrawlerTargets; import software.amazon.awssdk.services.glue.model.GlueException; import software.amazon.awssdk.services.glue.model.S3Target; import java.util.ArrayList; import java.util.List; /** * Before running this Java V2 code example, set up your development environment, including your credentials. * * For more information, see the following documentation topic: * * https://docs.aws.amazon.com/sdk-for-java/latest/developer-guide/get-started.html */ public class CreateCrawler { public static void main(String[] args) { final String usage = "\n" + "Usage:\n" + " <IAM> <s3Path> <cron> <dbName> <crawlerName>\n\n" + "Where:\n" + " IAM - The ARN of the IAM role that has AWS Glue and S3 permissions. \n" + " s3Path - The Amazon Simple Storage Service (Amazon S3) target that contains data (for example, CSV data).\n" + " cron - A cron expression used to specify the schedule (i.e., cron(15 12 * * ? *).\n" + " dbName - The database name. \n" + " crawlerName - The name of the crawler. \n" ; if (args.length != 5) { System.out.println(usage); System.exit(1); } String iam = args[0]; String s3Path = args[1]; String cron = args[2]; String dbName = args[3]; String crawlerName = args[4]; Region region = Region.US_EAST_1; GlueClient glueClient = GlueClient.builder() .region(region) .credentialsProvider(ProfileCredentialsProvider.create()) .build(); createGlueCrawler(glueClient, iam, s3Path, cron,dbName, crawlerName); glueClient.close(); } public static void createGlueCrawler(GlueClient glueClient, String iam, String s3Path, String cron, String dbName, String crawlerName) { try { S3Target s3Target = S3Target.builder() .path(s3Path) .build(); // Add the S3Target to a list. List<S3Target> targetList = new ArrayList<>(); targetList.add(s3Target); CrawlerTargets targets = CrawlerTargets.builder() .s3Targets(targetList) .build(); CreateCrawlerRequest crawlerRequest = CreateCrawlerRequest.builder() .databaseName(dbName) .name(crawlerName) .description("Created by the AWS Glue Java API") .targets(targets) .role(iam) .schedule(cron) .build(); glueClient.createCrawler(crawlerRequest); System.out.println(crawlerName +" was successfully created"); } catch (GlueException e) { System.err.println(e.awsErrorDetails().errorMessage()); System.exit(1); } } }