diff --git a/.doc_gen/metadata/glue_metadata.yaml b/.doc_gen/metadata/glue_metadata.yaml index ed2c3ca45c9..9a87b2d6474 100644 --- a/.doc_gen/metadata/glue_metadata.yaml +++ b/.doc_gen/metadata/glue_metadata.yaml @@ -498,6 +498,15 @@ glue_GetTables: glue: {GetTables} glue_CreateJob: languages: + Java: + versions: + - sdk_version: 2 + github: javav2/example_code/glue + sdkguide: + excerpts: + - description: + snippet_tags: + - glue.java2.create_job.main .NET: versions: - sdk_version: 3 @@ -568,6 +577,15 @@ glue_CreateJob: glue: {CreateJob} glue_StartJobRun: languages: + Java: + versions: + - sdk_version: 2 + github: javav2/example_code/glue + sdkguide: + excerpts: + - description: + snippet_tags: + - glue.java2.start.job.main .NET: versions: - sdk_version: 3 @@ -715,6 +733,15 @@ glue_GetJobRuns: - description: snippet_tags: - Glue.dotnetv3.GetJobRuns + Java: + versions: + - sdk_version: 2 + github: javav2/example_code/glue + sdkguide: + excerpts: + - description: + snippet_tags: + - glue.java2.get_job.main PHP: versions: - sdk_version: 3 @@ -836,6 +863,15 @@ glue_GetJobRun: glue: {GetJobRun} glue_DeleteJob: languages: + Java: + versions: + - sdk_version: 2 + github: javav2/example_code/glue + sdkguide: + excerpts: + - description: + snippet_tags: + - glue.java2.delete_job.main .NET: versions: - sdk_version: 3 @@ -961,6 +997,15 @@ glue_DeleteTable: glue: {DeleteTable} glue_DeleteDatabase: languages: + Java: + versions: + - sdk_version: 2 + github: javav2/example_code/glue + sdkguide: + excerpts: + - description: + snippet_tags: + - glue.java2.delete_database.main .NET: versions: - sdk_version: 3 @@ -1029,6 +1074,15 @@ glue_DeleteDatabase: glue: {DeleteDatabase} glue_DeleteCrawler: languages: + Java: + versions: + - sdk_version: 2 + github: javav2/example_code/glue + sdkguide: + excerpts: + - description: + snippet_tags: + - glue.java2.delete_crawler.main .NET: versions: - sdk_version: 3 diff --git a/javav2/example_code/glue/README.md b/javav2/example_code/glue/README.md index 38e998832b9..010f49a527e 100644 --- a/javav2/example_code/glue/README.md +++ b/javav2/example_code/glue/README.md @@ -38,18 +38,24 @@ For prerequisites, see the [README](../../README.md#Prerequisites) in the `javav Code examples that show you how to perform the essential operations within a service. -- [Learn the basics](src/main/java/com/example/glue/GlueScenario.java) +- [Learn the basics](src/main/java/com/example/glue/scenario/GlueScenario.java) ### Single actions Code excerpts that show you how to call individual service functions. -- [CreateCrawler](src/main/java/com/example/glue/CreateCrawler.java#L6) -- [GetCrawler](src/main/java/com/example/glue/GetCrawler.java#L6) -- [GetDatabase](src/main/java/com/example/glue/GetDatabase.java#L6) -- [GetTables](src/main/java/com/example/glue/GetTable.java#L6) -- [StartCrawler](src/main/java/com/example/glue/StartCrawler.java#L6) +- [CreateCrawler](src/main/java/com/example/glue/scenario/GlueScenario.java#L335) +- [CreateJob](src/main/java/com/example/glue/scenario/GlueScenario.java#L538) +- [DeleteCrawler](src/main/java/com/example/glue/scenario/GlueScenario.java#L702) +- [DeleteDatabase](src/main/java/com/example/glue/scenario/GlueScenario.java#L679) +- [DeleteJob](src/main/java/com/example/glue/scenario/GlueScenario.java#L655) +- [GetCrawler](src/main/java/com/example/glue/scenario/GlueScenario.java#L383) +- [GetDatabase](src/main/java/com/example/glue/scenario/GlueScenario.java#L437) +- [GetJobRuns](src/main/java/com/example/glue/scenario/GlueScenario.java#L602) +- [GetTables](src/main/java/com/example/glue/scenario/GlueScenario.java#L468) +- [StartCrawler](src/main/java/com/example/glue/scenario/GlueScenario.java#L414) +- [StartJobRun](src/main/java/com/example/glue/scenario/GlueScenario.java#L502) diff --git a/javav2/example_code/glue/src/main/java/com/example/glue/CreateCrawler.java b/javav2/example_code/glue/src/main/java/com/example/glue/CreateCrawler.java deleted file mode 100644 index 478f79137db..00000000000 --- a/javav2/example_code/glue/src/main/java/com/example/glue/CreateCrawler.java +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 - -package com.example.glue; - -// snippet-start:[glue.java2.create_crawler.main] -// snippet-start:[glue.java2.create_crawler.import] -import software.amazon.awssdk.regions.Region; -import software.amazon.awssdk.services.glue.GlueClient; -import software.amazon.awssdk.services.glue.model.CreateCrawlerRequest; -import software.amazon.awssdk.services.glue.model.CrawlerTargets; -import software.amazon.awssdk.services.glue.model.GlueException; -import software.amazon.awssdk.services.glue.model.S3Target; -import java.util.ArrayList; -import java.util.List; -// snippet-end:[glue.java2.create_crawler.import] - -/** - * Before running this Java V2 code example, set up your development - * environment, including your credentials. - * - * For more information, see the following documentation topic: - * - * https://docs.aws.amazon.com/sdk-for-java/latest/developer-guide/get-started.html - */ -public class CreateCrawler { - public static void main(String[] args) { - final String usage = """ - - Usage: - - - Where: - IAM - The ARN of the IAM role that has AWS Glue and S3 permissions.\s - s3Path - The Amazon Simple Storage Service (Amazon S3) target that contains data (for example, CSV data). - cron - A cron expression used to specify the schedule (i.e., cron(15 12 * * ? *). - dbName - The database name.\s - crawlerName - The name of the crawler.\s - """; - - if (args.length != 5) { - System.out.println(usage); - System.exit(1); - } - - String iam = args[0]; - String s3Path = args[1]; - String cron = args[2]; - String dbName = args[3]; - String crawlerName = args[4]; - Region region = Region.US_EAST_1; - GlueClient glueClient = GlueClient.builder() - .region(region) - .build(); - - createGlueCrawler(glueClient, iam, s3Path, cron, dbName, crawlerName); - glueClient.close(); - } - - public static void createGlueCrawler(GlueClient glueClient, - String iam, - String s3Path, - String cron, - String dbName, - String crawlerName) { - - try { - S3Target s3Target = S3Target.builder() - .path(s3Path) - .build(); - - // Add the S3Target to a list. - List targetList = new ArrayList<>(); - targetList.add(s3Target); - - CrawlerTargets targets = CrawlerTargets.builder() - .s3Targets(targetList) - .build(); - - CreateCrawlerRequest crawlerRequest = CreateCrawlerRequest.builder() - .databaseName(dbName) - .name(crawlerName) - .description("Created by the AWS Glue Java API") - .targets(targets) - .role(iam) - .schedule(cron) - .build(); - - glueClient.createCrawler(crawlerRequest); - System.out.println(crawlerName + " was successfully created"); - - } catch (GlueException e) { - System.err.println(e.awsErrorDetails().errorMessage()); - System.exit(1); - } - } -} -// snippet-end:[glue.java2.create_crawler.main] diff --git a/javav2/example_code/glue/src/main/java/com/example/glue/DeleteCrawler.java b/javav2/example_code/glue/src/main/java/com/example/glue/DeleteCrawler.java deleted file mode 100644 index c6c475dd76e..00000000000 --- a/javav2/example_code/glue/src/main/java/com/example/glue/DeleteCrawler.java +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 - -package com.example.glue; - -// snippet-start:[glue.java2.delete_crawler.main] -// snippet-start:[glue.java2.delete_crawler.import] -import software.amazon.awssdk.regions.Region; -import software.amazon.awssdk.services.glue.GlueClient; -import software.amazon.awssdk.services.glue.model.DeleteCrawlerRequest; -import software.amazon.awssdk.services.glue.model.GlueException; -// snippet-end:[glue.java2.delete_crawler.import] - -/** - * Before running this Java V2 code example, set up your development - * environment, including your credentials. - * - * For more information, see the following documentation topic: - * - * https://docs.aws.amazon.com/sdk-for-java/latest/developer-guide/get-started.html - */ -public class DeleteCrawler { - public static void main(String[] args) { - final String usage = """ - - Usage: - - - Where: - crawlerName - The name of the crawler.\s - """; - - if (args.length != 1) { - System.out.println(usage); - System.exit(1); - } - - String crawlerName = args[0]; - Region region = Region.US_EAST_1; - GlueClient glueClient = GlueClient.builder() - .region(region) - .build(); - - deleteSpecificCrawler(glueClient, crawlerName); - glueClient.close(); - } - - public static void deleteSpecificCrawler(GlueClient glueClient, String crawlerName) { - try { - DeleteCrawlerRequest deleteCrawlerRequest = DeleteCrawlerRequest.builder() - .name(crawlerName) - .build(); - - glueClient.deleteCrawler(deleteCrawlerRequest); - System.out.println(crawlerName + " was deleted"); - - } catch (GlueException e) { - System.err.println(e.awsErrorDetails().errorMessage()); - System.exit(1); - } - } -} -// snippet-end:[glue.java2.delete_crawler.main] \ No newline at end of file diff --git a/javav2/example_code/glue/src/main/java/com/example/glue/GetCrawler.java b/javav2/example_code/glue/src/main/java/com/example/glue/GetCrawler.java deleted file mode 100644 index 422e92c4608..00000000000 --- a/javav2/example_code/glue/src/main/java/com/example/glue/GetCrawler.java +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 - -package com.example.glue; - -// snippet-start:[glue.java2.get_crawler.main] -// snippet-start:[glue.java2.get_crawler.import] -import software.amazon.awssdk.regions.Region; -import software.amazon.awssdk.services.glue.GlueClient; -import software.amazon.awssdk.services.glue.model.GetCrawlerRequest; -import software.amazon.awssdk.services.glue.model.GetCrawlerResponse; -import software.amazon.awssdk.services.glue.model.GlueException; -import java.time.Instant; -import java.time.ZoneId; -import java.time.format.DateTimeFormatter; -import java.time.format.FormatStyle; -import java.util.Locale; -// snippet-end:[glue.java2.get_crawler.import] - -/** - * Before running this Java V2 code example, set up your development - * environment, including your credentials. - * - * For more information, see the following documentation topic: - * - * https://docs.aws.amazon.com/sdk-for-java/latest/developer-guide/get-started.html - */ -public class GetCrawler { - public static void main(String[] args) { - final String usage = """ - - Usage: - - - Where: - crawlerName - The name of the crawler.\s - """; - - if (args.length != 1) { - System.out.println(usage); - System.exit(1); - } - - String crawlerName = args[0]; - Region region = Region.US_EAST_1; - GlueClient glueClient = GlueClient.builder() - .region(region) - .build(); - - getSpecificCrawler(glueClient, crawlerName); - glueClient.close(); - } - - public static void getSpecificCrawler(GlueClient glueClient, String crawlerName) { - try { - GetCrawlerRequest crawlerRequest = GetCrawlerRequest.builder() - .name(crawlerName) - .build(); - - GetCrawlerResponse response = glueClient.getCrawler(crawlerRequest); - Instant createDate = response.crawler().creationTime(); - - // Convert the Instant to readable date - DateTimeFormatter formatter = DateTimeFormatter.ofLocalizedDateTime(FormatStyle.SHORT) - .withLocale(Locale.US) - .withZone(ZoneId.systemDefault()); - - formatter.format(createDate); - System.out.println("The create date of the Crawler is " + createDate); - - } catch (GlueException e) { - System.err.println(e.awsErrorDetails().errorMessage()); - System.exit(1); - } - } -} -// snippet-end:[glue.java2.get_crawler.main] diff --git a/javav2/example_code/glue/src/main/java/com/example/glue/GetCrawlers.java b/javav2/example_code/glue/src/main/java/com/example/glue/GetCrawlers.java deleted file mode 100644 index 047ed58af26..00000000000 --- a/javav2/example_code/glue/src/main/java/com/example/glue/GetCrawlers.java +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 - -package com.example.glue; - -// snippet-start:[glue.java2.get_crawlers.main] -// snippet-start:[glue.java2.get_crawlers.import] -import software.amazon.awssdk.regions.Region; -import software.amazon.awssdk.services.glue.GlueClient; -import software.amazon.awssdk.services.glue.model.GetCrawlersRequest; -import software.amazon.awssdk.services.glue.model.GetCrawlersResponse; -import software.amazon.awssdk.services.glue.model.Crawler; -import software.amazon.awssdk.services.glue.model.GlueException; -import java.util.List; -// snippet-end:[glue.java2.get_crawlers.import] - -/** - * Before running this Java V2 code example, set up your development - * environment, including your credentials. - * - * For more information, see the following documentation topic: - * - * https://docs.aws.amazon.com/sdk-for-java/latest/developer-guide/get-started.html - */ -public class GetCrawlers { - public static void main(String[] args) { - Region region = Region.US_EAST_1; - GlueClient glueClient = GlueClient.builder() - .region(region) - .build(); - - getAllCrawlers(glueClient); - glueClient.close(); - } - - public static void getAllCrawlers(GlueClient glueClient) { - try { - GetCrawlersRequest crawlersRequest = GetCrawlersRequest.builder() - .maxResults(10) - .build(); - - GetCrawlersResponse response = glueClient.getCrawlers(crawlersRequest); - List crawlers = response.crawlers(); - for (Crawler crawler : crawlers) { - System.out.println("The crawler name is : " + crawler.name()); - } - - } catch (GlueException e) { - System.err.println(e.awsErrorDetails().errorMessage()); - System.exit(1); - } - } -} -// snippet-end:[glue.java2.get_crawlers.main] diff --git a/javav2/example_code/glue/src/main/java/com/example/glue/GetDatabase.java b/javav2/example_code/glue/src/main/java/com/example/glue/GetDatabase.java deleted file mode 100644 index d8b39f289ed..00000000000 --- a/javav2/example_code/glue/src/main/java/com/example/glue/GetDatabase.java +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 - -package com.example.glue; - -// snippet-start:[glue.java2.get_database.main] -// snippet-start:[glue.java2.get_database.import] -import software.amazon.awssdk.regions.Region; -import software.amazon.awssdk.services.glue.GlueClient; -import software.amazon.awssdk.services.glue.model.GetDatabaseRequest; -import software.amazon.awssdk.services.glue.model.GetDatabaseResponse; -import software.amazon.awssdk.services.glue.model.GlueException; -import java.time.Instant; -import java.time.ZoneId; -import java.time.format.DateTimeFormatter; -import java.time.format.FormatStyle; -import java.util.Locale; -// snippet-end:[glue.java2.get_database.import] - -/** - * Before running this Java V2 code example, set up your development - * environment, including your credentials. - * - * For more information, see the following documentation topic: - * - * https://docs.aws.amazon.com/sdk-for-java/latest/developer-guide/get-started.html - */ -public class GetDatabase { - public static void main(String[] args) { - final String usage = """ - - Usage: - - - Where: - databaseName - The name of the database.\s - """; - - if (args.length != 1) { - System.out.println(usage); - System.exit(1); - } - - String databaseName = args[0]; - Region region = Region.US_EAST_1; - GlueClient glueClient = GlueClient.builder() - .region(region) - .build(); - - getSpecificDatabase(glueClient, databaseName); - glueClient.close(); - } - - public static void getSpecificDatabase(GlueClient glueClient, String databaseName) { - try { - GetDatabaseRequest databasesRequest = GetDatabaseRequest.builder() - .name(databaseName) - .build(); - - GetDatabaseResponse response = glueClient.getDatabase(databasesRequest); - Instant createDate = response.database().createTime(); - - // Convert the Instant to readable date. - DateTimeFormatter formatter = DateTimeFormatter.ofLocalizedDateTime(FormatStyle.SHORT) - .withLocale(Locale.US) - .withZone(ZoneId.systemDefault()); - - formatter.format(createDate); - System.out.println("The create date of the database is " + createDate); - - } catch (GlueException e) { - System.err.println(e.awsErrorDetails().errorMessage()); - System.exit(1); - } - } -} -// snippet-end:[glue.java2.get_database.main] diff --git a/javav2/example_code/glue/src/main/java/com/example/glue/GetDatabases.java b/javav2/example_code/glue/src/main/java/com/example/glue/GetDatabases.java deleted file mode 100644 index 412a7a781b2..00000000000 --- a/javav2/example_code/glue/src/main/java/com/example/glue/GetDatabases.java +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 - -package com.example.glue; - -// snippet-start:[glue.java2.get_databases.main] -// snippet-start:[glue.java2.get_databases.import] -import software.amazon.awssdk.regions.Region; -import software.amazon.awssdk.services.glue.GlueClient; -import software.amazon.awssdk.services.glue.model.Database; -import software.amazon.awssdk.services.glue.model.GetDatabasesRequest; -import software.amazon.awssdk.services.glue.model.GetDatabasesResponse; -import software.amazon.awssdk.services.glue.model.GlueException; -import java.util.List; -// snippet-end:[glue.java2.get_databases.import] - -/** - * Before running this Java V2 code example, set up your development - * environment, including your credentials. - * - * For more information, see the following documentation topic: - * - * https://docs.aws.amazon.com/sdk-for-java/latest/developer-guide/get-started.html - */ -public class GetDatabases { - public static void main(String[] args) { - Region region = Region.US_EAST_1; - GlueClient glueClient = GlueClient.builder() - .region(region) - .build(); - - getAllDatabases(glueClient); - glueClient.close(); - } - - public static void getAllDatabases(GlueClient glueClient) { - try { - GetDatabasesRequest databasesRequest = GetDatabasesRequest.builder() - .maxResults(10) - .build(); - - GetDatabasesResponse response = glueClient.getDatabases(databasesRequest); - List databases = response.databaseList(); - for (Database database : databases) { - System.out.println("The Database name is : " + database.name()); - } - - } catch (GlueException e) { - System.err.println(e.awsErrorDetails().errorMessage()); - System.exit(1); - } - } -} -// snippet-end:[glue.java2.get_databases.main] diff --git a/javav2/example_code/glue/src/main/java/com/example/glue/GetJobRun.java b/javav2/example_code/glue/src/main/java/com/example/glue/GetJobRun.java deleted file mode 100644 index 677263f89c1..00000000000 --- a/javav2/example_code/glue/src/main/java/com/example/glue/GetJobRun.java +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 - -package com.example.glue; - -// snippet-start:[glue.java2.get_job.main] -// snippet-start:[glue.java2.get_job.import] -import software.amazon.awssdk.regions.Region; -import software.amazon.awssdk.services.glue.GlueClient; -import software.amazon.awssdk.services.glue.model.GetJobRunRequest; -import software.amazon.awssdk.services.glue.model.GetJobRunResponse; -import software.amazon.awssdk.services.glue.model.GlueException; -// snippet-end:[glue.java2.get_job.import] - -/** - * Before running this Java V2 code example, set up your development - * environment, including your credentials. - * - * For more information, see the following documentation topic: - * - * https://docs.aws.amazon.com/sdk-for-java/latest/developer-guide/get-started.html - */ -public class GetJobRun { - public static void main(String[] args) { - final String usage = """ - - Usage: - - - Where: - jobName - The name of the job.\s - runId - The run id value.\s - """; - - if (args.length != 2) { - System.out.println(usage); - System.exit(1); - } - - String jobName = args[0]; - String runId = args[1]; - Region region = Region.US_EAST_1; - GlueClient glueClient = GlueClient.builder() - .region(region) - .build(); - - getGlueJobRun(glueClient, jobName, runId); - glueClient.close(); - } - - public static void getGlueJobRun(GlueClient glueClient, String jobName, String runId) { - try { - GetJobRunRequest jobRunRequest = GetJobRunRequest.builder() - .jobName(jobName) - .runId(runId) - .build(); - - GetJobRunResponse runResponse = glueClient.getJobRun(jobRunRequest); - System.out.println("Job status is : " + runResponse.jobRun().jobRunStateAsString()); - - } catch (GlueException e) { - System.err.println(e.awsErrorDetails().errorMessage()); - System.exit(1); - } - } -} -// snippet-end:[glue.java2.get_job.main] diff --git a/javav2/example_code/glue/src/main/java/com/example/glue/GetJobs.java b/javav2/example_code/glue/src/main/java/com/example/glue/GetJobs.java deleted file mode 100644 index cb1b88c4c47..00000000000 --- a/javav2/example_code/glue/src/main/java/com/example/glue/GetJobs.java +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 - -package com.example.glue; - -// snippet-start:[glue.java2.get_jobs.main] -// snippet-start:[glue.java2.get_jobs.import] -import software.amazon.awssdk.regions.Region; -import software.amazon.awssdk.services.glue.GlueClient; -import software.amazon.awssdk.services.glue.model.GetJobsRequest; -import software.amazon.awssdk.services.glue.model.GetJobsResponse; -import software.amazon.awssdk.services.glue.model.GlueException; -import software.amazon.awssdk.services.glue.model.Job; -import java.util.List; -// snippet-end:[glue.java2.get_jobs.import] - -/** - * Before running this Java V2 code example, set up your development - * environment, including your credentials. - * - * For more information, see the following documentation topic: - * - * https://docs.aws.amazon.com/sdk-for-java/latest/developer-guide/get-started.html - */ -public class GetJobs { - public static void main(String[] args) { - Region region = Region.US_EAST_1; - GlueClient glueClient = GlueClient.builder() - .region(region) - .build(); - - getAllJobs(glueClient); - glueClient.close(); - } - - public static void getAllJobs(GlueClient glueClient) { - try { - GetJobsRequest jobsRequest = GetJobsRequest.builder() - .maxResults(10) - .build(); - - GetJobsResponse jobsResponse = glueClient.getJobs(jobsRequest); - List jobs = jobsResponse.jobs(); - for (Job job : jobs) { - System.out.println("Job name is : " + job.name()); - System.out.println("The job worker type is : " + job.workerType().name()); - } - - } catch (GlueException e) { - System.err.println(e.awsErrorDetails().errorMessage()); - System.exit(1); - } - } -} -// snippet-end:[glue.java2.get_jobs.main] diff --git a/javav2/example_code/glue/src/main/java/com/example/glue/GetTable.java b/javav2/example_code/glue/src/main/java/com/example/glue/GetTable.java deleted file mode 100644 index cf231c4dd59..00000000000 --- a/javav2/example_code/glue/src/main/java/com/example/glue/GetTable.java +++ /dev/null @@ -1,80 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 - -package com.example.glue; - -// snippet-start:[glue.java2.get_table.main] -// snippet-start:[glue.java2.get_table.import] -import software.amazon.awssdk.regions.Region; -import software.amazon.awssdk.services.glue.GlueClient; -import software.amazon.awssdk.services.glue.model.GetTableRequest; -import software.amazon.awssdk.services.glue.model.GetTableResponse; -import software.amazon.awssdk.services.glue.model.GlueException; -import java.time.Instant; -import java.time.ZoneId; -import java.time.format.DateTimeFormatter; -import java.time.format.FormatStyle; -import java.util.Locale; -// snippet-end:[glue.java2.get_table.import] - -/** - * Before running this Java V2 code example, set up your development - * environment, including your credentials. - * - * For more information, see the following documentation topic: - * - * https://docs.aws.amazon.com/sdk-for-java/latest/developer-guide/get-started.html - */ -public class GetTable { - public static void main(String[] args) { - final String usage = """ - - Usage: - - - Where: - dbName - The database name.\s - tableName - The name of the table.\s - """; - - if (args.length != 2) { - System.out.println(usage); - System.exit(1); - } - - String dbName = args[0]; - String tableName = args[1]; - Region region = Region.US_EAST_1; - GlueClient glueClient = GlueClient.builder() - .region(region) - .build(); - - getGlueTable(glueClient, dbName, tableName); - glueClient.close(); - } - - public static void getGlueTable(GlueClient glueClient, String dbName, String tableName) { - try { - GetTableRequest tableRequest = GetTableRequest.builder() - .databaseName(dbName) - .name(tableName) - .build(); - - GetTableResponse tableResponse = glueClient.getTable(tableRequest); - Instant createDate = tableResponse.table().createTime(); - - // Convert the Instant to readable date. - DateTimeFormatter formatter = DateTimeFormatter.ofLocalizedDateTime(FormatStyle.SHORT) - .withLocale(Locale.US) - .withZone(ZoneId.systemDefault()); - - formatter.format(createDate); - System.out.println("The create date of the table is " + createDate); - - } catch (GlueException e) { - System.err.println(e.awsErrorDetails().errorMessage()); - System.exit(1); - } - } -} -// snippet-end:[glue.java2.get_table.main] diff --git a/javav2/example_code/glue/src/main/java/com/example/glue/ListWorkflows.java b/javav2/example_code/glue/src/main/java/com/example/glue/ListWorkflows.java deleted file mode 100644 index 9ca8f9c2370..00000000000 --- a/javav2/example_code/glue/src/main/java/com/example/glue/ListWorkflows.java +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 - -package com.example.glue; - -// snippet-start:[glue.java2.list_wfs.main] -// snippet-start:[glue.java2.list_wfs.import] -import software.amazon.awssdk.regions.Region; -import software.amazon.awssdk.services.glue.GlueClient; -import software.amazon.awssdk.services.glue.model.GlueException; -import software.amazon.awssdk.services.glue.model.ListWorkflowsRequest; -import software.amazon.awssdk.services.glue.model.ListWorkflowsResponse; -import java.util.List; -// snippet-end:[glue.java2.list_wfs.import] - -/** - * Before running this Java V2 code example, set up your development - * environment, including your credentials. - * - * For more information, see the following documentation topic: - * - * https://docs.aws.amazon.com/sdk-for-java/latest/developer-guide/get-started.html - */ -public class ListWorkflows { - public static void main(String[] args) { - Region region = Region.US_EAST_1; - GlueClient glueClient = GlueClient.builder() - .region(region) - .build(); - - listAllWorkflows(glueClient); - glueClient.close(); - } - - public static void listAllWorkflows(GlueClient glueClient) { - try { - ListWorkflowsRequest workflowsRequest = ListWorkflowsRequest.builder() - .maxResults(10) - .build(); - - ListWorkflowsResponse workflowsResponse = glueClient.listWorkflows(workflowsRequest); - List workflows = workflowsResponse.workflows(); - for (String workflow : workflows) { - System.out.println("Workflow name is: " + workflow); - } - - } catch (GlueException e) { - System.err.println(e.awsErrorDetails().errorMessage()); - System.exit(1); - } - } -} -// snippet-end:[glue.java2.list_wfs.main] diff --git a/javav2/example_code/glue/src/main/java/com/example/glue/StartCrawler.java b/javav2/example_code/glue/src/main/java/com/example/glue/StartCrawler.java deleted file mode 100644 index 4a175272f19..00000000000 --- a/javav2/example_code/glue/src/main/java/com/example/glue/StartCrawler.java +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 - -package com.example.glue; - -// snippet-start:[glue.java2.start_crawler.main] -// snippet-start:[glue.java2.start_crawler.import] -import software.amazon.awssdk.regions.Region; -import software.amazon.awssdk.services.glue.GlueClient; -import software.amazon.awssdk.services.glue.model.GlueException; -import software.amazon.awssdk.services.glue.model.StartCrawlerRequest; -// snippet-end:[glue.java2.start_crawler.import] - -/** - * Before running this Java V2 code example, set up your development - * environment, including your credentials. - * - * For more information, see the following documentation topic: - * - * https://docs.aws.amazon.com/sdk-for-java/latest/developer-guide/get-started.html - */ -public class StartCrawler { - public static void main(String[] args) { - final String usage = """ - - Usage: - - - Where: - crawlerName - The name of the crawler.\s - """; - - if (args.length != 1) { - System.out.println(usage); - System.exit(1); - } - - String crawlerName = args[0]; - Region region = Region.US_EAST_1; - GlueClient glueClient = GlueClient.builder() - .region(region) - .build(); - - startSpecificCrawler(glueClient, crawlerName); - glueClient.close(); - } - - public static void startSpecificCrawler(GlueClient glueClient, String crawlerName) { - try { - StartCrawlerRequest crawlerRequest = StartCrawlerRequest.builder() - .name(crawlerName) - .build(); - - glueClient.startCrawler(crawlerRequest); - - } catch (GlueException e) { - System.err.println(e.awsErrorDetails().errorMessage()); - System.exit(1); - } - } -} -// snippet-end:[glue.java2.start_crawler.main] diff --git a/javav2/example_code/glue/src/main/java/com/example/glue/GlueScenario.java b/javav2/example_code/glue/src/main/java/com/example/glue/scenario/GlueScenario.java similarity index 51% rename from javav2/example_code/glue/src/main/java/com/example/glue/GlueScenario.java rename to javav2/example_code/glue/src/main/java/com/example/glue/scenario/GlueScenario.java index e69f8972467..429747796d0 100644 --- a/javav2/example_code/glue/src/main/java/com/example/glue/GlueScenario.java +++ b/javav2/example_code/glue/src/main/java/com/example/glue/scenario/GlueScenario.java @@ -1,501 +1,743 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 - -package com.example.glue; - -// snippet-start:[glue.java2.scenario.import] -import software.amazon.awssdk.regions.Region; -import software.amazon.awssdk.services.glue.GlueClient; -import java.time.Instant; -import java.time.ZoneId; -import java.time.format.DateTimeFormatter; -import java.time.format.FormatStyle; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.concurrent.TimeUnit; -import software.amazon.awssdk.services.glue.model.DatabaseInput; -import software.amazon.awssdk.services.glue.model.CreateDatabaseRequest; -import software.amazon.awssdk.services.glue.model.GlueException; -import software.amazon.awssdk.services.glue.model.GetCrawlerResponse; -import software.amazon.awssdk.services.glue.model.GetCrawlerRequest; -import software.amazon.awssdk.services.glue.model.CreateCrawlerRequest; -import software.amazon.awssdk.services.glue.model.GetTablesResponse; -import software.amazon.awssdk.services.glue.model.GetTablesRequest; -import software.amazon.awssdk.services.glue.model.StartCrawlerRequest; -import software.amazon.awssdk.services.glue.model.GetDatabaseRequest; -import software.amazon.awssdk.services.glue.model.CrawlerTargets; -import software.amazon.awssdk.services.glue.model.GetDatabaseResponse; -import software.amazon.awssdk.services.glue.model.Table; -import software.amazon.awssdk.services.glue.model.StartJobRunResponse; -import software.amazon.awssdk.services.glue.model.StartJobRunRequest; -import software.amazon.awssdk.services.glue.model.GetJobsRequest; -import software.amazon.awssdk.services.glue.model.CreateJobRequest; -import software.amazon.awssdk.services.glue.model.GetJobRunsResponse; -import software.amazon.awssdk.services.glue.model.GetJobsResponse; -import software.amazon.awssdk.services.glue.model.Job; -import software.amazon.awssdk.services.glue.model.WorkerType; -import software.amazon.awssdk.services.glue.model.JobCommand; -import software.amazon.awssdk.services.glue.model.GetJobRunsRequest; -import software.amazon.awssdk.services.glue.model.JobRun; -import software.amazon.awssdk.services.glue.model.S3Target; -import software.amazon.awssdk.services.glue.model.DeleteJobRequest; -import software.amazon.awssdk.services.glue.model.DeleteDatabaseRequest; -import software.amazon.awssdk.services.glue.model.DeleteCrawlerRequest; -// snippet-end:[glue.java2.scenario.import] - -// snippet-start:[glue.java2.scenario.main] -/** - * - * Before running this Java V2 code example, set up your development - * environment, including your credentials. - * - * For more information, see the following documentation topic: - * - * https://docs.aws.amazon.com/sdk-for-java/latest/developer-guide/get-started.html - * - * To set up the resources, see this documentation topic: - * - * https://docs.aws.amazon.com/glue/latest/ug/tutorial-add-crawler.html - * - * This example performs the following tasks: - * - * 1. Create a database. - * 2. Create a crawler. - * 3. Get a crawler. - * 4. Start a crawler. - * 5. Get a database. - * 6. Get tables. - * 7. Create a job. - * 8. Start a job run. - * 9. List all jobs. - * 10. Get job runs. - * 11. Delete a job. - * 12. Delete a database. - * 13. Delete a crawler. - */ - -public class GlueScenario { - public static final String DASHES = new String(new char[80]).replace("\0", "-"); - - public static void main(String[] args) throws InterruptedException { - final String usage = """ - - Usage: - \s - - Where: - iam - The ARN of the IAM role that has AWS Glue and S3 permissions.\s - s3Path - The Amazon Simple Storage Service (Amazon S3) target that contains data (for example, CSV data). - cron - A cron expression used to specify the schedule (i.e., cron(15 12 * * ? *). - dbName - The database name.\s - crawlerName - The name of the crawler.\s - jobName - The name you assign to this job definition. - scriptLocation - The Amazon S3 path to a script that runs a job. - locationUri - The location of the database - bucketNameSc - The Amazon S3 bucket name used when creating a job - """; - - if (args.length != 9) { - System.out.println(usage); - System.exit(1); - } - - String iam = args[0]; - String s3Path = args[1]; - String cron = args[2]; - String dbName = args[3]; - String crawlerName = args[4]; - String jobName = args[5]; - String scriptLocation = args[6]; - String locationUri = args[7]; - String bucketNameSc = args[8]; - - Region region = Region.US_EAST_1; - GlueClient glueClient = GlueClient.builder() - .region(region) - .build(); - System.out.println(DASHES); - System.out.println("Welcome to the AWS Glue scenario."); - System.out.println(DASHES); - - System.out.println(DASHES); - System.out.println("1. Create a database."); - createDatabase(glueClient, dbName, locationUri); - System.out.println(DASHES); - - System.out.println(DASHES); - System.out.println("2. Create a crawler."); - createGlueCrawler(glueClient, iam, s3Path, cron, dbName, crawlerName); - System.out.println(DASHES); - - System.out.println(DASHES); - System.out.println("3. Get a crawler."); - getSpecificCrawler(glueClient, crawlerName); - System.out.println(DASHES); - - System.out.println(DASHES); - System.out.println("4. Start a crawler."); - startSpecificCrawler(glueClient, crawlerName); - System.out.println(DASHES); - - System.out.println(DASHES); - System.out.println("5. Get a database."); - getSpecificDatabase(glueClient, dbName); - System.out.println(DASHES); - - System.out.println(DASHES); - System.out.println("*** Wait 5 min for the tables to become available"); - TimeUnit.MINUTES.sleep(5); - System.out.println("6. Get tables."); - String myTableName = getGlueTables(glueClient, dbName); - System.out.println(DASHES); - - System.out.println(DASHES); - System.out.println("7. Create a job."); - createJob(glueClient, jobName, iam, scriptLocation); - System.out.println(DASHES); - - System.out.println(DASHES); - System.out.println("8. Start a Job run."); - startJob(glueClient, jobName, dbName, myTableName, bucketNameSc); - System.out.println(DASHES); - - System.out.println(DASHES); - System.out.println("9. List all jobs."); - getAllJobs(glueClient); - System.out.println(DASHES); - - System.out.println(DASHES); - System.out.println("10. Get job runs."); - getJobRuns(glueClient, jobName); - System.out.println(DASHES); - - System.out.println(DASHES); - System.out.println("11. Delete a job."); - deleteJob(glueClient, jobName); - System.out.println("*** Wait 5 MIN for the " + crawlerName + " to stop"); - TimeUnit.MINUTES.sleep(5); - System.out.println(DASHES); - - System.out.println(DASHES); - System.out.println("12. Delete a database."); - deleteDatabase(glueClient, dbName); - System.out.println(DASHES); - - System.out.println(DASHES); - System.out.println("Delete a crawler."); - deleteSpecificCrawler(glueClient, crawlerName); - System.out.println(DASHES); - - System.out.println(DASHES); - System.out.println("Successfully completed the AWS Glue Scenario"); - System.out.println(DASHES); - } - - public static void createDatabase(GlueClient glueClient, String dbName, String locationUri) { - try { - DatabaseInput input = DatabaseInput.builder() - .description("Built with the AWS SDK for Java V2") - .name(dbName) - .locationUri(locationUri) - .build(); - - CreateDatabaseRequest request = CreateDatabaseRequest.builder() - .databaseInput(input) - .build(); - - glueClient.createDatabase(request); - System.out.println(dbName + " was successfully created"); - - } catch (GlueException e) { - System.err.println(e.awsErrorDetails().errorMessage()); - System.exit(1); - } - } - - public static void createGlueCrawler(GlueClient glueClient, - String iam, - String s3Path, - String cron, - String dbName, - String crawlerName) { - - try { - S3Target s3Target = S3Target.builder() - .path(s3Path) - .build(); - - List targetList = new ArrayList<>(); - targetList.add(s3Target); - CrawlerTargets targets = CrawlerTargets.builder() - .s3Targets(targetList) - .build(); - - CreateCrawlerRequest crawlerRequest = CreateCrawlerRequest.builder() - .databaseName(dbName) - .name(crawlerName) - .description("Created by the AWS Glue Java API") - .targets(targets) - .role(iam) - .schedule(cron) - .build(); - - glueClient.createCrawler(crawlerRequest); - System.out.println(crawlerName + " was successfully created"); - - } catch (GlueException e) { - System.err.println(e.awsErrorDetails().errorMessage()); - System.exit(1); - } - } - - public static void getSpecificCrawler(GlueClient glueClient, String crawlerName) { - try { - GetCrawlerRequest crawlerRequest = GetCrawlerRequest.builder() - .name(crawlerName) - .build(); - - boolean ready = false; - while (!ready) { - GetCrawlerResponse response = glueClient.getCrawler(crawlerRequest); - String status = response.crawler().stateAsString(); - if (status.compareTo("READY") == 0) { - ready = true; - } - Thread.sleep(3000); - } - - System.out.println("The crawler is now ready"); - - } catch (GlueException | InterruptedException e) { - System.err.println(e.getMessage()); - System.exit(1); - } - } - - public static void startSpecificCrawler(GlueClient glueClient, String crawlerName) { - try { - StartCrawlerRequest crawlerRequest = StartCrawlerRequest.builder() - .name(crawlerName) - .build(); - - glueClient.startCrawler(crawlerRequest); - System.out.println(crawlerName + " was successfully started!"); - - } catch (GlueException e) { - System.err.println(e.awsErrorDetails().errorMessage()); - System.exit(1); - } - } - - public static void getSpecificDatabase(GlueClient glueClient, String databaseName) { - try { - GetDatabaseRequest databasesRequest = GetDatabaseRequest.builder() - .name(databaseName) - .build(); - - GetDatabaseResponse response = glueClient.getDatabase(databasesRequest); - Instant createDate = response.database().createTime(); - - // Convert the Instant to readable date. - DateTimeFormatter formatter = DateTimeFormatter.ofLocalizedDateTime(FormatStyle.SHORT) - .withLocale(Locale.US) - .withZone(ZoneId.systemDefault()); - - formatter.format(createDate); - System.out.println("The create date of the database is " + createDate); - - } catch (GlueException e) { - System.err.println(e.awsErrorDetails().errorMessage()); - System.exit(1); - } - } - - public static String getGlueTables(GlueClient glueClient, String dbName) { - String myTableName = ""; - try { - GetTablesRequest tableRequest = GetTablesRequest.builder() - .databaseName(dbName) - .build(); - - GetTablesResponse response = glueClient.getTables(tableRequest); - List tables = response.tableList(); - if (tables.isEmpty()) { - System.out.println("No tables were returned"); - } else { - for (Table table : tables) { - myTableName = table.name(); - System.out.println("Table name is: " + myTableName); - } - } - - } catch (GlueException e) { - System.err.println(e.awsErrorDetails().errorMessage()); - System.exit(1); - } - return myTableName; - } - - public static void startJob(GlueClient glueClient, String jobName, String inputDatabase, String inputTable, - String outBucket) { - try { - Map myMap = new HashMap<>(); - myMap.put("--input_database", inputDatabase); - myMap.put("--input_table", inputTable); - myMap.put("--output_bucket_url", outBucket); - - StartJobRunRequest runRequest = StartJobRunRequest.builder() - .workerType(WorkerType.G_1_X) - .numberOfWorkers(10) - .arguments(myMap) - .jobName(jobName) - .build(); - - StartJobRunResponse response = glueClient.startJobRun(runRequest); - System.out.println("The request Id of the job is " + response.responseMetadata().requestId()); - - } catch (GlueException e) { - System.err.println(e.awsErrorDetails().errorMessage()); - System.exit(1); - } - } - - public static void createJob(GlueClient glueClient, String jobName, String iam, String scriptLocation) { - try { - JobCommand command = JobCommand.builder() - .pythonVersion("3") - .name("glueetl") - .scriptLocation(scriptLocation) - .build(); - - CreateJobRequest jobRequest = CreateJobRequest.builder() - .description("A Job created by using the AWS SDK for Java V2") - .glueVersion("2.0") - .workerType(WorkerType.G_1_X) - .numberOfWorkers(10) - .name(jobName) - .role(iam) - .command(command) - .build(); - - glueClient.createJob(jobRequest); - System.out.println(jobName + " was successfully created."); - - } catch (GlueException e) { - System.err.println(e.awsErrorDetails().errorMessage()); - System.exit(1); - } - } - - public static void getAllJobs(GlueClient glueClient) { - try { - GetJobsRequest jobsRequest = GetJobsRequest.builder() - .maxResults(10) - .build(); - - GetJobsResponse jobsResponse = glueClient.getJobs(jobsRequest); - List jobs = jobsResponse.jobs(); - for (Job job : jobs) { - System.out.println("Job name is : " + job.name()); - System.out.println("The job worker type is : " + job.workerType().name()); - } - - } catch (GlueException e) { - System.err.println(e.awsErrorDetails().errorMessage()); - System.exit(1); - } - } - - public static void getJobRuns(GlueClient glueClient, String jobName) { - try { - GetJobRunsRequest runsRequest = GetJobRunsRequest.builder() - .jobName(jobName) - .maxResults(20) - .build(); - - boolean jobDone = false; - while (!jobDone) { - GetJobRunsResponse response = glueClient.getJobRuns(runsRequest); - List jobRuns = response.jobRuns(); - for (JobRun jobRun : jobRuns) { - String jobState = jobRun.jobRunState().name(); - if (jobState.compareTo("SUCCEEDED") == 0) { - System.out.println(jobName + " has succeeded"); - jobDone = true; - - } else if (jobState.compareTo("STOPPED") == 0) { - System.out.println("Job run has stopped"); - jobDone = true; - - } else if (jobState.compareTo("FAILED") == 0) { - System.out.println("Job run has failed"); - jobDone = true; - - } else if (jobState.compareTo("TIMEOUT") == 0) { - System.out.println("Job run has timed out"); - jobDone = true; - - } else { - System.out.println("*** Job run state is " + jobRun.jobRunState().name()); - System.out.println("Job run Id is " + jobRun.id()); - System.out.println("The Glue version is " + jobRun.glueVersion()); - } - TimeUnit.SECONDS.sleep(5); - } - } - - } catch (GlueException | InterruptedException e) { - System.err.println(e.getMessage()); - System.exit(1); - } - } - - public static void deleteJob(GlueClient glueClient, String jobName) { - try { - DeleteJobRequest jobRequest = DeleteJobRequest.builder() - .jobName(jobName) - .build(); - - glueClient.deleteJob(jobRequest); - System.out.println(jobName + " was successfully deleted"); - - } catch (GlueException e) { - System.err.println(e.awsErrorDetails().errorMessage()); - System.exit(1); - } - } - - public static void deleteDatabase(GlueClient glueClient, String databaseName) { - try { - DeleteDatabaseRequest request = DeleteDatabaseRequest.builder() - .name(databaseName) - .build(); - - glueClient.deleteDatabase(request); - System.out.println(databaseName + " was successfully deleted"); - - } catch (GlueException e) { - System.err.println(e.awsErrorDetails().errorMessage()); - System.exit(1); - } - } - - public static void deleteSpecificCrawler(GlueClient glueClient, String crawlerName) { - try { - DeleteCrawlerRequest deleteCrawlerRequest = DeleteCrawlerRequest.builder() - .name(crawlerName) - .build(); - - glueClient.deleteCrawler(deleteCrawlerRequest); - System.out.println(crawlerName + " was deleted"); - - } catch (GlueException e) { - System.err.println(e.awsErrorDetails().errorMessage()); - System.exit(1); - } - } -} +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package com.example.glue.scenario; + +// snippet-start:[glue.java2.scenario.import] + +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.glue.GlueClient; + +import java.time.Instant; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.time.format.FormatStyle; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Scanner; +import java.util.concurrent.TimeUnit; + +import software.amazon.awssdk.services.glue.model.DatabaseInput; +import software.amazon.awssdk.services.glue.model.CreateDatabaseRequest; +import software.amazon.awssdk.services.glue.model.GlueException; +import software.amazon.awssdk.services.glue.model.GetCrawlerResponse; +import software.amazon.awssdk.services.glue.model.GetCrawlerRequest; +import software.amazon.awssdk.services.glue.model.CreateCrawlerRequest; +import software.amazon.awssdk.services.glue.model.GetTablesResponse; +import software.amazon.awssdk.services.glue.model.GetTablesRequest; +import software.amazon.awssdk.services.glue.model.StartCrawlerRequest; +import software.amazon.awssdk.services.glue.model.GetDatabaseRequest; +import software.amazon.awssdk.services.glue.model.CrawlerTargets; +import software.amazon.awssdk.services.glue.model.GetDatabaseResponse; +import software.amazon.awssdk.services.glue.model.Table; +import software.amazon.awssdk.services.glue.model.StartJobRunResponse; +import software.amazon.awssdk.services.glue.model.StartJobRunRequest; +import software.amazon.awssdk.services.glue.model.GetJobsRequest; +import software.amazon.awssdk.services.glue.model.CreateJobRequest; +import software.amazon.awssdk.services.glue.model.GetJobRunsResponse; +import software.amazon.awssdk.services.glue.model.GetJobsResponse; +import software.amazon.awssdk.services.glue.model.Job; +import software.amazon.awssdk.services.glue.model.WorkerType; +import software.amazon.awssdk.services.glue.model.JobCommand; +import software.amazon.awssdk.services.glue.model.GetJobRunsRequest; +import software.amazon.awssdk.services.glue.model.JobRun; +import software.amazon.awssdk.services.glue.model.S3Target; +import software.amazon.awssdk.services.glue.model.DeleteJobRequest; +import software.amazon.awssdk.services.glue.model.DeleteDatabaseRequest; +import software.amazon.awssdk.services.glue.model.DeleteCrawlerRequest; +// snippet-end:[glue.java2.scenario.import] + +// snippet-start:[glue.java2.scenario.main] + +/** + * Before running this Java V2 code example, set up your development + * environment, including your credentials. + *

+ * For more information, see the following documentation topic: + * + * https://docs.aws.amazon.com/sdk-for-java/latest/developer-guide/get-started.html + * + * To set up the resources, see this documentation topic: + * + * https://docs.aws.amazon.com/glue/latest/ug/tutorial-add-crawler.html + * + * This example performs the following tasks: + * + * 1. Create a database. + * 2. Create a crawler. + * 3. Get a crawler. + * 4. Start a crawler. + * 5. Get a database. + * 6. Get tables. + * 7. Create a job. + * 8. Start a job run. + * 9. List all jobs. + * 10. Get job runs. + * 11. Delete a job. + * 12. Delete a database. + * 13. Delete a crawler. + */ + +public class GlueScenario { + public static final String DASHES = new String(new char[80]).replace("\0", "-"); + + public static void main(String[] args) throws InterruptedException { + final String usage = """ + + Usage: + \s + + Where: + iam - The ARN of the IAM role that has AWS Glue and S3 permissions.\s + s3Path - The Amazon Simple Storage Service (Amazon S3) target that contains data (for example, s3:///read). + cron - A cron expression used to specify the schedule (i.e., cron(15 12 * * ? *). + dbName - The database name.\s + crawlerName - The name of the crawler.\s + jobName - The name you assign to this job definition. + scriptLocation - The Amazon S3 path to a script that runs a job. + locationUri - The location of the database (you can find this file in resources folder). + bucketNameSc - The Amazon S3 bucket name used when creating a job + """; + + if (args.length != 9) { + System.out.println(usage); + return; + } + Scanner scanner = new Scanner(System.in); + String iam = args[0]; + String s3Path = args[1]; + String cron = args[2]; + String dbName = args[3]; + String crawlerName = args[4]; + String jobName = args[5]; + String scriptLocation = args[6]; + String locationUri = args[7]; + String bucketNameSc = args[8]; + + Region region = Region.US_EAST_1; + GlueClient glueClient = GlueClient.builder() + .region(region) + .build(); + System.out.println(DASHES); + System.out.println("Welcome to the AWS Glue scenario."); + System.out.println(""" + AWS Glue is a fully managed extract, transform, and load (ETL) service provided by Amazon + Web Services (AWS). It is designed to simplify the process of building, running, and maintaining + ETL pipelines, which are essential for data integration and data warehousing tasks. + + One of the key features of AWS Glue is its ability to automatically discover and catalog data + stored in various sources, such as Amazon S3, Amazon RDS, Amazon Redshift, and other databases. + This cataloging process creates a central metadata repository, known as the AWS Glue Data Catalog, + which provides a unified view of an organization's data assets. This metadata can then be used to + create ETL jobs, which can be scheduled and run on-demand or on a regular basis. + + Lets get started. + + """); + waitForInputToContinue(scanner); + System.out.println(DASHES); + + System.out.println(DASHES); + System.out.println("1. Create a database."); + try { + createDatabase(glueClient, dbName, locationUri); + } catch (GlueException e) { + if (e.awsErrorDetails().errorMessage().equals("Database already exists.")) { + System.out.println("Database " + dbName + " already exists. Skipping creation."); + } else { + System.err.println(e.awsErrorDetails().errorMessage()); + return; + } + } + + waitForInputToContinue(scanner); + System.out.println(DASHES); + + System.out.println(DASHES); + System.out.println("2. Create a crawler."); + try { + createGlueCrawler(glueClient, iam, s3Path, cron, dbName, crawlerName); + } catch (GlueException e) { + if (e.awsErrorDetails().errorMessage().contains("already exists")) { + System.out.println("Crawler " + crawlerName + " already exists. Skipping creation."); + } else { + System.err.println(e.awsErrorDetails().errorMessage()); + System.exit(1); + } + } + waitForInputToContinue(scanner); + System.out.println(DASHES); + + System.out.println(DASHES); + System.out.println("3. Get a crawler."); + try { + getSpecificCrawler(glueClient, crawlerName); + } catch (GlueException e) { + System.err.println(e.awsErrorDetails().errorMessage()); + return; + } + waitForInputToContinue(scanner); + System.out.println(DASHES); + + System.out.println(DASHES); + System.out.println("4. Start a crawler."); + try { + startSpecificCrawler(glueClient, crawlerName); + } catch (GlueException e) { + System.err.println(e.awsErrorDetails().errorMessage()); + return; + } + waitForInputToContinue(scanner); + System.out.println(DASHES); + + System.out.println(DASHES); + System.out.println("5. Get a database."); + try { + getSpecificDatabase(glueClient, dbName); + } catch (GlueException e) { + System.err.println(e.awsErrorDetails().errorMessage()); + return; + } + waitForInputToContinue(scanner); + System.out.println(DASHES); + + System.out.println(DASHES); + System.out.println("*** Wait 5 min for the tables to become available"); + TimeUnit.MINUTES.sleep(5); + System.out.println("6. Get tables."); + String myTableName; + try { + myTableName = getGlueTables(glueClient, dbName); + } catch (GlueException e) { + System.err.println(e.awsErrorDetails().errorMessage()); + return; + } + waitForInputToContinue(scanner); + System.out.println(DASHES); + + System.out.println(DASHES); + System.out.println("7. Create a job."); + try { + createJob(glueClient, jobName, iam, scriptLocation); + } catch (GlueException e) { + System.err.println(e.awsErrorDetails().errorMessage()); + return; + } + waitForInputToContinue(scanner); + System.out.println(DASHES); + + System.out.println(DASHES); + System.out.println("8. Start a Job run."); + try { + startJob(glueClient, jobName, dbName, myTableName, bucketNameSc); + } catch (GlueException e) { + System.err.println(e.awsErrorDetails().errorMessage()); + return; + } + waitForInputToContinue(scanner); + System.out.println(DASHES); + + System.out.println(DASHES); + System.out.println("9. List all jobs."); + try { + getAllJobs(glueClient); + } catch (GlueException e) { + System.err.println(e.awsErrorDetails().errorMessage()); + return; + } + waitForInputToContinue(scanner); + System.out.println(DASHES); + + System.out.println(DASHES); + System.out.println("10. Get job runs."); + try { + getJobRuns(glueClient, jobName); + } catch (GlueException e) { + System.err.println(e.awsErrorDetails().errorMessage()); + return; + } + waitForInputToContinue(scanner); + System.out.println(DASHES); + + System.out.println(DASHES); + System.out.println("11. Delete a job."); + try { + deleteJob(glueClient, jobName); + } catch (GlueException e) { + System.err.println(e.awsErrorDetails().errorMessage()); + return; + } + System.out.println("*** Wait 5 MIN for the " + crawlerName + " to stop"); + TimeUnit.MINUTES.sleep(5); + waitForInputToContinue(scanner); + System.out.println(DASHES); + + System.out.println(DASHES); + System.out.println("12. Delete a database."); + try { + deleteDatabase(glueClient, dbName); + } catch (GlueException e) { + System.err.println(e.awsErrorDetails().errorMessage()); + return; + } + waitForInputToContinue(scanner); + System.out.println(DASHES); + + System.out.println(DASHES); + System.out.println("Delete a crawler."); + try { + deleteSpecificCrawler(glueClient, crawlerName); + } catch (GlueException e) { + System.err.println(e.awsErrorDetails().errorMessage()); + return; + } + waitForInputToContinue(scanner); + System.out.println(DASHES); + + System.out.println(DASHES); + System.out.println("Successfully completed the AWS Glue Scenario"); + System.out.println(DASHES); + } + + // snippet-start:[glue.java2.create_database.main] + + /** + * Creates a Glue database with the specified name and location URI. + * + * @param glueClient The Glue client to use for the database creation. + * @param dbName The name of the database to create. + * @param locationUri The location URI for the database. + */ + public static void createDatabase(GlueClient glueClient, String dbName, String locationUri) { + try { + DatabaseInput input = DatabaseInput.builder() + .description("Built with the AWS SDK for Java V2") + .name(dbName) + .locationUri(locationUri) + .build(); + + CreateDatabaseRequest request = CreateDatabaseRequest.builder() + .databaseInput(input) + .build(); + + glueClient.createDatabase(request); + System.out.println(dbName + " was successfully created"); + + } catch (GlueException e) { + throw e; + } + } + // snippet-end:[glue.java2.create_database.main] + + // snippet-start:[glue.java2.create_crawler.main] + + /** + * Creates a new AWS Glue crawler using the AWS Glue Java API. + * + * @param glueClient the AWS Glue client used to interact with the AWS Glue service + * @param iam the IAM role that the crawler will use to access the data source + * @param s3Path the S3 path that the crawler will scan for data + * @param cron the cron expression that defines the crawler's schedule + * @param dbName the name of the AWS Glue database where the crawler will store the metadata + * @param crawlerName the name of the crawler to be created + */ + public static void createGlueCrawler(GlueClient glueClient, + String iam, + String s3Path, + String cron, + String dbName, + String crawlerName) { + + try { + S3Target s3Target = S3Target.builder() + .path(s3Path) + .build(); + + List targetList = new ArrayList<>(); + targetList.add(s3Target); + CrawlerTargets targets = CrawlerTargets.builder() + .s3Targets(targetList) + .build(); + + CreateCrawlerRequest crawlerRequest = CreateCrawlerRequest.builder() + .databaseName(dbName) + .name(crawlerName) + .description("Created by the AWS Glue Java API") + .targets(targets) + .role(iam) + .schedule(cron) + .build(); + + glueClient.createCrawler(crawlerRequest); + System.out.println(crawlerName + " was successfully created"); + + } catch (GlueException e) { + throw e; + } + } + // snippet-end:[glue.java2.create_crawler.main] + + // snippet-start:[glue.java2.get_crawler.main] + /** + * Retrieves a specific crawler from the AWS Glue service and waits for it to be in the "READY" state. + * + * @param glueClient the AWS Glue client used to interact with the Glue service + * @param crawlerName the name of the crawler to be retrieved + */ + public static void getSpecificCrawler(GlueClient glueClient, String crawlerName) throws InterruptedException { + try { + GetCrawlerRequest crawlerRequest = GetCrawlerRequest.builder() + .name(crawlerName) + .build(); + + boolean ready = false; + while (!ready) { + GetCrawlerResponse response = glueClient.getCrawler(crawlerRequest); + String status = response.crawler().stateAsString(); + if (status.compareTo("READY") == 0) { + ready = true; + } + Thread.sleep(3000); + } + + System.out.println("The crawler is now ready"); + + } catch (GlueException | InterruptedException e) { + throw e; + } + } + // snippet-end:[glue.java2.get_crawler.main] + + // snippet-start:[glue.java2.start_crawler.main] + /** + * Starts a specific AWS Glue crawler. + * + * @param glueClient the AWS Glue client to use for the crawler operation + * @param crawlerName the name of the crawler to start + * @throws GlueException if there is an error starting the crawler + */ + public static void startSpecificCrawler(GlueClient glueClient, String crawlerName) { + try { + StartCrawlerRequest crawlerRequest = StartCrawlerRequest.builder() + .name(crawlerName) + .build(); + + glueClient.startCrawler(crawlerRequest); + System.out.println(crawlerName + " was successfully started!"); + + } catch (GlueException e) { + throw e; + } + } + // snippet-end:[glue.java2.start_crawler.main] + + // snippet-start:[glue.java2.get_database.main] + /** + * Retrieves the specific database from the AWS Glue service. + * + * @param glueClient an instance of the AWS Glue client used to interact with the service + * @param databaseName the name of the database to retrieve + * @throws GlueException if there is an error retrieving the database from the AWS Glue service + */ + public static void getSpecificDatabase(GlueClient glueClient, String databaseName) { + try { + GetDatabaseRequest databasesRequest = GetDatabaseRequest.builder() + .name(databaseName) + .build(); + + GetDatabaseResponse response = glueClient.getDatabase(databasesRequest); + Instant createDate = response.database().createTime(); + + // Convert the Instant to readable date. + DateTimeFormatter formatter = DateTimeFormatter.ofLocalizedDateTime(FormatStyle.SHORT) + .withLocale(Locale.US) + .withZone(ZoneId.systemDefault()); + + formatter.format(createDate); + System.out.println("The create date of the database is " + createDate); + + } catch (GlueException e) { + throw e; + } + } + // snippet-end:[glue.java2.get_database.main] + + // snippet-start:[glue.java2.get_table.main] + + /** + * Retrieves the names of the tables in the specified Glue database. + * + * @param glueClient the Glue client to use for the operation + * @param dbName the name of the Glue database to retrieve the table names from + * @return the name of the first table retrieved, or an empty string if no tables were found + */ + public static String getGlueTables(GlueClient glueClient, String dbName) { + String myTableName = ""; + try { + GetTablesRequest tableRequest = GetTablesRequest.builder() + .databaseName(dbName) + .build(); + + GetTablesResponse response = glueClient.getTables(tableRequest); + List

tables = response.tableList(); + if (tables.isEmpty()) { + System.out.println("No tables were returned"); + } else { + for (Table table : tables) { + myTableName = table.name(); + System.out.println("Table name is: " + myTableName); + } + } + + } catch (GlueException e) { + throw e; + } + return myTableName; + } + // snippet-end:[glue.java2.get_table.main] + + // snippet-start:[glue.java2.start.job.main] + + /** + * Starts a job run in AWS Glue. + * + * @param glueClient the AWS Glue client to use for the job run + * @param jobName the name of the Glue job to run + * @param inputDatabase the name of the input database + * @param inputTable the name of the input table + * @param outBucket the URL of the output S3 bucket + * @throws GlueException if there is an error starting the job run + */ + public static void startJob(GlueClient glueClient, String jobName, String inputDatabase, String inputTable, + String outBucket) { + try { + Map myMap = new HashMap<>(); + myMap.put("--input_database", inputDatabase); + myMap.put("--input_table", inputTable); + myMap.put("--output_bucket_url", outBucket); + + StartJobRunRequest runRequest = StartJobRunRequest.builder() + .workerType(WorkerType.G_1_X) + .numberOfWorkers(10) + .arguments(myMap) + .jobName(jobName) + .build(); + + StartJobRunResponse response = glueClient.startJobRun(runRequest); + System.out.println("The request Id of the job is " + response.responseMetadata().requestId()); + + } catch (GlueException e) { + throw e; + } + } + // snippet-end:[glue.java2.start.job.main] + + // snippet-start:[glue.java2.create_job.main] + + /** + * Creates a new AWS Glue job. + * + * @param glueClient the AWS Glue client to use for the operation + * @param jobName the name of the job to create + * @param iam the IAM role to associate with the job + * @param scriptLocation the location of the script to be used by the job + * @throws GlueException if there is an error creating the job + */ + public static void createJob(GlueClient glueClient, String jobName, String iam, String scriptLocation) { + try { + JobCommand command = JobCommand.builder() + .pythonVersion("3") + .name("glueetl") + .scriptLocation(scriptLocation) + .build(); + + CreateJobRequest jobRequest = CreateJobRequest.builder() + .description("A Job created by using the AWS SDK for Java V2") + .glueVersion("2.0") + .workerType(WorkerType.G_1_X) + .numberOfWorkers(10) + .name(jobName) + .role(iam) + .command(command) + .build(); + + glueClient.createJob(jobRequest); + System.out.println(jobName + " was successfully created."); + + } catch (GlueException e) { + throw e; + } + } + // snippet-end:[glue.java2.create_job.main] + + // snippet-start:[glue.java2.get_jobs.main] + + /** + * Retrieves and prints information about all the jobs in the Glue data catalog. + * + * @param glueClient the Glue client used to interact with the AWS Glue service + */ + public static void getAllJobs(GlueClient glueClient) { + try { + GetJobsRequest jobsRequest = GetJobsRequest.builder() + .maxResults(10) + .build(); + + GetJobsResponse jobsResponse = glueClient.getJobs(jobsRequest); + List jobs = jobsResponse.jobs(); + for (Job job : jobs) { + System.out.println("Job name is : " + job.name()); + System.out.println("The job worker type is : " + job.workerType().name()); + } + + } catch (GlueException e) { + throw e; + } + } + // snippet-end:[glue.java2.get_jobs.main] + + // snippet-start:[glue.java2.get_job.main] + /** + * Retrieves the job runs for a given Glue job and prints the status of the job runs. + * + * @param glueClient the Glue client used to make API calls + * @param jobName the name of the Glue job to retrieve the job runs for + */ + public static void getJobRuns(GlueClient glueClient, String jobName) { + try { + GetJobRunsRequest runsRequest = GetJobRunsRequest.builder() + .jobName(jobName) + .maxResults(20) + .build(); + + boolean jobDone = false; + while (!jobDone) { + GetJobRunsResponse response = glueClient.getJobRuns(runsRequest); + List jobRuns = response.jobRuns(); + for (JobRun jobRun : jobRuns) { + String jobState = jobRun.jobRunState().name(); + if (jobState.compareTo("SUCCEEDED") == 0) { + System.out.println(jobName + " has succeeded"); + jobDone = true; + + } else if (jobState.compareTo("STOPPED") == 0) { + System.out.println("Job run has stopped"); + jobDone = true; + + } else if (jobState.compareTo("FAILED") == 0) { + System.out.println("Job run has failed"); + jobDone = true; + + } else if (jobState.compareTo("TIMEOUT") == 0) { + System.out.println("Job run has timed out"); + jobDone = true; + + } else { + System.out.println("*** Job run state is " + jobRun.jobRunState().name()); + System.out.println("Job run Id is " + jobRun.id()); + System.out.println("The Glue version is " + jobRun.glueVersion()); + } + TimeUnit.SECONDS.sleep(5); + } + } + + } catch (GlueException e) { + throw e; + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + // snippet-end:[glue.java2.get_job.main] + + // snippet-start:[glue.java2.delete_job.main] + + /** + * Deletes a Glue job. + * + * @param glueClient the Glue client to use for the operation + * @param jobName the name of the job to be deleted + * @throws GlueException if there is an error deleting the job + */ + public static void deleteJob(GlueClient glueClient, String jobName) { + try { + DeleteJobRequest jobRequest = DeleteJobRequest.builder() + .jobName(jobName) + .build(); + + glueClient.deleteJob(jobRequest); + System.out.println(jobName + " was successfully deleted"); + + } catch (GlueException e) { + throw e; + } + } + // snippet-end:[glue.java2.delete_job.main] + + // snippet-start:[glue.java2.delete_database.main] + /** + * Deletes a AWS Glue Database. + * + * @param glueClient An instance of the AWS Glue client used to interact with the AWS Glue service. + * @param databaseName The name of the database to be deleted. + * @throws GlueException If an error occurs while deleting the database. + */ + public static void deleteDatabase(GlueClient glueClient, String databaseName) { + try { + DeleteDatabaseRequest request = DeleteDatabaseRequest.builder() + .name(databaseName) + .build(); + + glueClient.deleteDatabase(request); + System.out.println(databaseName + " was successfully deleted"); + + } catch (GlueException e) { + throw e; + } + } + // snippet-end:[glue.java2.delete_database.main] + + // snippet-start:[glue.java2.delete_crawler.main] + + /** + * Deletes a specific AWS Glue crawler. + * + * @param glueClient the AWS Glue client object + * @param crawlerName the name of the crawler to be deleted + * @throws GlueException if an error occurs during the deletion process + */ + public static void deleteSpecificCrawler(GlueClient glueClient, String crawlerName) { + try { + DeleteCrawlerRequest deleteCrawlerRequest = DeleteCrawlerRequest.builder() + .name(crawlerName) + .build(); + + glueClient.deleteCrawler(deleteCrawlerRequest); + System.out.println(crawlerName + " was deleted"); + + } catch (GlueException e) { + throw e; + } + } + // snippet-end:[glue.java2.delete_crawler.main] + + private static void waitForInputToContinue(Scanner scanner) { + while (true) { + System.out.println(""); + System.out.println("Enter 'c' followed by to continue:"); + String input = scanner.nextLine(); + + if (input.trim().equalsIgnoreCase("c")) { + System.out.println("Continuing with the program..."); + System.out.println(""); + break; + } else { + // Handle invalid input. + System.out.println("Invalid input. Please try again."); + } + } + } +} // snippet-end:[glue.java2.scenario.main] \ No newline at end of file diff --git a/javav2/example_code/glue/src/main/resources/flight_etl_job_script.py b/javav2/example_code/glue/src/main/resources/flight_etl_job_script.py new file mode 100644 index 00000000000..e32e60c31e0 --- /dev/null +++ b/javav2/example_code/glue/src/main/resources/flight_etl_job_script.py @@ -0,0 +1,79 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +""" +This script is used by the AWS Glue _getting started with crawlers and jobs_ scenario to +perform extract, transform, and load (ETL) operations on sample flight data. +As part of the example, it is uploaded to an Amazon Simple Storage Service (Amazon S3) +bucket so that AWS Glue can access it. +""" + +# pylint: disable=undefined-variable + +import sys +from awsglue.transforms import * +from awsglue.utils import getResolvedOptions +from pyspark.context import SparkContext +from awsglue.context import GlueContext +from awsglue.job import Job + +""" +These custom arguments must be passed as Arguments to the StartJobRun request. + --input_database The name of a metadata database that is contained in your + AWS Glue Data Catalog and that contains tables that describe + the data to be processed. + --input_table The name of a table in the database that describes the data to + be processed. + --output_bucket_url An S3 bucket that receives the transformed output data. +""" +args = getResolvedOptions( + sys.argv, ["JOB_NAME", "input_database", "input_table", "output_bucket_url"] +) +sc = SparkContext() +glueContext = GlueContext(sc) +spark = glueContext.spark_session +job = Job(glueContext) +job.init(args["JOB_NAME"], args) + +# Script generated for node S3 Flight Data. +S3FlightData_node1 = glueContext.create_dynamic_frame.from_catalog( + database=args["input_database"], + table_name=args["input_table"], + transformation_ctx="S3FlightData_node1", +) + +# This mapping performs two main functions: +# 1. It simplifies the output by removing most of the fields from the data. +# 2. It renames some fields. For example, `fl_date` is renamed to `flight_date`. +ApplyMapping_node2 = ApplyMapping.apply( + frame=S3FlightData_node1, + mappings=[ + ("year", "long", "year", "long"), + ("month", "long", "month", "tinyint"), + ("day_of_month", "long", "day", "tinyint"), + ("fl_date", "string", "flight_date", "string"), + ("carrier", "string", "carrier", "string"), + ("fl_num", "long", "flight_num", "long"), + ("origin_city_name", "string", "origin_city_name", "string"), + ("origin_state_abr", "string", "origin_state_abr", "string"), + ("dest_city_name", "string", "dest_city_name", "string"), + ("dest_state_abr", "string", "dest_state_abr", "string"), + ("dep_time", "long", "departure_time", "long"), + ("wheels_off", "long", "wheels_off", "long"), + ("wheels_on", "long", "wheels_on", "long"), + ("arr_time", "long", "arrival_time", "long"), + ("mon", "string", "mon", "string"), + ], + transformation_ctx="ApplyMapping_node2", +) + +# Script generated for node Revised Flight Data. +RevisedFlightData_node3 = glueContext.write_dynamic_frame.from_options( + frame=ApplyMapping_node2, + connection_type="s3", + format="json", + connection_options={"path": args["output_bucket_url"], "partitionKeys": []}, + transformation_ctx="RevisedFlightData_node3", +) + +job.commit() diff --git a/javav2/example_code/glue/src/test/java/GlueTest.java b/javav2/example_code/glue/src/test/java/GlueTest.java index 7aacd8f95bc..bb26061c38c 100644 --- a/javav2/example_code/glue/src/test/java/GlueTest.java +++ b/javav2/example_code/glue/src/test/java/GlueTest.java @@ -1,7 +1,7 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -import com.example.glue.*; +import com.example.glue.scenario.GlueScenario; import com.google.gson.Gson; import software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider; import software.amazon.awssdk.services.glue.GlueClient; @@ -10,10 +10,13 @@ import software.amazon.awssdk.services.secretsmanager.SecretsManagerClient; import software.amazon.awssdk.services.secretsmanager.model.GetSecretValueRequest; import software.amazon.awssdk.services.secretsmanager.model.GetSecretValueResponse; + import java.io.*; import java.net.URISyntaxException; import java.util.concurrent.TimeUnit; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; + /** * To run these integration tests, you must set the required values * in the config.properties file or AWS Secrets Manager. @@ -41,11 +44,11 @@ public class GlueTest { private static String bucketNameSc = ""; @BeforeAll - public static void setUp() throws IOException, URISyntaxException { + public static void setUp() { glueClient = GlueClient.builder() - .region(Region.US_EAST_1) - .credentialsProvider(EnvironmentVariableCredentialsProvider.create()) - .build(); + .region(Region.US_EAST_1) + .credentialsProvider(EnvironmentVariableCredentialsProvider.create()) + .build(); // Get the values to run these tests from AWS Secrets Manager. Gson gson = new Gson(); @@ -61,86 +64,148 @@ public static void setUp() throws IOException, URISyntaxException { existingDatabaseName = values.getExistingDatabaseName(); existingCrawlerName = values.getExistingCrawlerName(); jobNameSc = values.getJobNameSc() + java.util.UUID.randomUUID(); - ; s3PathSc = values.getS3PathSc() + java.util.UUID.randomUUID(); - ; dbNameSc = values.getDbNameSc() + java.util.UUID.randomUUID(); crawlerNameSc = values.getCrawlerNameSc() + java.util.UUID.randomUUID(); scriptLocationSc = values.getScriptLocationSc(); locationUri = values.getLocationUri(); bucketNameSc = values.getBucketNameSc(); - - // Uncomment this code block if you prefer using a config.properties file to - // retrieve AWS values required for these tests. - /* - * - * try (InputStream input = - * GlueTest.class.getClassLoader().getResourceAsStream("config.properties")) { - * Properties prop = new Properties(); - * if (input == null) { - * System.out.println("Sorry, unable to find config.properties"); - * return; - * } - * prop.load(input); - * crawlerName = prop.getProperty("crawlerName"); - * s3Path = prop.getProperty("s3Path"); - * cron = prop.getProperty("cron"); - * IAM = prop.getProperty("IAM"); - * databaseName = prop.getProperty("databaseName"); - * tableName = prop.getProperty("tableName"); - * text = prop.getProperty("text"); - * existingDatabaseName = prop.getProperty("existingDatabaseName"); - * existingCrawlerName = prop.getProperty("existingCrawlerName"); - * jobNameSc = prop.getProperty("jobNameSc")+ java.util.UUID.randomUUID();; - * s3PathSc = prop.getProperty("s3PathSc")+ java.util.UUID.randomUUID();; - * dbNameSc = prop.getProperty("dbNameSc")+ java.util.UUID.randomUUID(); - * crawlerNameSc = prop.getProperty("crawlerNameSc")+ - * java.util.UUID.randomUUID(); - * scriptLocationSc = prop.getProperty("scriptLocationSc"); - * locationUri = prop.getProperty("locationUri"); - * bucketNameSc = prop.getProperty("bucketNameSc"); - * - * } catch (IOException ex) { - * ex.printStackTrace(); - * } - */ } @Test @Tag("IntegrationTest") @Order(1) - public void ScenarioTest() throws InterruptedException { - GlueScenario.createDatabase(glueClient, dbNameSc, locationUri); - GlueScenario.createGlueCrawler(glueClient, IAM, s3PathSc, cron, dbNameSc, crawlerNameSc); - GlueScenario.getSpecificCrawler(glueClient, crawlerNameSc); - GlueScenario.startSpecificCrawler(glueClient, crawlerNameSc); - GlueScenario.getSpecificDatabase(glueClient, dbNameSc); - - System.out.println("Wait 5 min for the tables to become available"); - TimeUnit.MINUTES.sleep(5);// Sleep for 5 minute to get tables ready - String myTableName = GlueScenario.getGlueTables(glueClient, dbNameSc); - GlueScenario.createJob(glueClient, jobNameSc, IAM, scriptLocationSc); - GlueScenario.startJob(glueClient, jobNameSc, dbNameSc, myTableName, bucketNameSc); - GlueScenario.getAllJobs(glueClient); - GlueScenario.getJobRuns(glueClient, jobNameSc); - GlueScenario.deleteJob(glueClient, jobNameSc); - System.out.println("*** Wait 5 MIN for the " + crawlerNameSc + " to stop"); - TimeUnit.MINUTES.sleep(5); - GlueScenario.deleteDatabase(glueClient, dbNameSc); - GlueScenario.deleteSpecificCrawler(glueClient, crawlerNameSc); + void testCreateDatabase() { + assertDoesNotThrow(() -> { + GlueScenario.createDatabase(glueClient, dbNameSc, locationUri); + }); + } + + @Test + @Tag("IntegrationTest") + @Order(2) + void testCreateGlueCrawler() { + assertDoesNotThrow(() -> { + GlueScenario.createGlueCrawler(glueClient, IAM, s3PathSc, cron, dbNameSc, crawlerNameSc); + }); + } + + @Test + @Tag("IntegrationTest") + @Order(3) + void testGetSpecificCrawler() { + assertDoesNotThrow(() -> { + GlueScenario.getSpecificCrawler(glueClient, crawlerNameSc); + }); + } + + @Test + @Tag("IntegrationTest") + @Order(4) + void testStartSpecificCrawler() { + assertDoesNotThrow(() -> { + GlueScenario.startSpecificCrawler(glueClient, crawlerNameSc); + }); + } + + @Test + @Tag("IntegrationTest") + @Order(5) + void testGetSpecificDatabase() { + assertDoesNotThrow(() -> { + GlueScenario.getSpecificDatabase(glueClient, dbNameSc); + }); + } + + @Test + @Tag("IntegrationTest") + @Order(6) + void testGetTable() { + assertDoesNotThrow(() -> { + System.out.println("*** Wait 5 min for the tables to become available"); + TimeUnit.MINUTES.sleep(5); + System.out.println("6. Get tables."); + GlueScenario.getGlueTables(glueClient, dbNameSc); + }); + } + + @Test + @Tag("IntegrationTest") + @Order(7) + void testCreateJob() { + assertDoesNotThrow(() -> { + GlueScenario.createJob(glueClient, jobNameSc, IAM, scriptLocationSc); + }); + } + + @Test + @Tag("IntegrationTest") + @Order(8) + void testStartJob() { + assertDoesNotThrow(() -> { + GlueScenario.startJob(glueClient, jobNameSc, dbNameSc, tableName, bucketNameSc); + }); + } + + @Test + @Tag("IntegrationTest") + @Order(9) + void testGetJobs() { + assertDoesNotThrow(() -> { + GlueScenario.getAllJobs(glueClient); + }); + } + + @Test + @Tag("IntegrationTest") + @Order(10) + void testRunJobs() { + assertDoesNotThrow(() -> { + GlueScenario.getJobRuns(glueClient, jobNameSc); + }); + } + + @Test + @Tag("IntegrationTest") + @Order(11) + void testDeleteJob() { + assertDoesNotThrow(() -> { + GlueScenario.deleteJob(glueClient, jobNameSc); + }); + } + + @Test + @Tag("IntegrationTest") + @Order(12) + void testDeleteDB() { + assertDoesNotThrow(() -> { + System.out.println("*** Wait 5 MIN for the " + crawlerNameSc + " to stop"); + TimeUnit.MINUTES.sleep(5); + GlueScenario.deleteDatabase(glueClient, dbNameSc); + }); + } + + @Test + @Tag("IntegrationTest") + @Order(13) + void testDelCrawler() { + assertDoesNotThrow(() -> { + System.out.println("*** Wait 5 MIN for the " + crawlerNameSc + " to stop"); + TimeUnit.MINUTES.sleep(5); + GlueScenario.deleteSpecificCrawler(glueClient, crawlerNameSc); + }); } private static String getSecretValues() { - // Get the Amazon RDS creds from Secrets Manager. SecretsManagerClient secretClient = SecretsManagerClient.builder() - .region(Region.US_EAST_1) - .credentialsProvider(EnvironmentVariableCredentialsProvider.create()) - .build(); + .region(Region.US_EAST_1) + .credentialsProvider(EnvironmentVariableCredentialsProvider.create()) + .build(); String secretName = "test/glue"; GetSecretValueRequest valueRequest = GetSecretValueRequest.builder() - .secretId(secretName) - .build(); + .secretId(secretName) + .build(); GetSecretValueResponse valueResponse = secretClient.getSecretValue(valueRequest); return valueResponse.secretString();