diff --git a/examples/pom.xml b/examples/pom.xml index 511cdd5a9..1fdbd85aa 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -45,6 +45,7 @@ powertools-examples-batch powertools-examples-validation powertools-examples-cloudformation + powertools-examples-large-messages diff --git a/examples/powertools-examples-large-messages/README.md b/examples/powertools-examples-large-messages/README.md new file mode 100644 index 000000000..ba0af5f49 --- /dev/null +++ b/examples/powertools-examples-large-messages/README.md @@ -0,0 +1,43 @@ +# Powertools for AWS Lambda (Java) - Large Messages Example + +This project contains an example of a Lambda function using the **Large Messages** module of Powertools for AWS Lambda (Java). For more information on this module, please refer to the [documentation](https://docs.aws.amazon.com/powertools/java/latest/utilities/large_messages/). + +The example demonstrates an SQS listener that processes messages using the `LargeMessages` functional utility. It handles the retrieval of large payloads offloaded to S3 automatically. + +## Deploy the sample application + +This sample is based on Serverless Application Model (SAM). To deploy it, check out the instructions for getting +started with SAM in [the examples directory](../README.md). + +## Test the application + +Since this function is triggered by an SQS Queue, you can test it by sending a message to the queue created by the SAM template. + +1. **Find your Queue URL:** + Run the following command (replacing `LargeMessageExample` with the name of your deployed stack): + ```bash + aws cloudformation describe-stacks --stack-name LargeMessageExample --query "Stacks[0].Outputs[?OutputKey=='QueueURL'].OutputValue" --output text + ``` +2. **Send a Test Message:** + Note: To test the actual "Large Message" functionality (payload offloading), you would typically use the SQS Extended Client in a producer application. However, you can verify the Lambda trigger with a standard message: + ```bash + aws sqs send-message --queue-url [YOUR_QUEUE_URL] --message-body '{"message": "Hello from CLI"}' + ``` +3. **Verify Logs:** + Go to AWS CloudWatch Logs and check the Log Group for your function. You should see the processed message logged by the application. + +### Run the Large Message Producer + +To test the handling of large messages involving S3 offloading, verify you have the SQS Queue URL and the S3 Bucket name created by the stack. + +1. **Build the producer tool:** + ```bash + cd examples/powertools-examples-large-messages/tools + mvn compile + ``` + +2. **Run the producer:** + ```bash + mvn exec:java -Dexec.args=" " + ``` + Replace `` and `` with the output values from the deployment. \ No newline at end of file diff --git a/examples/powertools-examples-large-messages/pom.xml b/examples/powertools-examples-large-messages/pom.xml new file mode 100644 index 000000000..31e75fefc --- /dev/null +++ b/examples/powertools-examples-large-messages/pom.xml @@ -0,0 +1,100 @@ + + 4.0.0 + + software.amazon.lambda.examples + powertools-examples-large-messages + 2.8.0 + Powertools for AWS Lambda (Java) - Examples - Large Messages + + + + software.amazon.lambda + powertools-large-messages + ${project.version} + + + + com.amazonaws + aws-lambda-java-events + 3.11.4 + + + + software.amazon.lambda + powertools-logging-log4j + ${project.version} + + + + org.aspectj + aspectjrt + ${aspectj.version} + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + 3.1.4 + + true + + + + + dev.aspectj + aspectj-maven-plugin + 1.14 + + ${maven.compiler.source} + ${maven.compiler.target} + ${maven.compiler.target} + + + software.amazon.lambda + powertools-logging + + + + + + + compile + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 3.6.1 + + + package + + shade + + + false + + + + + + + + + org.apache.logging.log4j + log4j-transform-maven-shade-plugin-extensions + 0.1.0 + + + + + + \ No newline at end of file diff --git a/examples/powertools-examples-large-messages/src/main/java/helloworld/App.java b/examples/powertools-examples-large-messages/src/main/java/helloworld/App.java new file mode 100644 index 000000000..e76bd2b03 --- /dev/null +++ b/examples/powertools-examples-large-messages/src/main/java/helloworld/App.java @@ -0,0 +1,49 @@ +/* + * Copyright 2023 Amazon.com, Inc. or its affiliates. + * Licensed under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package helloworld; + +import com.amazonaws.services.lambda.runtime.Context; +import com.amazonaws.services.lambda.runtime.RequestHandler; +import com.amazonaws.services.lambda.runtime.events.SQSEvent; +import com.amazonaws.services.lambda.runtime.events.SQSEvent.SQSMessage; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.lambda.powertools.largemessages.LargeMessages; +import software.amazon.lambda.powertools.logging.Logging; + +/** + * Example handler showing how to use LargeMessageProcessor functionally. + * This approach gives you more control than the @LargeMessage annotation. + */ +public final class App implements RequestHandler { + + private static final Logger LOG = LoggerFactory.getLogger(App.class); + + @Logging + @Override + public String handleRequest(final SQSEvent event, final Context context) { + LOG.info("Received event with {} records", event.getRecords().size()); + + for (SQSMessage message : event.getRecords()) { + LargeMessages.processLargeMessage(message, (processedMessage) -> { + LOG.info("Processing message ID: {}", processedMessage.getMessageId()); + LOG.info("Processing body content: {}", processedMessage.getBody()); + return "Processed"; + }); + } + + return "SUCCESS"; + } +} diff --git a/examples/powertools-examples-large-messages/src/main/resources/log4j2.xml b/examples/powertools-examples-large-messages/src/main/resources/log4j2.xml new file mode 100644 index 000000000..5dede7b58 --- /dev/null +++ b/examples/powertools-examples-large-messages/src/main/resources/log4j2.xml @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + diff --git a/examples/powertools-examples-large-messages/template.yaml b/examples/powertools-examples-large-messages/template.yaml new file mode 100644 index 000000000..4fc3368a6 --- /dev/null +++ b/examples/powertools-examples-large-messages/template.yaml @@ -0,0 +1,62 @@ +AWSTemplateFormatVersion: "2010-09-09" +Transform: AWS::Serverless-2016-10-31 +Description: > + Large Message Handling Example using SQS and S3 offloading + +Globals: + Function: + Timeout: 30 + Runtime: java11 + MemorySize: 512 + Tracing: Active + Environment: + Variables: + JAVA_TOOL_OPTIONS: "-XX:+TieredCompilation -XX:TieredStopAtLevel=1" + POWERTOOLS_LOG_LEVEL: INFO + POWERTOOLS_SERVICE_NAME: LargeMessageExample + +Resources: + LargeMessageBucket: + Type: AWS::S3::Bucket + Properties: + LifecycleConfiguration: + Rules: + - Id: DeleteOldMessages + Status: Enabled + ExpirationInDays: 1 + MyLargeMessageQueue: + Type: AWS::SQS::Queue + Properties: + VisibilityTimeout: 30 + LargeMessageProcessingFunction: + Type: AWS::Serverless::Function + Properties: + CodeUri: . + Handler: helloworld.App::handleRequest + + Policies: + - S3CrudPolicy: + BucketName: !Ref LargeMessageBucket + - SQSPollerPolicy: + QueueName: !GetAtt MyLargeMessageQueue.QueueName + + Environment: + Variables: + POWERTOOLS_LARGE_MESSAGES_BUCKET: !Ref LargeMessageBucket + + Events: + SQSEvent: + Type: SQS + Properties: + Queue: !GetAtt MyLargeMessageQueue.Arn + BatchSize: 1 +Outputs: + LargeMessageBucketName: + Description: "S3 Bucket for large payloads" + Value: !Ref LargeMessageBucket + QueueURL: + Description: "SQS Queue URL" + Value: !Ref MyLargeMessageQueue + FunctionArn: + Description: "Lambda Function ARN" + Value: !GetAtt LargeMessageProcessingFunction.Arn diff --git a/examples/powertools-examples-large-messages/tools/pom.xml b/examples/powertools-examples-large-messages/tools/pom.xml new file mode 100644 index 000000000..61b496791 --- /dev/null +++ b/examples/powertools-examples-large-messages/tools/pom.xml @@ -0,0 +1,64 @@ + + + 4.0.0 + + software.amazon.lambda.examples + powertools-examples-large-messages-tools + 1.0.0 + + + 11 + 11 + 2.29.0 + 2.1.2 + + + + + + software.amazon.awssdk + bom + ${aws.sdk.version} + pom + import + + + + + + + com.amazonaws + amazon-sqs-java-extended-client-lib + ${sqs-extended.version} + + + software.amazon.awssdk + s3 + + + software.amazon.awssdk + sqs + + + org.slf4j + slf4j-simple + 2.0.16 + + + + + + + org.codehaus.mojo + exec-maven-plugin + 3.1.0 + + software.amazon.lambda.powertools.largemessages.tools.LargeMessageProducer + compile + + + + + diff --git a/examples/powertools-examples-large-messages/tools/src/main/java/software/amazon/lambda/powertools/largemessages/tools/LargeMessageProducer.java b/examples/powertools-examples-large-messages/tools/src/main/java/software/amazon/lambda/powertools/largemessages/tools/LargeMessageProducer.java new file mode 100644 index 000000000..2037e46d2 --- /dev/null +++ b/examples/powertools-examples-large-messages/tools/src/main/java/software/amazon/lambda/powertools/largemessages/tools/LargeMessageProducer.java @@ -0,0 +1,77 @@ +/* + * Copyright 2023 Amazon.com, Inc. or its affiliates. + * Licensed under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package software.amazon.lambda.powertools.largemessages.tools; + +import com.amazon.sqs.javamessaging.AmazonSQSExtendedClient; +import com.amazon.sqs.javamessaging.ExtendedClientConfiguration; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.sqs.SqsClient; +import software.amazon.awssdk.services.sqs.model.SendMessageRequest; +import software.amazon.awssdk.services.sqs.model.SendMessageResponse; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Arrays; + +public class LargeMessageProducer { + private static final Logger LOG = LoggerFactory.getLogger(LargeMessageProducer.class); + + public static void main(String[] args) { + if (args.length < 2) { + LOG.error("Usage: LargeMessageProducer "); + System.exit(1); + } + + String queueUrl = args[0]; + String bucketName = args[1]; + + LOG.info("Starting Large Message Producer..."); + LOG.info("Queue URL: {}", queueUrl); + LOG.info("S3 Bucket: {}", bucketName); + + // Initialize S3 and SQS clients + S3Client s3Client = S3Client.create(); + SqsClient sqsClient = SqsClient.create(); + + // Configure Extended Client + ExtendedClientConfiguration extendedClientConfig = new ExtendedClientConfiguration() + .withPayloadSupportEnabled(s3Client, bucketName); + + SqsClient extendedSqsClient = new AmazonSQSExtendedClient(sqsClient, extendedClientConfig); + + // Generate large payload (> 256KB). 300KB to be safe. + String payload = generateLargePayload(300 * 1024); + + try { + SendMessageRequest request = SendMessageRequest.builder() + .queueUrl(queueUrl) + .messageBody(payload) + .build(); + + SendMessageResponse response = extendedSqsClient.sendMessage(request); + LOG.info("Message sent successfully. Message ID: {}", response.messageId()); + } catch (Exception e) { + LOG.error("Failed to send message", e); + System.exit(1); + } + } + + private static String generateLargePayload(int sizeInBytes) { + char[] chars = new char[sizeInBytes]; + Arrays.fill(chars, 'A'); + // create a simple JSON structure + return String.format("{\"data\": \"%s\"}", new String(chars)); + } +}