diff --git a/.gitignore b/.gitignore index 2a1a6f4..e83c143 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ Cargo.lock terraform.tfvars .terraform* terraform.tfstate* +.env diff --git a/Cargo.toml b/Cargo.toml index 0c3d31c..a1b7ed5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,7 +19,7 @@ path = "src/bin/lambda.rs" [dependencies] anyhow = "*" chrono = "*" -deltalake = { version = "0.16.3", features = ["s3", "json"]} +deltalake = { version = "0.16.4", features = ["s3", "json"]} futures = "*" tracing = { version = "0.1", features = ["log"] } tracing-subscriber = { version = "0.3", default-features = false, features = ["fmt", "env-filter", "tracing-log"] } diff --git a/deployment.tf b/deployment.tf index 13e7b2c..1525296 100644 --- a/deployment.tf +++ b/deployment.tf @@ -4,7 +4,7 @@ resource "aws_lambda_function" "oxbow" { description = "A simple lambda for converting parquet files to delta tables" filename = "target/lambda/oxbow-lambda/bootstrap.zip" - function_name = "oxbow-conversion" + function_name = "oxbow-delta-lake-conversion" role = aws_iam_role.iam_for_lambda.arn handler = "provided" runtime = "provided.al2" @@ -26,6 +26,15 @@ resource "aws_lambda_event_source_mapping" "oxbow-trigger" { resource "aws_sqs_queue" "oxbow" { name = "oxbow-notification-queue" policy = data.aws_iam_policy_document.queue.json + + redrive_policy = jsonencode({ + deadLetterTargetArn = aws_sqs_queue.oxbow_dlq.arn + maxReceiveCount = 8 + }) +} + +resource "aws_sqs_queue" "oxbow_dlq" { + name = "obxow-notification-dlq" } resource "aws_s3_bucket" "parquets" { @@ -135,11 +144,7 @@ resource "aws_dynamodb_table" "oxbow_locking" { name = "key" type = "S" } - # The leaseDuration is used by dynamodb-lock-rs and *must* be a Number type - attribute { - name = "leaseDuration" - type = "N" - } + ttl { attribute_name = "leaseDuration" enabled = true