buoyant-data
/
oxbow
1
0
Fork 0
Code Pull Requests Releases Activity

Upgrade to the latest deltalake 0l16.4 which includes TTL fixes for locks 

This change also includes some terraform changes I made while doing some
acceptance testing here
This commit is contained in:
R Tyler Croy 2023-11-12 13:44:24 -08:00
parent acaea5c05c
commit 3cbce25f2f
3 changed files with 13 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -5,3 +5,4 @@ Cargo.lock
terraform.tfvars terraform.tfvars
.terraform* .terraform*
terraform.tfstate* terraform.tfstate*
.env

2
Cargo.toml
View File

@ -19,7 +19,7 @@ path = "src/bin/lambda.rs"
[dependencies] [dependencies]
anyhow = "*" anyhow = "*"
chrono = "*" chrono = "*"
deltalake = { version = "0.16.3", features = ["s3", "json"]} deltalake = { version = "0.16.4", features = ["s3", "json"]}
futures = "*" futures = "*"
tracing = { version = "0.1", features = ["log"] } tracing = { version = "0.1", features = ["log"] }
tracing-subscriber = { version = "0.3", default-features = false, features = ["fmt", "env-filter", "tracing-log"] } tracing-subscriber = { version = "0.3", default-features = false, features = ["fmt", "env-filter", "tracing-log"] }

17
deployment.tf
View File

@ -4,7 +4,7 @@
resource "aws_lambda_function" "oxbow" { resource "aws_lambda_function" "oxbow" {
description = "A simple lambda for converting parquet files to delta tables" description = "A simple lambda for converting parquet files to delta tables"
filename = "target/lambda/oxbow-lambda/bootstrap.zip" filename = "target/lambda/oxbow-lambda/bootstrap.zip"
function_name = "oxbow-conversion" function_name = "oxbow-delta-lake-conversion"
role = aws_iam_role.iam_for_lambda.arn role = aws_iam_role.iam_for_lambda.arn
handler = "provided" handler = "provided"
runtime = "provided.al2" runtime = "provided.al2"
@ -26,6 +26,15 @@ resource "aws_lambda_event_source_mapping" "oxbow-trigger" {
resource "aws_sqs_queue" "oxbow" { resource "aws_sqs_queue" "oxbow" {
name = "oxbow-notification-queue" name = "oxbow-notification-queue"
policy = data.aws_iam_policy_document.queue.json policy = data.aws_iam_policy_document.queue.json
redrive_policy = jsonencode({
deadLetterTargetArn = aws_sqs_queue.oxbow_dlq.arn
maxReceiveCount = 8
})
}
resource "aws_sqs_queue" "oxbow_dlq" {
name = "obxow-notification-dlq"
} }
resource "aws_s3_bucket" "parquets" { resource "aws_s3_bucket" "parquets" {
@ -135,11 +144,7 @@ resource "aws_dynamodb_table" "oxbow_locking" {
name = "key" name = "key"
type = "S" type = "S"
} }
# The leaseDuration is used by dynamodb-lock-rs and *must* be a Number type
attribute {
name = "leaseDuration"
type = "N"
}
ttl { ttl {
attribute_name = "leaseDuration" attribute_name = "leaseDuration"
enabled = true enabled = true