WIP

lambdas/sqs-ingest/.gitignore

@@ -0,0 +1 @@
+/target

lambdas/sqs-ingest/Cargo.toml

@@ -0,0 +1,22 @@
+[package]
+name = "sqs-ingest"
+version = "0.1.0"
+edition = "2021"
+
+# Starting in Rust 1.62 you can use `cargo add` to add dependencies 
+# to your project.
+#
+# If you're using an older Rust version,
+# download cargo-edit(https://github.com/killercup/cargo-edit#installation) 
+# to install the `add` subcommand.
+#
+# Running `cargo add DEPENDENCY_NAME` will
+# add the latest version of a dependency to the list,
+# and it will keep the alphabetic ordering for you.
+
+[dependencies]
+aws_lambda_events = { version = "0.15.0", default-features = false, features = ["sqs"] }
+
+lambda_runtime = "0.11.1"
+tokio = { version = "1", features = ["macros"] }
+

lambdas/sqs-ingest/README.adoc

@@ -0,0 +1,56 @@
+ifdef::env-github[]
+:tip-caption: :bulb:
+:note-caption: :information_source:
+:important-caption: :heavy_exclamation_mark:
+:caution-caption: :fire:
+:warning-caption: :warning:
+endif::[]
+:toc: macro
+
+= SQS Ingest
+
+The `sqs-ingest` is the AWS SQS equivalent of
+link:https://github.com/delta-io/kafka-delta-ingest[kafka-delta-ingest] insofar
+that it allows the ingestion of JSON data from SQS and translatest each record
+into a row appended onto a link:https://delta.io[Delta Lake] table.
+
+toc::[]
+
+
+for simplicity's sake this Lambda is intended to only work with a single Delta
+table. For ingestion to multiple Delta tables, deploy different instances of
+this Lambda triggered by different SQS queues.
+
+[dot]
+----
+digraph {
+  label="SQS Ingest";
+  labelloc=t;
+
+  sqs -> lambda -> delta;
+}
+----
+
+== Environment Variables
+
+|===
+
+| Name | Default Value | Notes
+
+| `RUST_LOG`
+| `error`
+| Set the log level, e.g. `info`, `warn`, `error`. Can be scoped to specific modules, i.e. `oxbow=debug`
+
+| `DELTA_TABLE_URI`
+|
+| Set to the `s3://` URL of the Delta table which should be appended
+
+| `AWS_S3_LOCKING_PROVIDER`
+|
+| Set to `dynamodb` to enable safe concurrent writes to the table
+
+| `DYNAMO_LOCK_TABLE_NAME`
+|
+| Set to the DynamoDB table used for locking, required for safe concurrent writes.
+
+|===

lambdas/sqs-ingest/src/main.rs

@@ -0,0 +1,21 @@
+use aws_lambda_events::event::sqs::SqsEvent;
+use lambda_runtime::{run, service_fn, tracing, Error, LambdaEvent};
+
+
+/// This is the main body for the function.
+/// Write your code inside it.
+/// There are some code example in the following URLs:
+/// - https://github.com/awslabs/aws-lambda-rust-runtime/tree/main/examples
+/// - https://github.com/aws-samples/serverless-rust-demo/
+async fn function_handler(event: LambdaEvent<SqsEvent>) -> Result<(), Error> {
+    // Extract some useful information from the request
+
+    Ok(())
+}
+
+#[tokio::main]
+async fn main() -> Result<(), Error> {
+    tracing::init_default_subscriber();
+
+    run(service_fn(function_handler)).await
+}