From 2667088175a5c2af949ec9f9b927413e40bca5c4 Mon Sep 17 00:00:00 2001 From: "R. Tyler Croy" Date: Sat, 12 Jun 2021 15:21:09 -0700 Subject: [PATCH] Properly build with the deltalake dependency and my features branch --- Cargo.lock | 216 +++++++++++++++++++++++++++++++++++++++++++++++++++- Cargo.toml | 2 +- src/main.rs | 10 +-- 3 files changed, 219 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f0f5431..4806231 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -279,12 +279,44 @@ dependencies = [ "ansi_term", "atty", "bitflags", - "strsim", - "textwrap", + "strsim 0.8.0", + "textwrap 0.11.0", "unicode-width", "vec_map", ] +[[package]] +name = "clap" +version = "3.0.0-beta.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bd1061998a501ee7d4b6d449020df3266ca3124b941ec56cf2005c3779ca142" +dependencies = [ + "atty", + "bitflags", + "clap_derive", + "indexmap", + "lazy_static", + "os_str_bytes", + "strsim 0.10.0", + "termcolor", + "textwrap 0.12.1", + "unicode-width", + "vec_map", +] + +[[package]] +name = "clap_derive" +version = "3.0.0-beta.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "370f715b81112975b1b69db93e0b56ea4cd4e5002ac43b2da8474106a54096a1" +dependencies = [ + "heck", + "proc-macro-error", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "const_fn" version = "0.4.8" @@ -373,7 +405,8 @@ dependencies = [ "anyhow", "arrow", "aws_lambda_events", - "clap", + "clap 2.33.3", + "deltalake", "dotenv", "lambda_runtime", "log", @@ -391,6 +424,42 @@ dependencies = [ "tokio", ] +[[package]] +name = "deltalake" +version = "0.4.0" +source = "git+https://github.com/rtyler/delta-rs?branch=features-for-rustls#b8cbe8e7007cd5ef3970dd2e0b98d517120356ad" +dependencies = [ + "anyhow", + "arrow", + "async-trait", + "bytes 1.0.1", + "cfg-if", + "chrono", + "clap 3.0.0-beta.2", + "env_logger", + "errno", + "futures", + "glibc_version", + "lazy_static", + "libc", + "log", + "maplit", + "parquet", + "parquet-format", + "regex", + "rusoto_core", + "rusoto_credential", + "rusoto_dynamodb", + "rusoto_s3", + "rusoto_sts", + "serde", + "serde_json", + "thiserror", + "tokio", + "tokio-stream", + "uuid", +] + [[package]] name = "digest" version = "0.9.0" @@ -452,6 +521,27 @@ dependencies = [ "termcolor", ] +[[package]] +name = "errno" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa68f2fb9cae9d37c9b2b3584aba698a2e97f72d7aef7b9f7aa71d8b54ce46fe" +dependencies = [ + "errno-dragonfly", + "libc", + "winapi", +] + +[[package]] +name = "errno-dragonfly" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14ca354e36190500e1e1fb267c647932382b54053c50b14970856c0b00a35067" +dependencies = [ + "gcc", + "libc", +] + [[package]] name = "flatbuffers" version = "0.8.4" @@ -585,6 +675,12 @@ dependencies = [ "slab", ] +[[package]] +name = "gcc" +version = "0.3.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f5f3913fa0bfe7ee1fd8248b6b9f42a5af4b9d65ec2dd2c3c26132b950ecfc2" + [[package]] name = "generic-array" version = "0.14.4" @@ -617,12 +713,30 @@ dependencies = [ "wasi 0.10.2+wasi-snapshot-preview1", ] +[[package]] +name = "glibc_version" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "803ff7635f1ab4e2c064b68a0c60da917d3d18dc8d086130f689d62ce4f1c33e" +dependencies = [ + "regex", +] + [[package]] name = "hashbrown" version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04" +[[package]] +name = "heck" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "hermit-abi" version = "0.1.18" @@ -872,6 +986,12 @@ dependencies = [ "libc", ] +[[package]] +name = "maplit" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" + [[package]] name = "matches" version = "0.1.8" @@ -1064,6 +1184,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "os_str_bytes" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afb2e1c3ee07430c2cf76151675e583e0f19985fa6efae47d6848a3e2c824f85" + [[package]] name = "parquet" version = "4.2.0" @@ -1127,6 +1253,30 @@ dependencies = [ "log", ] +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", +] + [[package]] name = "proc-macro-hack" version = "0.5.19" @@ -1304,6 +1454,20 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rusoto_dynamodb" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f26af40f36409cb8fae3069690f78f638f747b55c7b90f338d5ed36016b0cda" +dependencies = [ + "async-trait", + "bytes 1.0.1", + "futures", + "rusoto_core", + "serde", + "serde_json", +] + [[package]] name = "rusoto_s3" version = "0.46.0" @@ -1358,6 +1522,21 @@ dependencies = [ "xml-rs", ] +[[package]] +name = "rusoto_sts" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f93005e0c3b9e40a424b50ca71886d2445cc19bb6cdac3ac84c2daff482eb59" +dependencies = [ + "async-trait", + "bytes 1.0.1", + "chrono", + "futures", + "rusoto_core", + "serde_urlencoded", + "xml-rs", +] + [[package]] name = "rustc_version" version = "0.2.3" @@ -1669,6 +1848,12 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + [[package]] name = "subtle" version = "2.4.0" @@ -1704,6 +1889,15 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "textwrap" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "203008d98caf094106cfaba70acfed15e18ed3ddb7d94e49baec153a2b462789" +dependencies = [ + "unicode-width", +] + [[package]] name = "thiserror" version = "1.0.25" @@ -1960,6 +2154,12 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-segmentation" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0d2e7be6ae3a5fa87eed5fb451aff96f2573d2694942e40543ae0bbe19c796" + [[package]] name = "unicode-width" version = "0.1.8" @@ -1990,6 +2190,16 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "uuid" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7" +dependencies = [ + "getrandom 0.2.3", + "serde", +] + [[package]] name = "vec_map" version = "0.8.2" diff --git a/Cargo.toml b/Cargo.toml index 30024cc..61e6455 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,7 +11,7 @@ anyhow = "*" aws_lambda_events = "*" arrow = "4" clap = "2" -#deltalake = { git = "https://github.com/rtyler/delta-rs", branch = "main", features = ["s3"] } +deltalake = { git = "https://github.com/rtyler/delta-rs", branch = "features-for-rustls", features = ["s3-rustls"] } dotenv = "0.15" log = "0.4" parquet = "4" diff --git a/src/main.rs b/src/main.rs index 4031bb6..38b3602 100644 --- a/src/main.rs +++ b/src/main.rs @@ -73,13 +73,15 @@ async fn main() -> Result<(), anyhow::Error> { loop { let rcv = ReceiveMessageRequest { queue_url: queue.to_string(), + // Always consuming the max number of available messages to reduce round trips max_number_of_messages: Some(10), - wait_time_seconds: Some(5), + // This wait time is being used as the effective loop interval for the main runloop + wait_time_seconds: Some(10), ..Default::default() }; let res = client.receive_message(rcv).await?; - debug!("Message(s) received: {:?}", res); + trace!("Message(s) received: {:?}", res); if let Some(messages) = &res.messages { for message in messages { @@ -93,7 +95,7 @@ async fn main() -> Result<(), anyhow::Error> { } } } else { - debug!("Message had no body: {:?}", &message); + warn!("Message had no body: {:?}", &message); } if let Some(receipt) = &message.receipt_handle { @@ -108,12 +110,10 @@ async fn main() -> Result<(), anyhow::Error> { } } } - break; } } else { panic!("When running in standalone mode the `queue` argument (or QUEUE_URL env variable) must be present"); } - Ok(()) } #[cfg(feature = "lambda")]