rtyler
/
delta-s3-loader
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Properly build with the deltalake dependency and my features branch

This commit is contained in:
R Tyler Croy 2021-06-12 15:21:09 -07:00
parent 559c977798
commit 2667088175
3 changed files with 219 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

216
Cargo.lock generated
View File

@ -279,12 +279,44 @@ dependencies = [
"ansi_term",
"atty",
"bitflags",
"strsim",
"textwrap",
"strsim 0.8.0",
"textwrap 0.11.0",
"unicode-width",
"vec_map",
]
[[package]]
name = "clap"
version = "3.0.0-beta.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4bd1061998a501ee7d4b6d449020df3266ca3124b941ec56cf2005c3779ca142"
dependencies = [
"atty",
"bitflags",
"clap_derive",
"indexmap",
"lazy_static",
"os_str_bytes",
"strsim 0.10.0",
"termcolor",
"textwrap 0.12.1",
"unicode-width",
"vec_map",
]
[[package]]
name = "clap_derive"
version = "3.0.0-beta.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "370f715b81112975b1b69db93e0b56ea4cd4e5002ac43b2da8474106a54096a1"
dependencies = [
"heck",
"proc-macro-error",
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "const_fn"
version = "0.4.8"
@ -373,7 +405,8 @@ dependencies = [
"anyhow",
"arrow",
"aws_lambda_events",
"clap",
"clap 2.33.3",
"deltalake",
"dotenv",
"lambda_runtime",
"log",
@ -391,6 +424,42 @@ dependencies = [
"tokio",
]
[[package]]
name = "deltalake"
version = "0.4.0"
source = "git+https://github.com/rtyler/delta-rs?branch=features-for-rustls#b8cbe8e7007cd5ef3970dd2e0b98d517120356ad"
dependencies = [
"anyhow",
"arrow",
"async-trait",
"bytes 1.0.1",
"cfg-if",
"chrono",
"clap 3.0.0-beta.2",
"env_logger",
"errno",
"futures",
"glibc_version",
"lazy_static",
"libc",
"log",
"maplit",
"parquet",
"parquet-format",
"regex",
"rusoto_core",
"rusoto_credential",
"rusoto_dynamodb",
"rusoto_s3",
"rusoto_sts",
"serde",
"serde_json",
"thiserror",
"tokio",
"tokio-stream",
"uuid",
]
[[package]]
name = "digest"
version = "0.9.0"
@ -452,6 +521,27 @@ dependencies = [
"termcolor",
]
[[package]]
name = "errno"
version = "0.2.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fa68f2fb9cae9d37c9b2b3584aba698a2e97f72d7aef7b9f7aa71d8b54ce46fe"
dependencies = [
"errno-dragonfly",
"libc",
"winapi",
]
[[package]]
name = "errno-dragonfly"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "14ca354e36190500e1e1fb267c647932382b54053c50b14970856c0b00a35067"
dependencies = [
"gcc",
"libc",
]
[[package]]
name = "flatbuffers"
version = "0.8.4"
@ -585,6 +675,12 @@ dependencies = [
"slab",
]
[[package]]
name = "gcc"
version = "0.3.55"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f5f3913fa0bfe7ee1fd8248b6b9f42a5af4b9d65ec2dd2c3c26132b950ecfc2"
[[package]]
name = "generic-array"
version = "0.14.4"
@ -617,12 +713,30 @@ dependencies = [
"wasi 0.10.2+wasi-snapshot-preview1",
]
[[package]]
name = "glibc_version"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "803ff7635f1ab4e2c064b68a0c60da917d3d18dc8d086130f689d62ce4f1c33e"
dependencies = [
"regex",
]
[[package]]
name = "hashbrown"
version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04"
[[package]]
name = "heck"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c"
dependencies = [
"unicode-segmentation",
]
[[package]]
name = "hermit-abi"
version = "0.1.18"
@ -872,6 +986,12 @@ dependencies = [
"libc",
]
[[package]]
name = "maplit"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d"
[[package]]
name = "matches"
version = "0.1.8"
@ -1064,6 +1184,12 @@ dependencies = [
"num-traits",
]
[[package]]
name = "os_str_bytes"
version = "2.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "afb2e1c3ee07430c2cf76151675e583e0f19985fa6efae47d6848a3e2c824f85"
[[package]]
name = "parquet"
version = "4.2.0"
@ -1127,6 +1253,30 @@ dependencies = [
"log",
]
[[package]]
name = "proc-macro-error"
version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c"
dependencies = [
"proc-macro-error-attr",
"proc-macro2",
"quote",
"syn",
"version_check",
]
[[package]]
name = "proc-macro-error-attr"
version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869"
dependencies = [
"proc-macro2",
"quote",
"version_check",
]
[[package]]
name = "proc-macro-hack"
version = "0.5.19"
@ -1304,6 +1454,20 @@ dependencies = [
"zeroize",
]
[[package]]
name = "rusoto_dynamodb"
version = "0.46.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0f26af40f36409cb8fae3069690f78f638f747b55c7b90f338d5ed36016b0cda"
dependencies = [
"async-trait",
"bytes 1.0.1",
"futures",
"rusoto_core",
"serde",
"serde_json",
]
[[package]]
name = "rusoto_s3"
version = "0.46.0"
@ -1358,6 +1522,21 @@ dependencies = [
"xml-rs",
]
[[package]]
name = "rusoto_sts"
version = "0.46.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2f93005e0c3b9e40a424b50ca71886d2445cc19bb6cdac3ac84c2daff482eb59"
dependencies = [
"async-trait",
"bytes 1.0.1",
"chrono",
"futures",
"rusoto_core",
"serde_urlencoded",
"xml-rs",
]
[[package]]
name = "rustc_version"
version = "0.2.3"
@ -1669,6 +1848,12 @@ version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a"
[[package]]
name = "strsim"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
[[package]]
name = "subtle"
version = "2.4.0"
@ -1704,6 +1889,15 @@ dependencies = [
"unicode-width",
]
[[package]]
name = "textwrap"
version = "0.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "203008d98caf094106cfaba70acfed15e18ed3ddb7d94e49baec153a2b462789"
dependencies = [
"unicode-width",
]
[[package]]
name = "thiserror"
version = "1.0.25"
@ -1960,6 +2154,12 @@ dependencies = [
"tinyvec",
]
[[package]]
name = "unicode-segmentation"
version = "1.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bb0d2e7be6ae3a5fa87eed5fb451aff96f2573d2694942e40543ae0bbe19c796"
[[package]]
name = "unicode-width"
version = "0.1.8"
@ -1990,6 +2190,16 @@ dependencies = [
"percent-encoding",
]
[[package]]
name = "uuid"
version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7"
dependencies = [
"getrandom 0.2.3",
"serde",
]
[[package]]
name = "vec_map"
version = "0.8.2"

2
Cargo.toml
View File

@ -11,7 +11,7 @@ anyhow = "*"
aws_lambda_events = "*"
arrow = "4"
clap = "2"
#deltalake = { git = "https://github.com/rtyler/delta-rs", branch = "main", features = ["s3"] }
deltalake = { git = "https://github.com/rtyler/delta-rs", branch = "features-for-rustls", features = ["s3-rustls"] }
dotenv = "0.15"
log = "0.4"
parquet = "4"

10
src/main.rs
View File

@ -73,13 +73,15 @@ async fn main() -> Result<(), anyhow::Error> {
loop {
let rcv = ReceiveMessageRequest {
queue_url: queue.to_string(),
// Always consuming the max number of available messages to reduce round trips
max_number_of_messages: Some(10),
wait_time_seconds: Some(5),
// This wait time is being used as the effective loop interval for the main runloop
wait_time_seconds: Some(10),
..Default::default()
};
let res = client.receive_message(rcv).await?;
debug!("Message(s) received: {:?}", res);
trace!("Message(s) received: {:?}", res);
if let Some(messages) = &res.messages {
for message in messages {
@ -93,7 +95,7 @@ async fn main() -> Result<(), anyhow::Error> {
}
}
} else {
debug!("Message had no body: {:?}", &message);
warn!("Message had no body: {:?}", &message);
}
if let Some(receipt) = &message.receipt_handle {
@ -108,12 +110,10 @@ async fn main() -> Result<(), anyhow::Error> {
}
}
}
break;
}
} else {
panic!("When running in standalone mode the `queue` argument (or QUEUE_URL env variable) must be present");
}
Ok(())
}
#[cfg(feature = "lambda")]