Properly build with the deltalake dependency and my features branch
This commit is contained in:
parent
559c977798
commit
2667088175
|
@ -279,12 +279,44 @@ dependencies = [
|
||||||
"ansi_term",
|
"ansi_term",
|
||||||
"atty",
|
"atty",
|
||||||
"bitflags",
|
"bitflags",
|
||||||
"strsim",
|
"strsim 0.8.0",
|
||||||
"textwrap",
|
"textwrap 0.11.0",
|
||||||
"unicode-width",
|
"unicode-width",
|
||||||
"vec_map",
|
"vec_map",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "clap"
|
||||||
|
version = "3.0.0-beta.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "4bd1061998a501ee7d4b6d449020df3266ca3124b941ec56cf2005c3779ca142"
|
||||||
|
dependencies = [
|
||||||
|
"atty",
|
||||||
|
"bitflags",
|
||||||
|
"clap_derive",
|
||||||
|
"indexmap",
|
||||||
|
"lazy_static",
|
||||||
|
"os_str_bytes",
|
||||||
|
"strsim 0.10.0",
|
||||||
|
"termcolor",
|
||||||
|
"textwrap 0.12.1",
|
||||||
|
"unicode-width",
|
||||||
|
"vec_map",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "clap_derive"
|
||||||
|
version = "3.0.0-beta.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "370f715b81112975b1b69db93e0b56ea4cd4e5002ac43b2da8474106a54096a1"
|
||||||
|
dependencies = [
|
||||||
|
"heck",
|
||||||
|
"proc-macro-error",
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"syn",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "const_fn"
|
name = "const_fn"
|
||||||
version = "0.4.8"
|
version = "0.4.8"
|
||||||
|
@ -373,7 +405,8 @@ dependencies = [
|
||||||
"anyhow",
|
"anyhow",
|
||||||
"arrow",
|
"arrow",
|
||||||
"aws_lambda_events",
|
"aws_lambda_events",
|
||||||
"clap",
|
"clap 2.33.3",
|
||||||
|
"deltalake",
|
||||||
"dotenv",
|
"dotenv",
|
||||||
"lambda_runtime",
|
"lambda_runtime",
|
||||||
"log",
|
"log",
|
||||||
|
@ -391,6 +424,42 @@ dependencies = [
|
||||||
"tokio",
|
"tokio",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "deltalake"
|
||||||
|
version = "0.4.0"
|
||||||
|
source = "git+https://github.com/rtyler/delta-rs?branch=features-for-rustls#b8cbe8e7007cd5ef3970dd2e0b98d517120356ad"
|
||||||
|
dependencies = [
|
||||||
|
"anyhow",
|
||||||
|
"arrow",
|
||||||
|
"async-trait",
|
||||||
|
"bytes 1.0.1",
|
||||||
|
"cfg-if",
|
||||||
|
"chrono",
|
||||||
|
"clap 3.0.0-beta.2",
|
||||||
|
"env_logger",
|
||||||
|
"errno",
|
||||||
|
"futures",
|
||||||
|
"glibc_version",
|
||||||
|
"lazy_static",
|
||||||
|
"libc",
|
||||||
|
"log",
|
||||||
|
"maplit",
|
||||||
|
"parquet",
|
||||||
|
"parquet-format",
|
||||||
|
"regex",
|
||||||
|
"rusoto_core",
|
||||||
|
"rusoto_credential",
|
||||||
|
"rusoto_dynamodb",
|
||||||
|
"rusoto_s3",
|
||||||
|
"rusoto_sts",
|
||||||
|
"serde",
|
||||||
|
"serde_json",
|
||||||
|
"thiserror",
|
||||||
|
"tokio",
|
||||||
|
"tokio-stream",
|
||||||
|
"uuid",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "digest"
|
name = "digest"
|
||||||
version = "0.9.0"
|
version = "0.9.0"
|
||||||
|
@ -452,6 +521,27 @@ dependencies = [
|
||||||
"termcolor",
|
"termcolor",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "errno"
|
||||||
|
version = "0.2.7"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "fa68f2fb9cae9d37c9b2b3584aba698a2e97f72d7aef7b9f7aa71d8b54ce46fe"
|
||||||
|
dependencies = [
|
||||||
|
"errno-dragonfly",
|
||||||
|
"libc",
|
||||||
|
"winapi",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "errno-dragonfly"
|
||||||
|
version = "0.1.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "14ca354e36190500e1e1fb267c647932382b54053c50b14970856c0b00a35067"
|
||||||
|
dependencies = [
|
||||||
|
"gcc",
|
||||||
|
"libc",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "flatbuffers"
|
name = "flatbuffers"
|
||||||
version = "0.8.4"
|
version = "0.8.4"
|
||||||
|
@ -585,6 +675,12 @@ dependencies = [
|
||||||
"slab",
|
"slab",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "gcc"
|
||||||
|
version = "0.3.55"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "8f5f3913fa0bfe7ee1fd8248b6b9f42a5af4b9d65ec2dd2c3c26132b950ecfc2"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "generic-array"
|
name = "generic-array"
|
||||||
version = "0.14.4"
|
version = "0.14.4"
|
||||||
|
@ -617,12 +713,30 @@ dependencies = [
|
||||||
"wasi 0.10.2+wasi-snapshot-preview1",
|
"wasi 0.10.2+wasi-snapshot-preview1",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "glibc_version"
|
||||||
|
version = "0.1.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "803ff7635f1ab4e2c064b68a0c60da917d3d18dc8d086130f689d62ce4f1c33e"
|
||||||
|
dependencies = [
|
||||||
|
"regex",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "hashbrown"
|
name = "hashbrown"
|
||||||
version = "0.9.1"
|
version = "0.9.1"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04"
|
checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "heck"
|
||||||
|
version = "0.3.3"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c"
|
||||||
|
dependencies = [
|
||||||
|
"unicode-segmentation",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "hermit-abi"
|
name = "hermit-abi"
|
||||||
version = "0.1.18"
|
version = "0.1.18"
|
||||||
|
@ -872,6 +986,12 @@ dependencies = [
|
||||||
"libc",
|
"libc",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "maplit"
|
||||||
|
version = "1.0.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "matches"
|
name = "matches"
|
||||||
version = "0.1.8"
|
version = "0.1.8"
|
||||||
|
@ -1064,6 +1184,12 @@ dependencies = [
|
||||||
"num-traits",
|
"num-traits",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "os_str_bytes"
|
||||||
|
version = "2.4.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "afb2e1c3ee07430c2cf76151675e583e0f19985fa6efae47d6848a3e2c824f85"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "parquet"
|
name = "parquet"
|
||||||
version = "4.2.0"
|
version = "4.2.0"
|
||||||
|
@ -1127,6 +1253,30 @@ dependencies = [
|
||||||
"log",
|
"log",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "proc-macro-error"
|
||||||
|
version = "1.0.4"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c"
|
||||||
|
dependencies = [
|
||||||
|
"proc-macro-error-attr",
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"syn",
|
||||||
|
"version_check",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "proc-macro-error-attr"
|
||||||
|
version = "1.0.4"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869"
|
||||||
|
dependencies = [
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"version_check",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "proc-macro-hack"
|
name = "proc-macro-hack"
|
||||||
version = "0.5.19"
|
version = "0.5.19"
|
||||||
|
@ -1304,6 +1454,20 @@ dependencies = [
|
||||||
"zeroize",
|
"zeroize",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "rusoto_dynamodb"
|
||||||
|
version = "0.46.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "0f26af40f36409cb8fae3069690f78f638f747b55c7b90f338d5ed36016b0cda"
|
||||||
|
dependencies = [
|
||||||
|
"async-trait",
|
||||||
|
"bytes 1.0.1",
|
||||||
|
"futures",
|
||||||
|
"rusoto_core",
|
||||||
|
"serde",
|
||||||
|
"serde_json",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "rusoto_s3"
|
name = "rusoto_s3"
|
||||||
version = "0.46.0"
|
version = "0.46.0"
|
||||||
|
@ -1358,6 +1522,21 @@ dependencies = [
|
||||||
"xml-rs",
|
"xml-rs",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "rusoto_sts"
|
||||||
|
version = "0.46.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "2f93005e0c3b9e40a424b50ca71886d2445cc19bb6cdac3ac84c2daff482eb59"
|
||||||
|
dependencies = [
|
||||||
|
"async-trait",
|
||||||
|
"bytes 1.0.1",
|
||||||
|
"chrono",
|
||||||
|
"futures",
|
||||||
|
"rusoto_core",
|
||||||
|
"serde_urlencoded",
|
||||||
|
"xml-rs",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "rustc_version"
|
name = "rustc_version"
|
||||||
version = "0.2.3"
|
version = "0.2.3"
|
||||||
|
@ -1669,6 +1848,12 @@ version = "0.8.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a"
|
checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "strsim"
|
||||||
|
version = "0.10.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "subtle"
|
name = "subtle"
|
||||||
version = "2.4.0"
|
version = "2.4.0"
|
||||||
|
@ -1704,6 +1889,15 @@ dependencies = [
|
||||||
"unicode-width",
|
"unicode-width",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "textwrap"
|
||||||
|
version = "0.12.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "203008d98caf094106cfaba70acfed15e18ed3ddb7d94e49baec153a2b462789"
|
||||||
|
dependencies = [
|
||||||
|
"unicode-width",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "thiserror"
|
name = "thiserror"
|
||||||
version = "1.0.25"
|
version = "1.0.25"
|
||||||
|
@ -1960,6 +2154,12 @@ dependencies = [
|
||||||
"tinyvec",
|
"tinyvec",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "unicode-segmentation"
|
||||||
|
version = "1.7.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "bb0d2e7be6ae3a5fa87eed5fb451aff96f2573d2694942e40543ae0bbe19c796"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "unicode-width"
|
name = "unicode-width"
|
||||||
version = "0.1.8"
|
version = "0.1.8"
|
||||||
|
@ -1990,6 +2190,16 @@ dependencies = [
|
||||||
"percent-encoding",
|
"percent-encoding",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "uuid"
|
||||||
|
version = "0.8.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7"
|
||||||
|
dependencies = [
|
||||||
|
"getrandom 0.2.3",
|
||||||
|
"serde",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "vec_map"
|
name = "vec_map"
|
||||||
version = "0.8.2"
|
version = "0.8.2"
|
||||||
|
|
|
@ -11,7 +11,7 @@ anyhow = "*"
|
||||||
aws_lambda_events = "*"
|
aws_lambda_events = "*"
|
||||||
arrow = "4"
|
arrow = "4"
|
||||||
clap = "2"
|
clap = "2"
|
||||||
#deltalake = { git = "https://github.com/rtyler/delta-rs", branch = "main", features = ["s3"] }
|
deltalake = { git = "https://github.com/rtyler/delta-rs", branch = "features-for-rustls", features = ["s3-rustls"] }
|
||||||
dotenv = "0.15"
|
dotenv = "0.15"
|
||||||
log = "0.4"
|
log = "0.4"
|
||||||
parquet = "4"
|
parquet = "4"
|
||||||
|
|
10
src/main.rs
10
src/main.rs
|
@ -73,13 +73,15 @@ async fn main() -> Result<(), anyhow::Error> {
|
||||||
loop {
|
loop {
|
||||||
let rcv = ReceiveMessageRequest {
|
let rcv = ReceiveMessageRequest {
|
||||||
queue_url: queue.to_string(),
|
queue_url: queue.to_string(),
|
||||||
|
// Always consuming the max number of available messages to reduce round trips
|
||||||
max_number_of_messages: Some(10),
|
max_number_of_messages: Some(10),
|
||||||
wait_time_seconds: Some(5),
|
// This wait time is being used as the effective loop interval for the main runloop
|
||||||
|
wait_time_seconds: Some(10),
|
||||||
..Default::default()
|
..Default::default()
|
||||||
};
|
};
|
||||||
|
|
||||||
let res = client.receive_message(rcv).await?;
|
let res = client.receive_message(rcv).await?;
|
||||||
debug!("Message(s) received: {:?}", res);
|
trace!("Message(s) received: {:?}", res);
|
||||||
|
|
||||||
if let Some(messages) = &res.messages {
|
if let Some(messages) = &res.messages {
|
||||||
for message in messages {
|
for message in messages {
|
||||||
|
@ -93,7 +95,7 @@ async fn main() -> Result<(), anyhow::Error> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
debug!("Message had no body: {:?}", &message);
|
warn!("Message had no body: {:?}", &message);
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Some(receipt) = &message.receipt_handle {
|
if let Some(receipt) = &message.receipt_handle {
|
||||||
|
@ -108,12 +110,10 @@ async fn main() -> Result<(), anyhow::Error> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
break;
|
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
panic!("When running in standalone mode the `queue` argument (or QUEUE_URL env variable) must be present");
|
panic!("When running in standalone mode the `queue` argument (or QUEUE_URL env variable) must be present");
|
||||||
}
|
}
|
||||||
Ok(())
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(feature = "lambda")]
|
#[cfg(feature = "lambda")]
|
||||||
|
|
Loading…
Reference in New Issue