23 lines
872 B
YAML
23 lines
872 B
YAML
# This is an example configuration for the delta-s3-loader lambda
|
|
---
|
|
sources:
|
|
# This entry maps the existing S3 structure into a delta table named
|
|
# `logs.audit_logs` which will be date stamp partitioned
|
|
#
|
|
# └── simple
|
|
# └── date=2021-04-16
|
|
# └── auditlog-1234.json
|
|
# The name of the bucket should _just_ be a name of the bucket since this
|
|
# will come from an AWS Lambda event
|
|
- bucket: 'raw'
|
|
# Everything in the simple/ prefix will be considered part of this table
|
|
# the prefix may be a regular expression that is compatible with the Rust
|
|
# regex crate
|
|
prefix: '^simple'
|
|
# When specifying partitions, the source data is expected to be parttioned,
|
|
# as it is in this example
|
|
partitions:
|
|
- 'date'
|
|
# The table path is the destination for the written delta
|
|
tablepath: 's3://delta/audit_logs'
|