WIP Proc Macros

This commit is contained in:
Francis Lalonde 2019-04-11 00:49:57 -04:00
parent 7bb718c1b4
commit da8d940a74
70 changed files with 473 additions and 425 deletions

View File

@ -1,51 +1,5 @@
[package]
name = "dipstick"
version = "0.7.3-alpha.0"
authors = ["Francis Lalonde <fralalonde@gmail.com>"]
description = """A featureful, fast and fun metrics library decoupling app instrumentation from reporting backends.
Similar to popular logging frameworks, but with counters, timers and gauges.
Lots of features like combined outputs (e.g. log + graphite), sampling, aggregation, periodical publication, etc."""
documentation = "https://docs.rs/dipstick"
homepage = "https://github.com/fralalonde/dipstick"
repository = "https://github.com/fralalonde/dipstick"
readme = "README.md"
keywords = ["metrics", "statsd", "graphite", "timer", "prometheus"]
license = "MIT/Apache-2.0"
[badges]
travis-ci = { repository = "fralalonde/dipstick", branch = "master" }
[dependencies]
log = "0.4"
lazy_static = "1"
atomic_refcell = "0.1"
skeptic = { version = "0.13", optional = true }
num = { version = "0.2", default-features = false }
crossbeam-channel = { version = "0.3", optional = true }
parking_lot = { version = "0.7", optional = true }
# FIXME required only for random seed for sampling
time = "0.1"
minreq = { version = "1.0.0" }
# optional dep for standalone http pull metrics
tiny_http = { version = "0.6", optional = true }
[build-dependencies]
skeptic = { version = "0.13", optional = true }
[features]
default = [ "self_metrics", "crossbeam-channel", "parking_lot" ]
bench = []
self_metrics = []
tokio = []
[package.metadata.release]
#sign-commit = true
#upload-handbook = true
pre-release-replacements = [
{file="README.md", search="dipstick = \"[a-z0-9\\.-]+\"", replace="dipstick = \"{{version}}\""}
]
[workspace]
members = [
"library",
"procmacros",
]

51
library/Cargo.toml Normal file
View File

@ -0,0 +1,51 @@
[package]
name = "dipstick"
version = "0.7.3-alpha.0"
authors = ["Francis Lalonde <fralalonde@gmail.com>"]
description = """A featureful, fast and fun metrics library decoupling app instrumentation from reporting backends.
Similar to popular logging frameworks, but with counters, timers and gauges.
Lots of features like combined outputs (e.g. log + graphite), sampling, aggregation, periodical publication, etc."""
documentation = "https://docs.rs/dipstick"
homepage = "https://github.com/fralalonde/dipstick"
repository = "https://github.com/fralalonde/dipstick"
readme = "README.md"
keywords = ["metrics", "statsd", "graphite", "timer", "prometheus"]
license = "MIT/Apache-2.0"
[badges]
travis-ci = { repository = "fralalonde/dipstick", branch = "master" }
[dependencies]
log = "0.4"
lazy_static = "1"
atomic_refcell = "0.1"
skeptic = { version = "0.13", optional = true }
num = { version = "0.2", default-features = false }
crossbeam-channel = { version = "0.3", optional = true }
parking_lot = { version = "0.7", optional = true }
# FIXME required only for random seed for sampling
time = "0.1"
minreq = { version = "1.0.0" }
# optional dep for standalone http pull metrics
tiny_http = { version = "0.6", optional = true }
[build-dependencies]
skeptic = { version = "0.13", optional = true }
[features]
default = [ "self_metrics", "crossbeam-channel", "parking_lot" ]
bench = []
self_metrics = []
tokio = []
[package.metadata.release]
#sign-commit = true
#upload-handbook = true
pre-release-replacements = [
{file="README.md", search="dipstick = \"[a-z0-9\\.-]+\"", replace="dipstick = \"{{version}}\""}
]

View File

@ -1,170 +1,170 @@
//! A fixed-size cache with LRU expiration criteria.
//! Stored values will be held onto as long as there is space.
//! When space runs out, the oldest unused value will get evicted to make room for a new value.
use std::collections::HashMap;
use std::hash::Hash;
struct CacheEntry<K, V> {
key: K,
value: Option<V>,
next: Option<usize>,
prev: Option<usize>,
}
/// A fixed-size cache.
pub struct LRUCache<K, V> {
table: HashMap<K, usize>,
entries: Vec<CacheEntry<K, V>>,
first: Option<usize>,
last: Option<usize>,
capacity: usize,
}
impl<K: Clone + Hash + Eq, V> LRUCache<K, V> {
/// Creates a new cache that can hold the specified number of elements.
pub fn with_capacity(size: usize) -> Self {
LRUCache {
table: HashMap::with_capacity(size),
entries: Vec::with_capacity(size),
first: None,
last: None,
capacity: size,
}
}
/// Inserts a key-value pair into the cache and returns the previous value, if any.
/// If there is no room in the cache the oldest item will be removed.
pub fn insert(&mut self, key: K, value: V) -> Option<V> {
if self.table.contains_key(&key) {
self.access(&key);
let entry = &mut self.entries[self.first.unwrap()];
let old = entry.value.take();
entry.value = Some(value);
old
} else {
self.ensure_room();
// Update old head
let idx = self.entries.len();
if let Some(e) = self.first {
let prev = Some(idx);
self.entries[e].prev = prev;
}
// This is the new head
self.entries.push(CacheEntry {
key: key.clone(),
value: Some(value),
next: self.first,
prev: None,
});
self.first = Some(idx);
self.last = self.last.or(self.first);
self.table.insert(key, idx);
None
}
}
/// Retrieves a reference to the item associated with `key` from the cache
/// without promoting it.
pub fn peek(&mut self, key: &K) -> Option<&V> {
let entries = &self.entries;
self.table
.get(key)
.and_then(move |i| entries[*i].value.as_ref())
}
/// Retrieves a reference to the item associated with `key` from the cache.
pub fn get(&mut self, key: &K) -> Option<&V> {
if self.contains_key(key) {
self.access(key);
}
self.peek(key)
}
/// Returns the number of elements currently in the cache.
pub fn len(&self) -> usize {
self.table.len()
}
/// Promotes the specified key to the top of the cache.
fn access(&mut self, key: &K) {
let i = self.table[key];
self.remove_from_list(i);
self.first = Some(i);
}
pub fn contains_key(&mut self, key: &K) -> bool {
self.table.contains_key(key)
}
/// Removes an item from the linked list.
fn remove_from_list(&mut self, i: usize) {
let (prev, next) = {
let entry = &mut self.entries[i];
(entry.prev, entry.next)
};
match (prev, next) {
// Item was in the middle of the list
(Some(j), Some(k)) => {
{
let first = &mut self.entries[j];
first.next = next;
}
let second = &mut self.entries[k];
second.prev = prev;
}
// Item was at the end of the list
(Some(j), None) => {
let first = &mut self.entries[j];
first.next = None;
self.last = prev;
}
// Item was at front
_ => (),
}
}
fn ensure_room(&mut self) {
if self.capacity == self.len() {
self.remove_last();
}
}
/// Removes the oldest item in the cache.
fn remove_last(&mut self) {
if let Some(idx) = self.last {
self.remove_from_list(idx);
let key = &self.entries[idx].key;
self.table.remove(key);
}
if self.last.is_none() {
self.first = None;
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn get_and_get_mut_promote() {
let mut cache: LRUCache<&str, _> = LRUCache::with_capacity(2);
cache.insert("foo", 1);
cache.insert("bar", 2);
cache.get(&"foo").unwrap();
cache.insert("baz", 3);
assert!(cache.contains_key(&"foo"));
assert!(cache.contains_key(&"baz"));
assert!(!cache.contains_key(&"bar"));
cache.get(&"foo").unwrap();
cache.insert("qux", 4);
assert!(cache.contains_key(&"foo"));
assert!(cache.contains_key(&"qux"));
assert!(!cache.contains_key(&"baz"));
}
}
//! A fixed-size cache with LRU expiration criteria.
//! Stored values will be held onto as long as there is space.
//! When space runs out, the oldest unused value will get evicted to make room for a new value.
use std::collections::HashMap;
use std::hash::Hash;
struct CacheEntry<K, V> {
key: K,
value: Option<V>,
next: Option<usize>,
prev: Option<usize>,
}
/// A fixed-size cache.
pub struct LRUCache<K, V> {
table: HashMap<K, usize>,
entries: Vec<CacheEntry<K, V>>,
first: Option<usize>,
last: Option<usize>,
capacity: usize,
}
impl<K: Clone + Hash + Eq, V> LRUCache<K, V> {
/// Creates a new cache that can hold the specified number of elements.
pub fn with_capacity(size: usize) -> Self {
LRUCache {
table: HashMap::with_capacity(size),
entries: Vec::with_capacity(size),
first: None,
last: None,
capacity: size,
}
}
/// Inserts a key-value pair into the cache and returns the previous value, if any.
/// If there is no room in the cache the oldest item will be removed.
pub fn insert(&mut self, key: K, value: V) -> Option<V> {
if self.table.contains_key(&key) {
self.access(&key);
let entry = &mut self.entries[self.first.unwrap()];
let old = entry.value.take();
entry.value = Some(value);
old
} else {
self.ensure_room();
// Update old head
let idx = self.entries.len();
if let Some(e) = self.first {
let prev = Some(idx);
self.entries[e].prev = prev;
}
// This is the new head
self.entries.push(CacheEntry {
key: key.clone(),
value: Some(value),
next: self.first,
prev: None,
});
self.first = Some(idx);
self.last = self.last.or(self.first);
self.table.insert(key, idx);
None
}
}
/// Retrieves a reference to the item associated with `key` from the cache
/// without promoting it.
pub fn peek(&mut self, key: &K) -> Option<&V> {
let entries = &self.entries;
self.table
.get(key)
.and_then(move |i| entries[*i].value.as_ref())
}
/// Retrieves a reference to the item associated with `key` from the cache.
pub fn get(&mut self, key: &K) -> Option<&V> {
if self.contains_key(key) {
self.access(key);
}
self.peek(key)
}
/// Returns the number of elements currently in the cache.
pub fn len(&self) -> usize {
self.table.len()
}
/// Promotes the specified key to the top of the cache.
fn access(&mut self, key: &K) {
let i = self.table[key];
self.remove_from_list(i);
self.first = Some(i);
}
pub fn contains_key(&mut self, key: &K) -> bool {
self.table.contains_key(key)
}
/// Removes an item from the linked list.
fn remove_from_list(&mut self, i: usize) {
let (prev, next) = {
let entry = &mut self.entries[i];
(entry.prev, entry.next)
};
match (prev, next) {
// Item was in the middle of the list
(Some(j), Some(k)) => {
{
let first = &mut self.entries[j];
first.next = next;
}
let second = &mut self.entries[k];
second.prev = prev;
}
// Item was at the end of the list
(Some(j), None) => {
let first = &mut self.entries[j];
first.next = None;
self.last = prev;
}
// Item was at front
_ => (),
}
}
fn ensure_room(&mut self) {
if self.capacity == self.len() {
self.remove_last();
}
}
/// Removes the oldest item in the cache.
fn remove_last(&mut self) {
if let Some(idx) = self.last {
self.remove_from_list(idx);
let key = &self.entries[idx].key;
self.table.remove(key);
}
if self.last.is_none() {
self.first = None;
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn get_and_get_mut_promote() {
let mut cache: LRUCache<&str, _> = LRUCache::with_capacity(2);
cache.insert("foo", 1);
cache.insert("bar", 2);
cache.get(&"foo").unwrap();
cache.insert("baz", 3);
assert!(cache.contains_key(&"foo"));
assert!(cache.contains_key(&"baz"));
assert!(!cache.contains_key(&"bar"));
cache.get(&"foo").unwrap();
cache.insert("qux", 4);
assert!(cache.contains_key(&"foo"));
assert!(cache.contains_key(&"qux"));
assert!(!cache.contains_key(&"baz"));
}
}

View File

@ -1,3 +1,3 @@
pub mod cache_in;
pub mod cache_out;
pub mod lru_cache;
pub mod cache_in;
pub mod cache_out;
pub mod lru_cache;

View File

@ -1,56 +1,56 @@
pub mod attributes;
pub mod clock;
pub mod error;
pub mod input;
pub mod label;
pub mod locking;
pub mod metrics;
pub mod name;
pub mod output;
pub mod pcg32;
pub mod proxy;
pub mod scheduler;
pub mod void;
/// Base type for recorded metric values.
pub type MetricValue = isize;
/// Both InputScope and OutputScope share the ability to flush the recorded data.
pub trait Flush {
/// Flush does nothing by default.
fn flush(&self) -> error::Result<()>;
}
#[cfg(test)]
pub mod test {
use super::input::*;
use super::*;
#[test]
fn test_to_void() {
let c = void::Void::new().metrics();
let m = c.new_metric("test".into(), input::InputKind::Marker);
m.write(33, labels![]);
}
}
#[cfg(feature = "bench")]
pub mod bench {
use super::super::bucket::atomic::*;
use super::clock::*;
use super::input::*;
use test;
#[bench]
fn get_instant(b: &mut test::Bencher) {
b.iter(|| test::black_box(TimeHandle::now()));
}
#[bench]
fn time_bench_direct_dispatch_event(b: &mut test::Bencher) {
let metrics = AtomicBucket::new();
let marker = metrics.marker("aaa");
b.iter(|| test::black_box(marker.mark()));
}
}
pub mod attributes;
pub mod clock;
pub mod error;
pub mod input;
pub mod label;
pub mod locking;
pub mod metrics;
pub mod name;
pub mod output;
pub mod pcg32;
pub mod proxy;
pub mod scheduler;
pub mod void;
/// Base type for recorded metric values.
pub type MetricValue = isize;
/// Both InputScope and OutputScope share the ability to flush the recorded data.
pub trait Flush {
/// Flush does nothing by default.
fn flush(&self) -> error::Result<()>;
}
#[cfg(test)]
pub mod test {
use super::input::*;
use super::*;
#[test]
fn test_to_void() {
let c = void::Void::new().metrics();
let m = c.new_metric("test".into(), input::InputKind::Marker);
m.write(33, labels![]);
}
}
#[cfg(feature = "bench")]
pub mod bench {
use super::super::bucket::atomic::*;
use super::clock::*;
use super::input::*;
use test;
#[bench]
fn get_instant(b: &mut test::Bencher) {
b.iter(|| test::black_box(TimeHandle::now()));
}
#[bench]
fn time_bench_direct_dispatch_event(b: &mut test::Bencher) {
let metrics = AtomicBucket::new();
let marker = metrics.marker("aaa");
b.iter(|| test::black_box(marker.mark()));
}
}

View File

@ -1,68 +1,68 @@
use core::input::InputKind;
use core::label::Labels;
use core::name::MetricName;
use core::void::Void;
use core::{Flush, MetricValue};
use std::rc::Rc;
/// Define metrics, write values and flush them.
pub trait OutputScope: Flush {
/// Define a raw metric of the specified type.
fn new_metric(&self, name: MetricName, kind: InputKind) -> OutputMetric;
}
/// Output metrics are not thread safe.
#[derive(Clone)]
pub struct OutputMetric {
inner: Rc<Fn(MetricValue, Labels)>,
}
impl OutputMetric {
/// Utility constructor
pub fn new<F: Fn(MetricValue, Labels) + 'static>(metric: F) -> OutputMetric {
OutputMetric {
inner: Rc::new(metric),
}
}
/// Some may prefer the `metric.write(value)` form to the `(metric)(value)` form.
/// This shouldn't matter as metrics should be of type Counter, Marker, etc.
#[inline]
pub fn write(&self, value: MetricValue, labels: Labels) {
(self.inner)(value, labels)
}
}
/// A function trait that opens a new metric capture scope.
pub trait Output: Send + Sync + 'static + OutputDyn {
/// The type of Scope returned byt this output.
type SCOPE: OutputScope;
/// Open a new scope for this output.
fn new_scope(&self) -> Self::SCOPE;
/// Open a new scope for this output.
#[deprecated(since = "0.7.2", note = "Use new_scope()")]
fn output(&self) -> Self::SCOPE {
self.new_scope()
}
}
/// A function trait that opens a new metric capture scope.
pub trait OutputDyn {
/// Open a new scope from this output.
fn output_dyn(&self) -> Rc<OutputScope + 'static>;
}
/// Blanket impl of dyn output trait
impl<T: Output + Send + Sync + 'static> OutputDyn for T {
fn output_dyn(&self) -> Rc<OutputScope + 'static> {
Rc::new(self.new_scope())
}
}
/// Discard all metric values sent to it.
pub fn output_none() -> Void {
Void {}
}
use core::input::InputKind;
use core::label::Labels;
use core::name::MetricName;
use core::void::Void;
use core::{Flush, MetricValue};
use std::rc::Rc;
/// Define metrics, write values and flush them.
pub trait OutputScope: Flush {
/// Define a raw metric of the specified type.
fn new_metric(&self, name: MetricName, kind: InputKind) -> OutputMetric;
}
/// Output metrics are not thread safe.
#[derive(Clone)]
pub struct OutputMetric {
inner: Rc<Fn(MetricValue, Labels)>,
}
impl OutputMetric {
/// Utility constructor
pub fn new<F: Fn(MetricValue, Labels) + 'static>(metric: F) -> OutputMetric {
OutputMetric {
inner: Rc::new(metric),
}
}
/// Some may prefer the `metric.write(value)` form to the `(metric)(value)` form.
/// This shouldn't matter as metrics should be of type Counter, Marker, etc.
#[inline]
pub fn write(&self, value: MetricValue, labels: Labels) {
(self.inner)(value, labels)
}
}
/// A function trait that opens a new metric capture scope.
pub trait Output: Send + Sync + 'static + OutputDyn {
/// The type of Scope returned byt this output.
type SCOPE: OutputScope;
/// Open a new scope for this output.
fn new_scope(&self) -> Self::SCOPE;
/// Open a new scope for this output.
#[deprecated(since = "0.7.2", note = "Use new_scope()")]
fn output(&self) -> Self::SCOPE {
self.new_scope()
}
}
/// A function trait that opens a new metric capture scope.
pub trait OutputDyn {
/// Open a new scope from this output.
fn output_dyn(&self) -> Rc<OutputScope + 'static>;
}
/// Blanket impl of dyn output trait
impl<T: Output + Send + Sync + 'static> OutputDyn for T {
fn output_dyn(&self) -> Rc<OutputScope + 'static> {
Rc::new(self.new_scope())
}
}
/// Discard all metric values sent to it.
pub fn output_none() -> Void {
Void {}
}

View File

@ -1,56 +1,56 @@
use core::input::{InputDyn, InputKind, InputScope};
use core::name::MetricName;
use core::output::{Output, OutputMetric, OutputScope};
use core::Flush;
use std::error::Error;
use std::sync::Arc;
lazy_static! {
/// The reference instance identifying an uninitialized metric config.
pub static ref VOID_INPUT: Arc<InputDyn + Send + Sync> = Arc::new(Void::new());
/// The reference instance identifying an uninitialized metric scope.
pub static ref NO_METRIC_SCOPE: Arc<InputScope + Send + Sync> = VOID_INPUT.input_dyn();
}
/// Discard metrics output.
#[derive(Clone)]
pub struct Void {}
/// Discard metrics output.
#[derive(Clone)]
pub struct VoidOutput {}
impl Void {
/// Void metrics builder.
#[deprecated(since = "0.7.2", note = "Use new()")]
pub fn metrics() -> Self {
Self::new()
}
/// Void metrics builder.
pub fn new() -> Self {
Void {}
}
}
impl Output for Void {
type SCOPE = VoidOutput;
fn new_scope(&self) -> Self::SCOPE {
VoidOutput {}
}
}
impl OutputScope for VoidOutput {
fn new_metric(&self, _name: MetricName, _kind: InputKind) -> OutputMetric {
OutputMetric::new(|_value, _labels| {})
}
}
impl Flush for VoidOutput {
fn flush(&self) -> Result<(), Box<Error + Send + Sync>> {
Ok(())
}
}
use core::input::{InputDyn, InputKind, InputScope};
use core::name::MetricName;
use core::output::{Output, OutputMetric, OutputScope};
use core::Flush;
use std::error::Error;
use std::sync::Arc;
lazy_static! {
/// The reference instance identifying an uninitialized metric config.
pub static ref VOID_INPUT: Arc<InputDyn + Send + Sync> = Arc::new(Void::new());
/// The reference instance identifying an uninitialized metric scope.
pub static ref NO_METRIC_SCOPE: Arc<InputScope + Send + Sync> = VOID_INPUT.input_dyn();
}
/// Discard metrics output.
#[derive(Clone)]
pub struct Void {}
/// Discard metrics output.
#[derive(Clone)]
pub struct VoidOutput {}
impl Void {
/// Void metrics builder.
#[deprecated(since = "0.7.2", note = "Use new()")]
pub fn metrics() -> Self {
Self::new()
}
/// Void metrics builder.
pub fn new() -> Self {
Void {}
}
}
impl Output for Void {
type SCOPE = VoidOutput;
fn new_scope(&self) -> Self::SCOPE {
VoidOutput {}
}
}
impl OutputScope for VoidOutput {
fn new_metric(&self, _name: MetricName, _kind: InputKind) -> OutputMetric {
OutputMetric::new(|_value, _labels| {})
}
}
impl Flush for VoidOutput {
fn flush(&self) -> Result<(), Box<Error + Send + Sync>> {
Ok(())
}
}

View File

@ -1,3 +1,3 @@
pub mod multi_in;
pub mod multi_out;
pub mod multi_in;
pub mod multi_out;

View File

@ -1,16 +1,16 @@
pub mod format;
pub mod map;
pub mod stream;
pub mod log;
pub mod socket;
pub mod graphite;
pub mod statsd;
//#[cfg(feature="prometheus")]
pub mod prometheus;
pub mod format;
pub mod map;
pub mod stream;
pub mod log;
pub mod socket;
pub mod graphite;
pub mod statsd;
//#[cfg(feature="prometheus")]
pub mod prometheus;

View File

@ -1,2 +1,2 @@
pub mod queue_in;
pub mod queue_out;
pub mod queue_in;
pub mod queue_out;

23
procmacros/Cargo.toml Normal file
View File

@ -0,0 +1,23 @@
[package]
name = "dipstick-procmacros"
version = "0.7.3-alpha.0"
authors = ["Francis Lalonde <fralalonde@gmail.com>"]
description = """Procmacros for Dipstick, providing #[timed] #[counted] and other declarative instrumentation.
Dipstick is a featureful, fast and fun metrics library decoupling app instrumentation from reporting backends.
Similar to popular logging frameworks, but with counters, timers and gauges.
Lots of features like combined outputs (e.g. log + graphite), sampling, aggregation, periodical publication, etc."""
documentation = "https://docs.rs/dipstick-procmacros"
homepage = "https://github.com/fralalonde/dipstick"
repository = "https://github.com/fralalonde/dipstick"
readme = "README.md"
keywords = ["metrics", "statsd", "graphite", "timer", "prometheus"]
license = "MIT/Apache-2.0"
[dependencies]
syn = {version= "0.15" }
quote = "0.6"
[lib]
proc-macro = true

20
procmacros/src/lib.rs Normal file
View File

@ -0,0 +1,20 @@
//! Procedural macros for Dipstick, a metric library for Rust.
//!
//! Please check the Dipstick crate for more documentation.
#![recursion_limit = "512"]
extern crate proc_macro;
#[macro_use]
extern crate syn;
#[macro_use]
extern crate quote;
use proc_macro::TokenStream;
#[proc_macro_attribute]
pub fn timed(attrs: TokenStream, item: TokenStream) -> TokenStream {
item
}