Begin to introduce the configuhration and the Databricks API passthrough support

Cargo.toml

@@ -17,7 +17,7 @@ license = "LICENSE"
 [workspace.dependencies]
 anyhow = "1.0.80"
 async-std = { version = "1.12.0", features = ["tokio1", "attributes"] }
-chrono = "0.4.33"
+chrono = { version = "0.4.33", features = ["serde"] }
 dotenvy = "0.15.7"
 futures = "0.3.30"
 sqlx = { version = "0.7.3", features = ["runtime-async-std", "chrono", "json", "postgres", "time", "uuid"] }
@@ -27,4 +27,4 @@ serde_yaml = "0.9.32"
 tracing = { version = "0.1.40", features = ["log"] }
 tracing-subscriber = { version = "0.3.18", features = ["env-filter"] }
 uuid = { version = "1.7.0", features = ["v4", "serde"] }
-url = "2.5.0"
+url = { version = "2.5.0", features = ["serde"]}

Jenkinsfile

@@ -0,0 +1,20 @@
+//
+// This Jenkinsfile does not run for every pull request
+
+
+pipeline {
+  agent { label 'linux && rust' }
+
+  stages {
+    stage('Build') {
+      steps {
+        sh 'make check'
+      }
+    }
+  }
+}
+
+
+
+
+// vim: ft=groovy ts=2 sw=2 et

Makefile

@@ -16,7 +16,7 @@ SOURCES=$(shell find . -type f -iname '*.rs')
 $(WEBAPP): Cargo.toml $(SOURCES)
 	DATABASE_URL=$(DATABASE_URL) $(CARGO) build
 
-check: Cargo.toml $(SOURCES) migrations
+check: Cargo.toml $(SOURCES) migrations ## Run all the checks and tests
 	$(CARGO) fmt
 	DATABASE_URL=$(DATABASE_URL) $(CARGO) check
 	DATABASE_URL=$(DATABASE_URL) $(CARGO) test

README.adoc

@@ -23,7 +23,7 @@ toc::[]
 The full development stack for `placementd` requires Kubernetes, which for
 local development purposes can be accomplished with
 link:https://kind.sigs.k8s.io/[kind]. Launching the cluster should be done
-with: `kind cluster create --config contrib/kind.yml`
+with: `kind create cluster --config contrib/kind.yml`
 
 Once `kind` is up and running, there are a number of `make` targets defined in
 the `Makefile` to help with development, simply running `make` will list the

common/Cargo.toml

@@ -6,6 +6,7 @@ edition.workspace = true
 [dependencies]
 async-std = { workspace = true }
 chrono = { workspace = true }
+futures = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
 serde_yaml = { workspace = true }

common/src/config.rs

@@ -0,0 +1,52 @@
+//!
+//! The `config` module contains the common configuration code for all of placementd
+//!
+use serde::Deserialize;
+use url::Url;
+
+#[derive(Clone, Debug, Deserialize)]
+struct Placementd {
+    general: General,
+    web: Option<WebApi>,
+}
+
+#[derive(Clone, Debug, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
+struct General {}
+
+#[derive(Clone, Debug, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
+struct WebApi {
+    databricks: Option<Databricks>,
+}
+
+#[derive(Clone, Debug, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
+struct Databricks {
+    mode: ApiMode,
+    url: Url,
+    token_env: String,
+}
+
+#[derive(Clone, Debug, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
+#[serde(rename_all = "lowercase")]
+enum ApiMode {
+    Passthrough,
+    Override,
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_load_example() {
+        let file =
+            std::fs::File::open("../config/placementd.example.yml").expect("Failed to find file");
+        let c: Placementd =
+            serde_yaml::from_reader(file).expect("Failed to load configuration file");
+        assert!(c.web.is_some());
+        let api = c.web.unwrap();
+        assert!(api.databricks.is_some());
+        if let Some(databricks) = api.databricks.as_ref() {
+            assert_eq!(ApiMode::Passthrough, databricks.mode);
+        }
+    }
+}

common/src/db.rs

@@ -254,7 +254,6 @@ mod integration_tests {
 
     #[async_std::test]
     async fn test_task_lock_conflict() {
-        use async_std::prelude::FutureExt;
         use async_std::task::JoinHandle;
 
         let pool = bootstrap().await;
@@ -293,7 +292,7 @@ mod integration_tests {
             Ok(())
         });
 
-        let (locked, checked) = locker.join(checker).await;
+        let (locked, checked) = futures::join!(locker, checker);
         assert!(locked.is_ok());
         assert!(checked.is_ok());
     }

common/src/lib.rs

@@ -1,3 +1,4 @@
+pub mod config;
 ///
 /// Common functionality for all the components of placementd
 pub mod db;
@@ -27,21 +28,5 @@ pub fn merge_json(a: &mut serde_json::Value, b: serde_json::Value) {
     }
 }
 
-/*
-pub mod config {
-    use serde::Deserialize;
-    use url::Url;
-
-    pub fn load<'life, T>(url: &Url) -> T where T: Deserialize<'life> {
-    }
-
-    #[cfg(test)]
-    mod tests {
-        use super::*;
-
-        #[test]
-        fn test_config() {
-        }
-    }
-}
-*/
+#[cfg(test)]
+mod tests {}

config/placementd.example.yml

@@ -0,0 +1,18 @@
+# This file contains the example configuration for placementd.
+#
+---
+general:
+web:
+  databricks:
+    # Mode can be `passthrough` or `override`
+    #
+    # In the passthrough mode, placementd simply logs and proxies the request
+    # to the Databricks api
+    #
+    # In the override mode, placementd behaves like a Databricks REST API but
+    # services the job submission itself
+    mode: passthrough
+    # Root URL of the Databricks workspace for sending requests
+    url: 'https://your-workspace.cloud.databricks.com/'
+    # This can be set 
+    token_env: DATABRICKS_API_TOKEN

workers/kube-provisioner/src/main.rs

@@ -10,9 +10,11 @@ use uuid::Uuid;
 
 use std::fs::File;
 
-
-use kube::{Client, api::{Api, ResourceExt, ListParams, PostParams}};
 use k8s_openapi::api::core::v1::Pod;
+use kube::{
+    api::{Api, ListParams, PostParams, ResourceExt},
+    Client,
+};
 
 #[async_std::main]
 async fn main() -> Result<(), Box<dyn std::error::Error>> {
@@ -120,7 +122,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
 async fn dispatch_task(_tx: &mut Transaction<'_>, task: Task, _resources: &Vec<DynamicObject>) {
     debug!("Dispatching task ident {:?}", task.ident);
 
-     // Infer the runtime environment and try to create a Kubernetes Client
+    // Infer the runtime environment and try to create a Kubernetes Client
     if let Ok(client) = Client::try_default().await {
         // Read pods in the configured namespace into the typed interface from k8s-openapi
         let pods: Api<Pod> = Api::default_namespaced(client);