Shuffle the parser to make stages truly optional, allowing inline orphan steps

This is the manifestation of @steven-terrana's feedback. Basically the pipeline
is linear from top to bottom, and the author can put un-staged steps anywhere
along the path that they want. This should make the simplest pipelines pretty
dang simple.

These steps will have to be rendered somehow by a frontend at some point, but
that's a problem for later! 😸

Cargo.lock

@@ -1550,6 +1550,7 @@ dependencies = [
  "serde_json",
  "serde_yaml",
  "tide",
+ "uuid",
 ]
 
 [[package]]

services/parser/Cargo.toml

@@ -22,3 +22,4 @@ pretty_env_logger = "~0.4.0"
 serde_json = "~1.0.59"
 serde_yaml = "~0.8.13"
 tide = "~0.14.0"
+uuid = { version = "~0.8.1", features = ["v4", "serde"]}

services/parser/apispec.yml

@@ -52,15 +52,13 @@ paths:
                 summary: 'Simple Pipeline'
                 value: |
                   pipeline {
-                      stages {
-                          stage {
-                              name = 'Build'
-                              steps {
-                                  sh 'ls'
-                              }
-                          }
-                      }
-                  }
+                        stage {
+                            name = 'Build'
+                            steps {
+                                sh 'ls'
+                            }
+                        }
+                    }
 
       responses:
         '200':

services/parser/src/lib.rs

@@ -6,6 +6,7 @@ use log::*;
 use otto_models::*;
 use pest::iterators::Pairs;
 use pest::Parser;
+use uuid::Uuid;
 
 #[derive(Parser)]
 #[grammar = "pipeline.pest"]
@@ -21,22 +22,25 @@ pub fn parse_pipeline_string(buffer: &str) -> Result<Pipeline, pest::error::Erro
 
     while let Some(parsed) = parser.next() {
         match parsed.as_rule() {
-            Rule::stages => {
-                let mut stages = parsed.into_inner();
-                while let Some(parsed) = stages.next() {
+            Rule::execBlocks => {
+                let mut parsed = parsed.into_inner();
+                while let Some(parsed) = parsed.next() {
                     match parsed.as_rule() {
+                        Rule::steps => {
+                            pipeline.steps.extend(parse_steps(&mut parsed.into_inner(), pipeline.uuid));
+                        },
                         Rule::stage => {
                             let (ctx, mut steps) = parse_stage(&mut parsed.into_inner());
                             pipeline.contexts.push(ctx);
                             pipeline.steps.append(&mut steps);
-                        }
-                        _ => {}
+                        },
+                        _ => {},
                     }
                 }
-            }
+            },
             _ => {}
         }
-    }
+    };
 
     Ok(pipeline)
 }
@@ -55,9 +59,36 @@ fn parse_str(parser: &mut pest::iterators::Pair<Rule>) -> String {
     "".to_string()
 }
 
-fn parse_stage(parser: &mut Pairs<Rule>) -> (Context, Vec<Step>) {
+/**
+ * Parse the steps
+ *
+ * In the case of orphan steps, the uuid should be the pipeline's uuid
+ */
+fn parse_steps(parser: &mut Pairs<Rule>, uuid: Uuid) -> Vec<Step> {
     use pest::iterators::Pair;
 
+    let mut steps = vec![];
+
+    while let Some(parsed) = parser.next() {
+        if Rule::step == parsed.as_rule() {
+            // Grab the step components
+            let mut parts: Vec<Pair<Rule>> = parsed.into_inner().collect();
+            // We need at least two parts here!
+            assert!(parts.len() > 1);
+
+            let symbol = parts[0].as_str().to_string();
+            let command = parse_str(&mut parts.pop().unwrap());
+
+            let parameters = serde_yaml::Value::String(command);
+            let parameters = StepParameters::Positional(vec![parameters]);
+            let step = Step::new(uuid, symbol, parameters);
+            steps.push(step);
+        }
+    }
+    steps
+}
+
+fn parse_stage(parser: &mut Pairs<Rule>) -> (Context, Vec<Step>) {
     let mut stage = Context::default();
     let mut steps: Vec<Step> = vec![];
 
@@ -86,23 +117,7 @@ fn parse_stage(parser: &mut Pairs<Rule>) -> (Context, Vec<Step>) {
             }
             Rule::steps => {
                 let mut inner = parsed.into_inner();
-
-                while let Some(parsed) = inner.next() {
-                    if Rule::step == parsed.as_rule() {
-                        // Grab the step components
-                        let mut parts: Vec<Pair<Rule>> = parsed.into_inner().collect();
-                        // We need at least two parts here!
-                        assert!(parts.len() > 1);
-
-                        let symbol = parts[0].as_str().to_string();
-                        let command = parse_str(&mut parts.pop().unwrap());
-
-                        let parameters = serde_yaml::Value::String(command);
-                        let parameters = StepParameters::Positional(vec![parameters]);
-                        let step = Step::new(stage.uuid, symbol, parameters);
-                        steps.push(step);
-                    }
-                }
+                steps.extend(parse_steps(&mut inner, stage.uuid));
             }
             _ => {}
         }
@@ -176,20 +191,18 @@ mod tests {
             Rule::pipeline,
             r#"
             pipeline {
-                stages {
-                    stage {
-                        name = 'Build'
-                        steps {
-                            sh 'ls'
-                            sh 'env'
-                        }
+                stage {
+                    name = 'Build'
+                    steps {
+                        sh 'ls'
+                        sh 'env'
                     }
+                }
 
-                    stage {
-                        name = 'Deploy'
-                        steps {
-                            sh 'make deploy'
-                        }
+                stage {
+                    name = 'Deploy'
+                    steps {
+                        sh 'make deploy'
                     }
                 }
             }
@@ -204,12 +217,10 @@ mod tests {
     fn parse_simple_pipeline() {
         let buf = r#"
             pipeline {
-                stages {
-                    stage {
-                        name = 'Build'
-                        steps {
-                            sh 'ls'
-                        }
+                stage {
+                    name = 'Build'
+                    steps {
+                        sh 'ls'
                     }
                 }
             }"#;
@@ -225,19 +236,17 @@ mod tests {
     fn parse_more_pipeline() {
         let buf = r#"
             pipeline {
-                stages {
-                    stage {
-                        name = 'Build'
-                        steps {
-                            sh 'ls'
-                        }
+                stage {
+                    name = 'Build'
+                    steps {
+                        sh 'ls'
                     }
-                    stage {
-                        name = 'Deploy'
-                        steps {
-                            sh 'ls -lah && touch deploy.lock'
-                            sh 'make depoy'
-                        }
+                }
+                stage {
+                    name = 'Deploy'
+                    steps {
+                        sh 'ls -lah && touch deploy.lock'
+                        sh 'make depoy'
                     }
                 }
             }"#;
@@ -247,4 +256,18 @@ mod tests {
         assert_eq!(pipeline.contexts.len(), 2);
         assert_eq!(pipeline.steps.len(), 3);
     }
+
+    #[test]
+    fn parse_orphan_steps() {
+        let buf = r#"
+            pipeline {
+                steps {
+                    sh 'make all'
+                }
+            }"#;
+        let pipeline = parse_pipeline_string(&buf).expect("Failed to parse");
+        assert!(!pipeline.uuid.is_nil());
+        assert_eq!(pipeline.contexts.len(), 0);
+        assert_eq!(pipeline.steps.len(), 1);
+    }
 }

services/parser/src/pipeline.pest

@@ -1,8 +1,12 @@
 // The pipeline PEG 
 
-pipeline = _{ SOI ~ "pipeline" ~ BLOCK_BEGIN ~ stages+ ~ BLOCK_END ~ EOI }
+pipeline = _{ SOI ~ "pipeline" ~
+                    BLOCK_BEGIN ~
+                    execBlocks ~
+                    BLOCK_END ~ EOI }
+
+execBlocks = { (stage | steps)* }
 
-stages = { "stages" ~ BLOCK_BEGIN ~ stage+ ~ BLOCK_END }
 stage = { "stage" ~
         BLOCK_BEGIN ~ 
         (property*) ~

services/parser/test_data/valid/00_multiple_stages.otto

@@ -1,24 +1,21 @@
 pipeline {
-    stages {
-
-        stage {
-            name = 'Build'
-            steps {
-                sh 'ls'
-            }
+    stage {
+        name = 'Build'
+        steps {
+            sh 'ls'
         }
+    }
 
-        stage {
-            name
-            = 
-            'Deploy'
+    stage {
+        name
+        = 
+        'Deploy'
 
-            steps
-            {
-                sh 'ls -lah && touch deploy.lock'
+        steps
+        {
+            sh 'ls -lah && touch deploy.lock'
 
-                sh 'make deploy'
-            }
+            sh 'make deploy'
         }
     }
 }

services/parser/test_data/valid/00_simple.otto

@@ -1,10 +1,8 @@
 pipeline {
-    stages {
-        stage {
-            name = 'Build'
-            steps {
-                sh 'ls'
-            }
+    stage {
+        name = 'Build'
+        steps {
+            sh 'ls'
         }
     }
 }

services/parser/test_data/valid/00_simple_kwargs.otto

@@ -1,10 +1,8 @@
 pipeline {
-    stages {
-        stage {
-            name = 'Build'
-            steps {
-                sh script: 'ls'
-            }
+    stage {
+        name = 'Build'
+        steps {
+            sh script: 'ls'
         }
     }
 }

services/parser/test_data/valid/91_orphan_steps.otto

@@ -0,0 +1,5 @@
+pipeline {
+    steps {
+        sh 'ls'
+    }
+}