Exploring using openapi and schemathesis to describe the parser service API

Running via:
    schemathesis run ./services/parser/apispec.yml --base-url=http://localhost:7672 --checks all

The parser service is also using `catflap` for development: https://github.com/passcod/catflap

See #13

.gitignore

@@ -7,3 +7,4 @@ build/
 .otto-ebc-history
 *.tar.gz
 tmp*
+.hypothesis/

apispec/eventbus.yml

@@ -1,200 +0,0 @@
----
-swagger: '2.0'
-info:
-  description: 'This specification describes the Otto event bus'
-  version: '1.0.0'
-  title: Otto Event Bus
-  contact:
-    email: 'rtyler@brokenco.de'
-  license:
-    name: 'GNU AGPL 3.0'
-    url: 'https://www.gnu.org/licenses/agpl-3.0.en.html'
-host: 'localhost:8080'
-externalDocs:
-  description: 'Find out more about Otto'
-  url: 'https://github.com/rtyler/otto'
-basePath: '/v1'
-schemes:
-- 'https'
-- 'http'
-
-parameters:
-  channelName:
-    name: name
-    description: 'The named identifier of an event channel'
-    in: path
-    required: true
-    type: string
-    example: 'inbound-webooks'
-
-  channelConsumer:
-    name: consumer
-    description: 'The named identifier of an event consumer'
-    in: path
-    required: true
-    type: string
-    example: 'hooks-consumer-0a'
-
-produces:
-  - 'application/json'
-
-paths:
-  /channel:
-    get:
-      summary: 'List existing channels in the event bus'
-      description: |
-        Enumerate all the channels visible and available to the current client's permission scope
-      responses:
-        200:
-          description: 'Channels successfully listed'
-          schema:
-            type: 'array'
-            items:
-              $ref: '#/definitions/Channel'
-        400:
-          description: 'Invalid request'
-
-  /channel/{name}:
-    get:
-      summary: 'Fetch the metadata about a specific channel'
-      parameters:
-        - $ref: '#/parameters/channelName'
-      responses:
-        200:
-          description: 'Successful retrieval of metadata'
-          schema:
-            $ref: '#/definitions/Channel'
-        400:
-          description: 'Invalid formatted channel name or request'
-        403:
-          description: 'User is not authorized to access the channel'
-        404:
-          description: 'Could not find the named channel'
-
-
-    put:
-      summary: 'Publish an item to the channel'
-      parameters:
-        - $ref: '#/parameters/channelName'
-      responses:
-        201:
-          description: 'Successful publish of the item'
-        403:
-          description: 'User is not authorized to publish to the channel'
-        404:
-          description: 'Could not find the named channel'
-
-    post:
-      summary: 'Create a channel'
-      parameters:
-        - $ref: '#/parameters/channelName'
-      responses:
-        200:
-          description: 'Channel created successfully'
-        400:
-          description: 'Suggested channel configuration was invalid'
-        403:
-          description: 'User is not authorized to create a channel'
-
-    patch:
-      summary: 'Modify the channel configuration'
-      parameters:
-        - $ref: '#/parameters/channelName'
-      responses:
-        200:
-          description: 'Successful update of the channel'
-        400:
-          description: 'Suggested channel configuration was invalid'
-        403:
-          description: 'User is not authorized to modify the channel'
-        404:
-          description: 'Could not find the named channel'
-
-  /channel/{name}/{offset}:
-    get:
-      summary: 'Fetch an item from the channel'
-      parameters:
-        - $ref: '#/parameters/channelName'
-        - name: offset
-          description: 'The offset at which the item is located in the channel'
-          in: path
-          required: true
-          type: integer
-          format: int64
-      responses:
-        200:
-          description: 'Successful fetch of the item'
-        404:
-          description: 'Could not find the named channel'
-        416:
-          description: 'Could not find an item at the given offset'
-
-  /offset/{consumer}:
-    get:
-      summary: 'List offset metadata about a named consumer'
-      parameters:
-        - $ref: '#/parameters/channelConsumer'
-      responses:
-        200:
-          description: 'Successful access of the consumer metadata'
-        400:
-          description: 'Improperly formatted consumer name'
-        403:
-          description: 'User is not authorized to access this consumer'
-        404:
-          description: 'Could not find the named consumer'
-
-    post:
-      summary: 'Create a named consumer to store metadata'
-      parameters:
-        - $ref: '#/parameters/channelConsumer'
-      responses:
-        200:
-          description: 'Successful creation of the named consumer'
-        400:
-          description: 'Improperly formatted consumer metadata'
-        403:
-          description: 'User is not authorized to create a consumer'
-        409:
-          description: 'The named consumer already exists and is in use'
-
-    patch:
-      summary: 'Update the offset for the named consumer'
-      parameters:
-        - $ref: '#/parameters/channelConsumer'
-      responses:
-        200:
-          description: 'Successful modification of the consumer metadata'
-        400:
-          description: 'Improperly formatted metadata'
-        403:
-          description: 'User is not authorized to modify this consumer'
-        404:
-          description: 'Could not find the named consumer'
-definitions:
-  Channel:
-    type: object
-    xml:
-      name: 'Channel'
-    properties:
-      id:
-        type: integer
-        format: int64
-      name:
-        type: string
-        description: 'Name for the channel'
-        example: 'inbound-hooks'
-      consumers:
-        type: integer
-        format: int64
-        description: 'Number of current consumers'
-      updatedAt:
-        type: string
-        format: date-time
-        description: 'Last time the channel metadata was updated'
-      status:
-        type: string
-        description: "The channel's status"
-        enum:
-        - 'ready'
-        - 'unavailable'

apispec/orchestrator.yml

@@ -1,117 +0,0 @@
----
-swagger: '2.0'
-info:
-  description: 'This specification describes the Otto orchestrator'
-  version: '1.0.0'
-  title: Otto Orchestrator
-  contact:
-    email: 'rtyler@brokenco.de'
-  license:
-    name: 'GNU AGPL 3.0'
-    url: 'https://www.gnu.org/licenses/agpl-3.0.en.html'
-host: 'localhost:3030'
-externalDocs:
-  description: 'Find out more about Otto'
-  url: 'https://github.com/rtyler/otto'
-basePath: '/v1'
-schemes:
-- 'http'
-paths:
-  /manifest/{agentId}:
-    get:
-      summary: 'Fetch manifest for execution by the given agent'
-      description: |
-        Return the full execution manifest for the given agent to execute.
-      operationId: 'fetchManifest'
-      produces:
-        - 'application/json'
-      parameters:
-        - name: agentId
-          in: path
-          required: true
-          type: string
-          x-example: otto-agent-1
-      responses:
-        200:
-          description: 'Agent ID found and manifest generated'
-          # https://github.com/apiaryio/dredd/issues/553#issuecomment-412265413
-          headers:
-            Content-Type:
-              type: string
-              default: application/json; charset=utf-8
-          schema:
-            $ref: '#/definitions/Manifest'
-        400:
-          description: 'Invalid request'
-definitions:
-  Manifest:
-    type: object
-    description: 'Agent execution manifest'
-    xml:
-      name: 'Manifest'
-    properties:
-      self:
-        type: string
-        description: 'The identifier of the agent'
-      services:
-        type: object
-        $ref: '#/definitions/Service'
-      ops:
-        type: array
-        items:
-          $ref: '#/definitions/Operation'
-    x-example:
-      self: 'otto-agent-1'
-      services:
-        $ref: '#/definitions/Service'
-      ops:
-        $ref: '#/definitions/Operation'
-
-  Service:
-    type: object
-    description: 'A service ID to URl mapping'
-    xml:
-      name: 'Service'
-    properties:
-      identifier:
-        type: string
-        description: 'Key to identify the different services'
-      url:
-        type: string
-        description: 'Resolvable URL to access APIs for the given service'
-    example:
-      datastore: 'http://localhost:3031/'
-
-  Operation:
-    type: object
-    description: 'A discrete idempotent operation'
-    properties:
-      id:
-        type: string
-        description: 'Globally unique ID to identify this specific operation in data stores, etc'
-      context:
-        type: string
-        description: 'Generally unique context ID to group different operations in the same context'
-      type:
-        type: string
-        description: 'Type of operation'
-        $ref: '#/definitions/OperationType'
-      data:
-        type: object
-        description: 'Operation type-specific data for the agent to use'
-    example:
-      id: '0xdeadbeef'
-      context: '0x1'
-      type: 'RUNPROC'
-      data:
-        script: 'echo "Hello World"'
-        env:
-        timeout_s: 600
-
-  OperationType:
-    type: string
-    description: 'Specific type of the given operation, implies different `data` fields'
-    enum:
-      - 'BEGINCTX'
-      - 'ENDCTX'
-      - 'RUNPROC'

config/custom-environment-variables.yml

@@ -1,4 +0,0 @@
----
-orchestrator:
-  host: OR_HOST
-  port: OR_PORT

config/default.yml

@@ -1,4 +0,0 @@
----
-orchestrator:
-  host: localhost
-  port: 3030

eventbus.yml

@@ -1,22 +0,0 @@
-# This file is an example otto-eventbus configuration file
----
-motd: 'Starting otto-eventbus'
-# Time in seconds for the heartbeat to pulse on all connected clients
-heartbeat: 30
-# Channels are available for different types of information. Clients are able
-# to read from some, none, or all channels.
-#
-# NOTE: there is a default "all" channel which is used to broadcast information
-# to all clients.
-#
-# WARN: Channels are not merged with the default configuration, and will overwrite
-# channels:
-#   stateful:
-#     - audit
-#     - tasks.for_auction
-#     - tasks.auction
-#     - tasks.bids
-#   stateless:
-#     - all
-#     - tasks.started
-#     - tasks.finished

grammar/Makefile

@@ -1,39 +0,0 @@
-# List of .g4 files to compile
-GRAMMAR=Otto.g4 OttoLexer.g4
-# Target languages for the grammars
-LANGS=JavaScript Java Cpp Go
-# Antlr binary for execution
-ANTLR_BIN=antlr-4.7.2-complete.jar
-ANTLR=../contrib/$(ANTLR_BIN)
-
-################################################################################
-## Phony targets
-################################################################################
-
-# Cute hack thanks to:
-# https://marmelab.com/blog/2016/02/29/auto-documented-makefile.html
-help: ## Display this help text
-	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
-
-
-build: $(GRAMMAR) $(ANTLR) ## Compile the grammars into their language stubs
-	@for target in $(LANGS); do \
-		java -cp $(ANTLR) org.antlr.v4.Tool \
-			-Dlanguage=$$target \
-			-o build/parser/$$target \
-			$(GRAMMAR); \
-		echo "--> Generated $$target stubs"; \
-	done;
-
-
-
-clean: ## Clean all temporary/working files
-	rm -f $(ANTLR)
-
-
-.PHONY: help
-################################################################################
-
-
-$(ANTLR): ## Download the latest ANTLR4 binary
-	wget -O $(ANTLR) https://www.antlr.org/download/$(ANTLR_BIN)

grammar/Otto.g4

@@ -1,272 +0,0 @@
-/*
- * This file contains the parser for the Otto description language
- *
- * This is to be considered the reference grammar for all .otto files
- */
-parser grammar Otto;
-
-options {
-    tokenVocab=OttoLexer;
-}
-
-// Start rule to parse the .otto pipeline declaration
-pipeline
-    : use_block?
-    configure_block?
-    envs_block?
-    pipeline_block
-    ;
-
-/*
- * The use {} block helps bring user defined libraries into scope for the
- * runtime of the pipeline, but does not influence parse time
- *
- * Example:
-    use {
-      stdlib
-    }
- *
- */
-use_block
-    : USE BEGIN statements? END
-    ;
-
-/*
- * The configure {} block allows the user to configure libraries or other
- * pipeline-specific settings.
- *
- * Example:
-    configure {
-      slack {
-        channel = '#otto'
-      }
-    }
- */
-configure_block
-    : CONFIGURE BEGIN setting_block* END
-    ;
-
-/* The environments {} block allows the definition of logical environments for
- * the pipeline to deliver into
- *
- * Example:
-    environments {
-      preprod {
-        settings {
-          HOSTNAME = "preprod-ottoapp.herokuapp.com"
-        }
-      }
-    }
- */
-envs_block
-    : ENVIRONMENTS BEGIN env_block+ END
-    ;
-
-/*
- * Handling an identified environment block.
- *
- * This block is typically responsible for configuring a single target
- * environment for the delivery of this pipeline.
- *
- * Example:
-    preprod {
-      settings {
-        HOSTNAME = "preprod-ottoapp.herokuapp.com"
-      }
-    }
- */
-env_block
-    : ID BEGIN settings_block? END
-    ;
-settings_block
-    :  SETTINGS BEGIN settings? END
-    ;
-
-/*
- * Set settings for an identified subcomponent
- *
- * Example:
-      slack {
-        channel = '#otto'
-      }
- *
- * The identified subcomponent is not expected to be known at parse time, but
- * should be looked up when the parsed .otto file has been processed to ensure
- * that a corresponding subcomponent is available
- */
-setting_block
-    : ID BEGIN settings? END
-    ;
-settings
-    : setting+
-    ;
-setting
-    : ID ASSIGN (StringLiteral | array | macro | macroKeywords)
-    ;
-array
-    : ARRAY_START (StringLiteral COMMA?)+ ARRAY_END
-    ;
-
-
-
-/*
- * The pipeline {} block contains the main execution definition of the
- * pipeline. Roughly modeled after the Jenkins Pipeline declarative syntax.
- */
-pipeline_block
-    : PIPELINE BEGIN stages_block END
-    ;
-
-stages_block
-    : STAGES BEGIN (macro? stages macro?)+ END
-    ;
-stages
-    : STAGE BEGIN stageStatements* END
-    ;
-
-stageStatements
-    : settings
-    | steps
-    | runtime
-    | cache
-    | gates
-    | deployExpr
-    | notifyExpr
-    | macro+
-    // And finally, allow nesting our stages!
-    | stages+
-    ;
-steps
-    : STEPS BEGIN statements+ END
-    ;
-cache
-    : CACHE BEGIN
-        (
-        (setting+)
-        | fromExpr
-        | cacheUseExpr
-        )
-     END
-    ;
-/*
- * cache {} `use` expressions allow stages to pull in cached entries from
- * elsewhere
- */
-cacheUseExpr
-    : USE ID
-    ;
-
-runtime
-    : RUNTIME BEGIN
-        (
-        setting_block
-        | fromExpr
-        )
-    END
-    ;
-/*
- * XXX: This syntax requires some test coverage to ensure that the grammar
- * allows for order independence properly, while still restricting only a
- * single enter block, for example
- */
-gates
-    : GATES BEGIN
-    (
-    enter
-    | exit
-    | fromExpr
-    )+
-    END
-    ;
-enter
-    : ENTER BEGIN enterExpr+ END
-    ;
-exit
-    : EXIT BEGIN exitExpr+ END
-    ;
-enterExpr
-    : (BRANCH EQUALS StringLiteral)
-    | statements
-    | setting_block
-    ;
-exitExpr
-    : statements
-    | setting_block
-    ;
-
-/*
- * A "deployment expression" signifies that the output of the given context
- * will result in binaries or some form of delivery to the environment being
- * pointed to
- */
-deployExpr
-    : ENVIRONMENT TO ID
-    ;
-
-
-notifyExpr
-    : NOTIFY BEGIN
-        (
-        (SUCCESS | FAILURE | COMPLETE)
-        BEGIN
-        statements+
-        END
-        )+
-    END
-    ;
-
-
-/*
- * A "from" expression is a shorthand in the syntax for coping the contents of
- * another block of "this" type, from another stage or location
- *
- * For exmaple, if one stage in the pipeline has a `cache` configuration
- * defined, a later stage can use: cache { from 'StageA' } to copy the settings
- * over verbatim
- */
-fromExpr
-    : FROM StringLiteral
-    ;
-
-statements
-    : statement+
-    ;
-statement
-    : keyword
-    | step
-    | StringLiteral
-    ;
-
-step
-    : ID StringLiteral
-    ;
-
-
-/*
- * Macro expressions can be a single line, or a single line with a block
- * attached
- */
-macro
-    : ID OPEN macroArguments CLOSE
-    (BEGIN (stages+)? END)?
-    ;
-macroArguments
-    :
-    (
-        (StringLiteral | macroKeywords)
-    COMMA?
-    )+
-    ;
-macroKeywords
-    : IT
-    ;
-
-
-
-/*
- * Keywords are expected to be semantically important after parse time and
- * effectively represent reserved words in the .otto language
- */
-keyword
-    : STDLIB
-    ;

grammar/OttoLexer.g4

@@ -1,112 +0,0 @@
-lexer grammar OttoLexer;
-
-USE : 'use';
-CONFIGURE : 'configure';
-ENVIRONMENTS : 'environments';
-ENVIRONMENT : 'environment';
-SETTINGS : 'settings';
-PIPELINE : 'pipeline';
-STAGES : 'stages';
-STAGE : 'stage';
-STEPS : 'steps';
-CACHE : 'cache';
-RUNTIME : 'runtime';
-
-NOTIFY : 'notify';
-SUCCESS : 'success';
-FAILURE : 'failure';
-COMPLETE : 'complete';
-
-GATES : 'gates';
-ENTER : 'enter';
-EXIT : 'exit';
-BRANCH : 'branch';
-EQUALS : '==';
-
-/*
- * The "to" token helps signify the output of the current context going "to" a
- * designated environment
- */
-TO : '->';
-
-FROM : 'from';
-
-// Keyword tokens
-STDLIB: 'stdlib';
-
-// Begin block
-BEGIN : '{';
-// End block
-END : '}';
-OPEN : '(';
-CLOSE : ')';
-ARRAY_START : '[';
-ARRAY_END : ']';
-COMMA : ',';
-ASSIGN : '=';
-
-IT : 'it';
-
-StringLiteral: ('"' DoubleStringCharacter* '"'
-             |  '\'' SingleStringCharacter* '\'')
-;
-
-fragment DoubleStringCharacter
-    : ~["\\\r\n]
-    | '\\' EscapeSequence
-    | LineContinuation
-    ;
-fragment SingleStringCharacter
-    : ~['\\\r\n]
-    | '\\' EscapeSequence
-    | LineContinuation
-    ;
-fragment EscapeSequence
-    : CharacterEscapeSequence
-    | '0' // no digit ahead! TODO
-    | HexEscapeSequence
-    | UnicodeEscapeSequence
-    | ExtendedUnicodeEscapeSequence
-    ;
-
-fragment CharacterEscapeSequence
-    : SingleEscapeCharacter
-    | NonEscapeCharacter
-    ;
-fragment HexEscapeSequence
-    : 'x' HexDigit HexDigit
-    ;
-fragment UnicodeEscapeSequence
-    : 'u' HexDigit HexDigit HexDigit HexDigit
-    ;
-fragment ExtendedUnicodeEscapeSequence
-    : 'u' '{' HexDigit+ '}'
-    ;
-
-fragment HexDigit
-    : [0-9a-fA-F]
-    ;
-fragment SingleEscapeCharacter
-    : ['"\\bfnrtv]
-;
-fragment NonEscapeCharacter
-    : ~['"\\bfnrtv0-9xu\r\n]
-    ;
-fragment EscapeCharacter
-    : SingleEscapeCharacter
-    | [0-9]
-    | [xu]
-    ;
-
-fragment LineContinuation
-    : '\\' [\r\n\u2028\u2029]
-    ;
-
-
-
-ID : [a-zA-Z_]+ ;
-
-// skip spaces, tabs, newlines
-WS : [ \t\r\n]+ -> skip ;
-MultiLineComment:               '/*' .*? '*/'             -> channel(HIDDEN);
-SingleLineComment: '//' ~[\r\n\u2028\u2029]* -> channel(HIDDEN);

grammar/README.adoc

@@ -1,6 +0,0 @@
-= Otto Grammars
-
-This directory contains the link:https://github.com/antlr/antlr4/[Antlr v4]
-grammars for parsing the modeling language that Otto uses for describing a
-continuous delivery process.
-

services/parser/apispec.yml

@@ -0,0 +1,76 @@
+---
+openapi: 3.0.0
+info:
+  title: Otto Parser Service
+  description: |
+    This specification describes the Otto Parser service which is responsible
+    for ingesting Otto Pipeline syntax (typically .otto files) and outputs
+    the internal Otto intermediate representation.
+  version: '1.0.0'
+  contact:
+    name: R Tyler Croy
+    email: 'rtyler@brokenco.de'
+    x-twitter: agentdero
+  license:
+    name: 'GNU AGPL 3.0'
+    url: 'https://www.gnu.org/licenses/agpl-3.0.en.html'
+externalDocs:
+  description: 'Find out more about Otto'
+  url: 'https://github.com/rtyler/otto'
+servers:
+  - url: 'http://localhost:7672'
+    description: 'Local dev server'
+paths:
+  '/v1/parse':
+    post:
+      operationId: ParsePipeline
+      description: |
+        The primary interface for the parser service which takes an uploaded Otto
+        Pipeline string and will attempt to parse the pipeline into an intermediate
+        representation which other parts of Otto can work with.
+      requestBody:
+        description: Pipeline syntax
+        required: true
+        content:
+          text/plain:
+            schema:
+              type: string
+
+            examples:
+              success:
+                summary: 'Simple Empty Pipeline'
+                value: |
+                  pipeline {
+                  }
+
+      responses:
+        '200':
+          description: Successfully parsed
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/ParsePipelineResponse'
+        '400':
+          description: Failed to parse the pipeline for some reason
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/ParsePipelineFailure'
+components:
+  schemas:
+    ParsePipelineResponse:
+      description: |
+        This response is passed on a successful parse of the provided pipeline
+      type: object
+      required:
+        - meta
+      example: {}
+      properties:
+        meta:
+          type: object
+
+    ParsePipelineFailure:
+      type: object
+      example: {}
+      properties: {}
+

services/parser/src/main.rs

@@ -3,7 +3,46 @@
  * readable (YAML) structures which other components in Otto can use.
  */
 
+use otto_parser::*;
+use tide::{Request, Response};
+
+async fn parse(mut req: Request<()>) -> tide::Result {
+    let buffer = req.body_string().await?;
+
+    let parsed = parse_pipeline_string(&buffer);
+
+    match parsed {
+        Err(e) => {
+            let resp = Response::builder(400)
+                .body("{}")
+                .content_type("application/json")
+                .build();
+            return Ok(resp);
+        },
+        Ok(pipeline) => {
+            let resp = Response::builder(200)
+                        .body(r#"{"meta" : {}}"#)
+                        .content_type("application/json")
+                        .build();
+            return Ok(resp);
+        }
+    }
+}
+
+
 #[async_std::main]
-async fn main() -> std::io::Result<()> {
+async fn main() -> Result<(), std::io::Error> {
+    use std::{env, net::TcpListener, os::unix::io::FromRawFd};
+    tide::log::start();
+    let mut app = tide::new();
+    app.at("/v1/parse").post(parse);
+
+    if let Some(fd) = env::var("LISTEN_FD").ok().and_then(|fd| fd.parse().ok()) {
+        app.listen(unsafe { TcpListener::from_raw_fd(fd) }).await?;
+    }
+    else {
+        app.listen("http://localhost:7672").await?;
+    }
     Ok(())
 }
+