From c0e7cd0c6c63e2608a936e101840c5df91551073 Mon Sep 17 00:00:00 2001
From: shmck <shawn.j.mckay@gmail.com>
Date: Thu, 4 Jun 2020 20:27:41 -0700
Subject: [PATCH 1/9] validate yaml setup

Signed-off-by: shmck <shawn.j.mckay@gmail.com>
---
 src/build.ts                                  |  4 +++
 src/utils/commits.ts                          |  2 +-
 src/utils/parse.ts                            |  1 -
 .../{commitOrder.ts => validateCommits.ts}    |  7 +----
 src/utils/validateYaml.ts                     |  3 ++
 src/validate.ts                               | 31 ++++++++++++++++---
 tests/commitOrder.test.ts                     |  2 +-
 tests/yaml.test.ts                            |  3 ++
 8 files changed, 40 insertions(+), 13 deletions(-)
 rename src/utils/{commitOrder.ts => validateCommits.ts} (92%)
 create mode 100644 src/utils/validateYaml.ts
 create mode 100644 tests/yaml.test.ts

diff --git a/src/build.ts b/src/build.ts
index dfce242..349f0d7 100644
--- a/src/build.ts
+++ b/src/build.ts
@@ -74,6 +74,10 @@ async function build(args: string[]) {
   let config;
   try {
     config = yamlParser.load(_yaml);
+    // TODO: validate yaml
+    if (!config || !config.length) {
+      throw new Error("Invalid yaml file contents");
+    }
   } catch (e) {
     console.error("Error parsing yaml");
     console.error(e.message);
diff --git a/src/utils/commits.ts b/src/utils/commits.ts
index 2072993..15bb312 100644
--- a/src/utils/commits.ts
+++ b/src/utils/commits.ts
@@ -2,7 +2,7 @@ import * as fs from "fs";
 import util from "util";
 import * as path from "path";
 import gitP, { SimpleGit } from "simple-git/promise";
-import { addToCommitOrder, validateCommitOrder } from "./commitOrder";
+import { validateCommitOrder } from "./validateCommits";
 
 const mkdir = util.promisify(fs.mkdir);
 const exists = util.promisify(fs.exists);
diff --git a/src/utils/parse.ts b/src/utils/parse.ts
index 59886bc..e796be1 100644
--- a/src/utils/parse.ts
+++ b/src/utils/parse.ts
@@ -106,7 +106,6 @@ export function parse(params: ParseParams): any {
 
   // add init commits
   if (params.commits.INIT && params.commits.INIT.length) {
-    console.log(JSON.stringify(parsed.config?.testRunner));
     // @ts-ignore
     parsed.config.testRunner.setup = {
       ...(parsed.config?.testRunner?.setup || {}),
diff --git a/src/utils/commitOrder.ts b/src/utils/validateCommits.ts
similarity index 92%
rename from src/utils/commitOrder.ts
rename to src/utils/validateCommits.ts
index dae894d..9535968 100644
--- a/src/utils/commitOrder.ts
+++ b/src/utils/validateCommits.ts
@@ -1,9 +1,5 @@
 // should flag commits that are out of order based on the previous commit
 // position is a string like 'INIT', 'L1', 'L1S1'
-export function addToCommitOrder(position: string) {
-  // add position to list
-}
-
 export function validateCommitOrder(positions: string[]): boolean {
   // loop over positions
   const errors: number[] = [];
@@ -12,7 +8,6 @@ export function validateCommitOrder(positions: string[]): boolean {
   positions.forEach((position: string, index: number) => {
     if (position === "INIT") {
       if (previous.level !== 0 && previous.step !== 0) {
-        console.log("ERROR HERE");
         errors.push(index);
       }
       current = { level: 0, step: 0 };
@@ -46,7 +41,7 @@ export function validateCommitOrder(positions: string[]): boolean {
     previous = current;
   });
 
-  if (errors.length) {
+  if (errors.length && process.env.NODE_ENV !== "test") {
     console.warn("Found commit positions out of order");
     positions.forEach((position, index) => {
       if (errors.includes(index)) {
diff --git a/src/utils/validateYaml.ts b/src/utils/validateYaml.ts
new file mode 100644
index 0000000..6b690a8
--- /dev/null
+++ b/src/utils/validateYaml.ts
@@ -0,0 +1,3 @@
+export function validateYaml(yaml: string) {
+  return true;
+}
diff --git a/src/validate.ts b/src/validate.ts
index 029a501..7106e4a 100644
--- a/src/validate.ts
+++ b/src/validate.ts
@@ -1,23 +1,46 @@
 import * as path from "path";
 import * as fs from "fs";
 import util from "util";
+import * as yamlParser from "js-yaml";
+import { getArg } from "./utils/args";
 import gitP, { SimpleGit } from "simple-git/promise";
 import { getCommits, CommitLogObject } from "./utils/commits";
 
 const mkdir = util.promisify(fs.mkdir);
 const exists = util.promisify(fs.exists);
 const rmdir = util.promisify(fs.rmdir);
+const read = util.promisify(fs.readFile);
 
 async function validate(args: string[]) {
   // dir - default .
   const dir = !args.length || args[0].match(/^-/) ? "." : args[0];
-  console.warn("Not yet implemented. Coming soon");
-
   const localDir = path.join(process.cwd(), dir);
-  const codeBranch = "";
 
-  const commits = getCommits({ localDir, codeBranch });
+  // -y --yaml - default coderoad-config.yml
+  const options = {
+    yaml: getArg(args, { name: "yaml", alias: "y" }) || "coderoad.yaml";
+  }
+
+  const _yaml = await read(path.join(localDir, options.yaml), "utf8");
+
+  // parse yaml config
+  let config;
+  try {
+    config = yamlParser.load(_yaml)
+    // TODO: validate yaml
+    if (!config || !config.length) {
+      throw new Error('Invalid yaml file contents')
+    }
+  } catch (e) {
+    console.error("Error parsing yaml");
+    console.error(e.message);
+  }
+
+  const codeBranch: string = config.config.repo.branch;
+
   // VALIDATE SKELETON WITH COMMITS
+  const commits = getCommits({ localDir, codeBranch });
+
   // parse tutorial skeleton for order and commands
 
   // on error, warn missing level/step
diff --git a/tests/commitOrder.test.ts b/tests/commitOrder.test.ts
index ac483e0..38e1035 100644
--- a/tests/commitOrder.test.ts
+++ b/tests/commitOrder.test.ts
@@ -1,4 +1,4 @@
-import { validateCommitOrder } from "../src/utils/commitOrder";
+import { validateCommitOrder } from "../src/utils/validateCommits";
 
 describe("commitOrder", () => {
   it("should return true if order is valid", () => {
diff --git a/tests/yaml.test.ts b/tests/yaml.test.ts
new file mode 100644
index 0000000..9d4ca43
--- /dev/null
+++ b/tests/yaml.test.ts
@@ -0,0 +1,3 @@
+describe("yaml", () => {
+  it.todo("should parse a valid yaml file");
+});

From e44942d6478ef8da4e5d7383be291edb1f228fb8 Mon Sep 17 00:00:00 2001
From: shmck <shawn.j.mckay@gmail.com>
Date: Sun, 7 Jun 2020 09:35:04 -0700
Subject: [PATCH 2/9] outline yaml validation tests

Signed-off-by: shmck <shawn.j.mckay@gmail.com>
---
 tests/yaml.test.ts | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/tests/yaml.test.ts b/tests/yaml.test.ts
index 9d4ca43..aa49b3d 100644
--- a/tests/yaml.test.ts
+++ b/tests/yaml.test.ts
@@ -1,3 +1,18 @@
 describe("yaml", () => {
   it.todo("should parse a valid yaml file");
+  it.todo("should fail if version is invalid");
+  it.todo("should fail if version is missing");
+  it.todo("should fail if config is missing");
+  it.todo("should fail if config testRunner is missing");
+  it.todo("should fail if config testRunner command is missing");
+  it.todo("should fail if config testRunner args tap is missing");
+  it.todo("should fail if repo is missing");
+  it.todo("should fail if repo uri is missing");
+  it.todo("should fail if repo uri is invalid");
+  it.todo("should fail if repo branch is missing");
+  it.todo("should fial if level is missing id");
+  it.todo("should fail if level setup is invalid");
+  it.todo("should fail if step is missing id");
+  it.todo("should fail if step setup is invalid");
+  it.todo("should fail if solution setup is invalid");
 });

From dd629ddd541f384b37327e0b0414ae52d728e18e Mon Sep 17 00:00:00 2001
From: shmck <shawn.j.mckay@gmail.com>
Date: Sun, 7 Jun 2020 09:53:51 -0700
Subject: [PATCH 3/9] fix example structure

Signed-off-by: shmck <shawn.j.mckay@gmail.com>
---
 src/templates/js-mocha/coderoad.yaml | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/src/templates/js-mocha/coderoad.yaml b/src/templates/js-mocha/coderoad.yaml
index dbe78b1..9d50acf 100644
--- a/src/templates/js-mocha/coderoad.yaml
+++ b/src/templates/js-mocha/coderoad.yaml
@@ -92,8 +92,7 @@ levels:
       ## Example Four: Subtasks
       - id: L1S4
         setup:
-          commands:
-            ## A filter is a regex that limits the test results
-            - filter: "^Example 2"
-            ## A feature that shows subtasks: all filtered active test names and the status of the tests (pass/fail).
-            - subtasks: true
+          ## A filter is a regex that limits the test results
+          filter: "^Example 2"
+          ## A feature that shows subtasks: all filtered active test names and the status of the tests (pass/fail).
+          subtasks: true

From feec144fe3f256b94b391f607d5ed28640a8bdf9 Mon Sep 17 00:00:00 2001
From: shmck <shawn.j.mckay@gmail.com>
Date: Sun, 7 Jun 2020 09:58:33 -0700
Subject: [PATCH 4/9] setup initial validate schema tests

Signed-off-by: shmck <shawn.j.mckay@gmail.com>
---
 src/schema/meta.ts                   |   4 -
 src/schema/skeleton.ts               | 185 +++++++++++++++++++++++++++
 src/schema/{index.ts => tutorial.ts} |   3 +
 src/utils/validateSchema.ts          |   2 +-
 src/utils/validateSkeleton.ts        |  29 +++++
 src/utils/validateYaml.ts            |   3 -
 tests/skeleton.test.ts               |  99 ++++++++++++++
 tests/validate.test.ts               |   2 +-
 tests/yaml.test.ts                   |  18 ---
 9 files changed, 318 insertions(+), 27 deletions(-)
 create mode 100644 src/schema/skeleton.ts
 rename src/schema/{index.ts => tutorial.ts} (97%)
 create mode 100644 src/utils/validateSkeleton.ts
 delete mode 100644 src/utils/validateYaml.ts
 create mode 100644 tests/skeleton.test.ts
 delete mode 100644 tests/yaml.test.ts

diff --git a/src/schema/meta.ts b/src/schema/meta.ts
index 4fd1520..71bf7b8 100644
--- a/src/schema/meta.ts
+++ b/src/schema/meta.ts
@@ -1,9 +1,6 @@
 export default {
   $schema: "http://json-schema.org/draft-07/schema#",
   $id: "https://coderoad.io/tutorial-schema.json",
-  title: "Tutorial Schema",
-  description:
-    "A CodeRoad tutorial schema data. This JSON data is converted into a tutorial with the CodeRoad editor extension",
   definitions: {
     semantic_version: {
       type: "string",
@@ -48,7 +45,6 @@ export default {
         "An array of command line commands that will be called when the user enters the level or step. Currently commands are limited for security purposes",
       items: {
         type: "string",
-        enum: ["npm install"],
       },
     },
     commit_array: {
diff --git a/src/schema/skeleton.ts b/src/schema/skeleton.ts
new file mode 100644
index 0000000..a44e461
--- /dev/null
+++ b/src/schema/skeleton.ts
@@ -0,0 +1,185 @@
+import meta from "./meta";
+
+export default {
+  title: "Skeleton Schema",
+  description:
+    "A CodeRoad tutorial config schema. This data is paired up with the markdown to create a tutorial",
+  ...meta,
+  type: "object",
+  properties: {
+    version: {
+      $ref: "#/definitions/semantic_version",
+      description: "The tutorial version. Must be unique for the tutorial.",
+      examples: ["0.1.0", "1.0.0"],
+    },
+
+    // config
+    config: {
+      type: "object",
+      properties: {
+        testRunner: {
+          type: "object",
+          description: "The test runner configuration",
+          properties: {
+            command: {
+              type: "string",
+              description: "Command line to start the test runner",
+              examples: ["./node_modules/.bin/mocha"],
+            },
+            args: {
+              type: "object",
+              description:
+                "A configuration of command line args for your test runner",
+              properties: {
+                filter: {
+                  type: "string",
+                  description:
+                    "the command line arg for filtering tests with a regex pattern",
+                  examples: ["--grep"],
+                },
+                tap: {
+                  type: "string",
+                  description:
+                    "The command line arg for configuring a TAP reporter. See https://github.com/sindresorhus/awesome-tap for examples.",
+                  examples: ["--reporter=mocha-tap-reporter"],
+                },
+              },
+              additionalProperties: false,
+              required: ["tap"],
+            },
+            directory: {
+              type: "string",
+              description: "An optional folder for the test runner",
+              examples: ["coderoad"],
+            },
+            setup: {
+              $ref: "#/definitions/setup_action",
+              description:
+                "Setup commits or commands used for setting up the test runner on tutorial launch",
+            },
+          },
+          required: ["command", "args"],
+        },
+        repo: {
+          type: "object",
+          description: "The repo holding the git commits for the tutorial",
+          properties: {
+            uri: {
+              type: "string",
+              description: "The uri source of the tutorial",
+              format: "uri",
+              examples: ["https://github.com/name/tutorial-name.git"],
+            },
+            branch: {
+              description:
+                "The branch of the repo where the tutorial config file exists",
+              type: "string",
+              examples: ["master"],
+            },
+          },
+          additionalProperties: false,
+          required: ["uri", "branch"],
+        },
+
+        dependencies: {
+          type: "array",
+          description: "A list of tutorial dependencies",
+          items: {
+            type: "object",
+            properties: {
+              name: {
+                type: "string",
+                description:
+                  "The command line process name of the dependency. It will be checked by running `name --version`",
+                examples: ["node", "python"],
+              },
+              version: {
+                type: "string",
+                description:
+                  "The version requirement. See https://github.com/npm/node-semver for options",
+                examples: [">=10"],
+              },
+            },
+            required: ["name", "version"],
+          },
+        },
+        appVersions: {
+          type: "object",
+          description:
+            "A list of compatable coderoad versions. Currently only a VSCode extension.",
+          properties: {
+            vscode: {
+              type: "string",
+              description:
+                "The version range for coderoad-vscode that this tutorial is compatable with",
+              examples: [">=0.7.0"],
+            },
+          },
+        },
+      },
+      additionalProperties: false,
+      required: ["testRunner", "repo"],
+    },
+
+    // levels
+    levels: {
+      type: "array",
+      description:
+        'Levels are the stages a user goes through in the tutorial. A level may contain a group of tasks called "steps" that must be completed to proceed',
+      items: {
+        type: "object",
+        properties: {
+          id: {
+            type: "string",
+            description: "A level id",
+            examples: ["L1", "L11"],
+          },
+          setup: {
+            $ref: "#/definitions/setup_action",
+            description:
+              "An optional point for loading commits, running commands or opening files",
+          },
+          steps: {
+            type: "array",
+            items: {
+              type: "object",
+              properties: {
+                id: {
+                  type: "string",
+                  description: "A level id",
+                  examples: ["L1S1", "L11S12"],
+                },
+                setup: {
+                  allOf: [
+                    {
+                      $ref: "#/definitions/setup_action",
+                      description:
+                        "A point for loading commits. It can also run commands and/or open files",
+                    },
+                  ],
+                },
+                solution: {
+                  allOf: [
+                    {
+                      $ref: "#/definitions/setup_action",
+                      description:
+                        "The solution commits that can be loaded if the user gets stuck. It can also run commands and/or open files",
+                    },
+                    {
+                      required: [],
+                    },
+                  ],
+                },
+              },
+              required: ["id", "setup"],
+            },
+          },
+        },
+        required: ["id"],
+      },
+      minItems: 1,
+    },
+  },
+  additionalProperties: false,
+  required: ["version", "config", "levels"],
+};
diff --git a/src/schema/index.ts b/src/schema/tutorial.ts
similarity index 97%
rename from src/schema/index.ts
rename to src/schema/tutorial.ts
index 98d3fd6..ed3cc15 100644
--- a/src/schema/index.ts
+++ b/src/schema/tutorial.ts
@@ -1,6 +1,9 @@
 import meta from "./meta";
 
 export default {
+  title: "Tutorial Schema",
+  description:
+    "A CodeRoad tutorial schema data. This JSON data is converted into a tutorial with the CodeRoad editor extension",
   ...meta,
   type: "object",
   properties: {
diff --git a/src/utils/validateSchema.ts b/src/utils/validateSchema.ts
index a7ec4fc..f1537ef 100644
--- a/src/utils/validateSchema.ts
+++ b/src/utils/validateSchema.ts
@@ -1,4 +1,4 @@
-import schema from "../schema";
+import schema from "../schema/tutorial";
 
 // https://www.npmjs.com/package/ajv
 // @ts-ignore ajv typings not working
diff --git a/src/utils/validateSkeleton.ts b/src/utils/validateSkeleton.ts
new file mode 100644
index 0000000..734ca4b
--- /dev/null
+++ b/src/utils/validateSkeleton.ts
@@ -0,0 +1,29 @@
+import schema from "../schema/skeleton";
+
+// https://www.npmjs.com/package/ajv
+// @ts-ignore ajv typings not working
+import JsonSchema from "ajv";
+
+export function validateSkeleton(json: any): Boolean | PromiseLike<Boolean> {
+  // validate using https://json-schema.org/
+  const jsonSchema = new JsonSchema({
+    allErrors: true,
+    // verbose: true,
+  });
+
+  const valid = jsonSchema.validate(schema, json);
+
+  if (!valid) {
+    // log errors
+    /* istanbul ignore next */
+    if (process.env.NODE_ENV !== "test") {
+      jsonSchema.errors?.forEach((error: JsonSchema.ErrorObject) => {
+        console.warn(
+          `Validation error at ${error.dataPath} - ${error.message}`
+        );
+      });
+    }
+  }
+
+  return valid;
+}
diff --git a/src/utils/validateYaml.ts b/src/utils/validateYaml.ts
deleted file mode 100644
index 6b690a8..0000000
--- a/src/utils/validateYaml.ts
+++ /dev/null
@@ -1,3 +0,0 @@
-export function validateYaml(yaml: string) {
-  return true;
-}
diff --git a/tests/skeleton.test.ts b/tests/skeleton.test.ts
new file mode 100644
index 0000000..b104ef2
--- /dev/null
+++ b/tests/skeleton.test.ts
@@ -0,0 +1,99 @@
+import { validateSkeleton } from "../src/utils/validateSkeleton";
+
+const validJson = {
+  version: "0.1.0",
+  config: {
+    testRunner: {
+      directory: "coderoad",
+      setup: {
+        commands: [],
+      },
+      args: {
+        filter: "--grep",
+        tap: "--reporter=mocha-tap-reporter",
+      },
+      command: "./node_modules/.bin/mocha",
+    },
+    repo: {
+      uri: "http://github.com/somePath/toRepo.git",
+      branch: "codeBranch",
+    },
+    dependencies: [],
+    appVersions: {
+      vscode: ">=0.7.0",
+    },
+  },
+  levels: [
+    {
+      steps: [
+        {
+          id: "L1S1",
+          setup: {
+            files: ["package.json"],
+          },
+          solution: {
+            files: ["package.json"],
+          },
+        },
+        {
+          id: "L1S2",
+          setup: {
+            commands: ["npm install"],
+          },
+          solution: {
+            commands: ["npm install"],
+          },
+        },
+        {
+          id: "L1S3",
+          setup: {
+            files: ["package.json"],
+            watchers: ["package.json", "node_modules/some-package"],
+          },
+          solution: {
+            files: ["package.json"],
+          },
+        },
+        {
+          id: "L1S4",
+          setup: {
+            commands: [],
+            filter: "^Example 2",
+            subtasks: true,
+          },
+        },
+      ],
+      id: "L1",
+    },
+  ],
+};
+
+describe("validate skeleton", () => {
+  it("should fail an empty skeleton file", () => {
+    const json = {};
+
+    const valid = validateSkeleton(json);
+    expect(valid).toBe(false);
+  });
+  it("should parse a valid skeleton file", () => {
+    const json = { ...validJson };
+
+    const valid = validateSkeleton(json);
+    expect(valid).toBe(true);
+  });
+  it.todo("should fail if version is invalid");
+  it.todo("should fail if version is missing");
+  it.todo("should fail if config is missing");
+  it.todo("should fail if config testRunner is missing");
+  it.todo("should fail if config testRunner command is missing");
+  it.todo("should fail if config testRunner args tap is missing");
+  it.todo("should fail if repo is missing");
+  it.todo("should fail if repo uri is missing");
+  it.todo("should fail if repo uri is invalid");
+  it.todo("should fail if repo branch is missing");
+  it.todo("should fial if level is missing id");
+  it.todo("should fail if level setup is invalid");
+  it.todo("should fail if step is missing id");
+  it.todo("should fail if step setup is invalid");
+  it.todo("should fail if solution setup is invalid");
+});
diff --git a/tests/validate.test.ts b/tests/validate.test.ts
index 1355fae..d8b3422 100644
--- a/tests/validate.test.ts
+++ b/tests/validate.test.ts
@@ -1,7 +1,7 @@
 import * as T from "../typings/tutorial";
 import { validateSchema } from "../src/utils/validateSchema";
 
-describe("validate", () => {
+describe("validate tutorial", () => {
   it("should reject an empty tutorial", () => {
     const json = { version: "here" };
 
diff --git a/tests/yaml.test.ts b/tests/yaml.test.ts
deleted file mode 100644
index aa49b3d..0000000
--- a/tests/yaml.test.ts
+++ /dev/null
@@ -1,18 +0,0 @@
-describe("yaml", () => {
-  it.todo("should parse a valid yaml file");
-  it.todo("should fail if version is invalid");
-  it.todo("should fail if version is missing");
-  it.todo("should fail if config is missing");
-  it.todo("should fail if config testRunner is missing");
-  it.todo("should fail if config testRunner command is missing");
-  it.todo("should fail if config testRunner args tap is missing");
-  it.todo("should fail if repo is missing");
-  it.todo("should fail if repo uri is missing");
-  it.todo("should fail if repo uri is invalid");
-  it.todo("should fail if repo branch is missing");
-  it.todo("should fial if level is missing id");
-  it.todo("should fail if level setup is invalid");
-  it.todo("should fail if step is missing id");
-  it.todo("should fail if step setup is invalid");
-  it.todo("should fail if solution setup is invalid");
-});

From 8cd8a8818ab0df4e2d938b13f76186be730180f1 Mon Sep 17 00:00:00 2001
From: shmck <shawn.j.mckay@gmail.com>
Date: Sun, 7 Jun 2020 10:02:57 -0700
Subject: [PATCH 5/9] refactor schema validation

Signed-off-by: shmck <shawn.j.mckay@gmail.com>
---
 src/build.ts                  |  3 ++-
 src/utils/validateSchema.ts   |  7 ++++---
 src/utils/validateSkeleton.ts | 29 -----------------------------
 tests/skeleton.test.ts        |  5 ++++-
 tests/validate.test.ts        |  7 +++++--
 5 files changed, 15 insertions(+), 36 deletions(-)
 delete mode 100644 src/utils/validateSkeleton.ts

diff --git a/src/build.ts b/src/build.ts
index 349f0d7..8512132 100644
--- a/src/build.ts
+++ b/src/build.ts
@@ -5,6 +5,7 @@ import * as util from "util";
 import { parse } from "./utils/parse";
 import { getArg } from "./utils/args";
 import { getCommits, CommitLogObject } from "./utils/commits";
+import tutorialSchema from "./schema/tutorial";
 import { validateSchema } from "./utils/validateSchema";
 import * as T from "../typings/tutorial";
 
@@ -112,7 +113,7 @@ async function build(args: string[]) {
 
   // validate tutorial based on json schema
   try {
-    const valid = validateSchema(tutorial);
+    const valid = validateSchema(tutorialSchema, tutorial);
     if (!valid) {
       console.error("Tutorial validation failed. See above to see what to fix");
       return;
diff --git a/src/utils/validateSchema.ts b/src/utils/validateSchema.ts
index f1537ef..aeca139 100644
--- a/src/utils/validateSchema.ts
+++ b/src/utils/validateSchema.ts
@@ -1,10 +1,11 @@
-import schema from "../schema/tutorial";
-
 // https://www.npmjs.com/package/ajv
 // @ts-ignore ajv typings not working
 import JsonSchema from "ajv";
 
-export function validateSchema(json: any): boolean | PromiseLike<boolean> {
+export function validateSchema(
+  schema: any,
+  json: any
+): boolean | PromiseLike<boolean> {
   // validate using https://json-schema.org/
   const jsonSchema = new JsonSchema({
     allErrors: true,
diff --git a/src/utils/validateSkeleton.ts b/src/utils/validateSkeleton.ts
deleted file mode 100644
index 734ca4b..0000000
--- a/src/utils/validateSkeleton.ts
+++ /dev/null
@@ -1,29 +0,0 @@
-import schema from "../schema/skeleton";
-
-// https://www.npmjs.com/package/ajv
-// @ts-ignore ajv typings not working
-import JsonSchema from "ajv";
-
-export function validateSkeleton(json: any): Boolean | PromiseLike<Boolean> {
-  // validate using https://json-schema.org/
-  const jsonSchema = new JsonSchema({
-    allErrors: true,
-    // verbose: true,
-  });
-
-  const valid = jsonSchema.validate(schema, json);
-
-  if (!valid) {
-    // log errors
-    /* istanbul ignore next */
-    if (process.env.NODE_ENV !== "test") {
-      jsonSchema.errors?.forEach((error: JsonSchema.ErrorObject) => {
-        console.warn(
-          `Validation error at ${error.dataPath} - ${error.message}`
-        );
-      });
-    }
-  }
-
-  return valid;
-}
diff --git a/tests/skeleton.test.ts b/tests/skeleton.test.ts
index b104ef2..43113b5 100644
--- a/tests/skeleton.test.ts
+++ b/tests/skeleton.test.ts
@@ -1,4 +1,7 @@
-import { validateSkeleton } from "../src/utils/validateSkeleton";
+import { validateSchema } from "../src/utils/validateSchema";
+import skeletonSchema from "../src/schema/skeleton";
+
+const validateSkeleton = (json: any) => validateSchema(skeletonSchema, json);
 
 const validJson = {
   version: "0.1.0",
diff --git a/tests/validate.test.ts b/tests/validate.test.ts
index d8b3422..2f7dde7 100644
--- a/tests/validate.test.ts
+++ b/tests/validate.test.ts
@@ -1,11 +1,14 @@
 import * as T from "../typings/tutorial";
+import tutorialSchema from "../src/schema/tutorial";
 import { validateSchema } from "../src/utils/validateSchema";
 
+const validateTutorial = (json: any) => validateSchema(tutorialSchema, json);
+
 describe("validate tutorial", () => {
   it("should reject an empty tutorial", () => {
     const json = { version: "here" };
 
-    const valid = validateSchema(json);
+    const valid = validateTutorial(json);
     expect(valid).toBe(false);
   });
   it("should return true for a valid tutorial", () => {
@@ -45,7 +48,7 @@ describe("validate tutorial", () => {
       ],
     };
 
-    const valid = validateSchema(json);
+    const valid = validateTutorial(json);
     expect(valid).toBe(true);
   });
 });

From 429cdf8398f39637fd9b32816d4381bd285b763a Mon Sep 17 00:00:00 2001
From: shmck <shawn.j.mckay@gmail.com>
Date: Sun, 7 Jun 2020 10:12:26 -0700
Subject: [PATCH 6/9] skeleton validation tests

Signed-off-by: shmck <shawn.j.mckay@gmail.com>
---
 tests/skeleton.test.ts | 198 +++++++++++++++++++++++++++++++++++++----
 1 file changed, 183 insertions(+), 15 deletions(-)

diff --git a/tests/skeleton.test.ts b/tests/skeleton.test.ts
index 43113b5..fdcfeb2 100644
--- a/tests/skeleton.test.ts
+++ b/tests/skeleton.test.ts
@@ -84,19 +84,187 @@ describe("validate skeleton", () => {
     const valid = validateSkeleton(json);
     expect(valid).toBe(true);
   });
-  it.todo("should fail if version is invalid");
-  it.todo("should fail if version is missing");
-  it.todo("should fail if config is missing");
-  it.todo("should fail if config testRunner is missing");
-  it.todo("should fail if config testRunner command is missing");
-  it.todo("should fail if config testRunner args tap is missing");
-  it.todo("should fail if repo is missing");
-  it.todo("should fail if repo uri is missing");
-  it.todo("should fail if repo uri is invalid");
-  it.todo("should fail if repo branch is missing");
-  it.todo("should fial if level is missing id");
-  it.todo("should fail if level setup is invalid");
-  it.todo("should fail if step is missing id");
-  it.todo("should fail if step setup is invalid");
-  it.todo("should fail if solution setup is invalid");
+  it("should fail if version is invalid", () => {
+    const json = { ...validJson, version: "NOT A VERSION" };
+
+    const valid = validateSkeleton(json);
+    expect(valid).toBe(false);
+  });
+  it("should fail if version is missing", () => {
+    const json = { ...validJson, version: undefined };
+
+    const valid = validateSkeleton(json);
+    expect(valid).toBe(false);
+  });
+  it("should fail if config is missing", () => {
+    const json = { ...validJson, config: undefined };
+
+    const valid = validateSkeleton(json);
+    expect(valid).toBe(false);
+  });
+  it("should fail if config testRunner is missing", () => {
+    const json = {
+      ...validJson,
+      config: { ...validJson.config, testRunner: undefined },
+    };
+
+    const valid = validateSkeleton(json);
+    expect(valid).toBe(false);
+  });
+  it("should fail if config testRunner command is missing", () => {
+    const json = {
+      ...validJson,
+      config: {
+        ...validJson.config,
+        testRunner: { ...validJson.config.testRunner, command: undefined },
+      },
+    };
+
+    const valid = validateSkeleton(json);
+    expect(valid).toBe(false);
+  });
+  it("should fail if config testRunner args tap is missing", () => {
+    const json = {
+      ...validJson,
+      config: {
+        ...validJson.config,
+        testRunner: {
+          ...validJson.config.testRunner,
+          args: { ...validJson.config.testRunner.args, tap: undefined },
+        },
+      },
+    };
+
+    const valid = validateSkeleton(json);
+    expect(valid).toBe(false);
+  });
+  it("should fail if repo is missing", () => {
+    const json = {
+      ...validJson,
+      config: {
+        ...validJson.config,
+        repo: undefined,
+      },
+    };
+
+    const valid = validateSkeleton(json);
+    expect(valid).toBe(false);
+  });
+  it("should fail if repo uri is missing", () => {
+    const json = {
+      ...validJson,
+      config: {
+        ...validJson.config,
+        repo: { ...validJson.config.repo, uri: undefined },
+      },
+    };
+
+    const valid = validateSkeleton(json);
+    expect(valid).toBe(false);
+  });
+  it("should fail if repo uri is invalid", () => {
+    const json = {
+      ...validJson,
+      config: {
+        ...validJson.config,
+        repo: { ...validJson.config.repo, uri: "NOT A VALID URI" },
+      },
+    };
+
+    const valid = validateSkeleton(json);
+    expect(valid).toBe(false);
+  });
+  it("should fail if repo branch is missing", () => {
+    const json = {
+      ...validJson,
+      config: {
+        ...validJson.config,
+        repo: { ...validJson.config.repo, branch: undefined },
+      },
+    };
+
+    const valid = validateSkeleton(json);
+    expect(valid).toBe(false);
+  });
+  it("should fial if level is missing id", () => {
+    const level1 = { ...validJson.levels[0], id: undefined };
+    const json = {
+      ...validJson,
+      levels: [level1],
+    };
+
+    const valid = validateSkeleton(json);
+    expect(valid).toBe(false);
+  });
+  it("should fail if level setup is invalid", () => {
+    const level1 = { ...validJson.levels[0], setup: { invalidThing: [] } };
+    const json = {
+      ...validJson,
+      levels: [level1],
+    };
+
+    const valid = validateSkeleton(json);
+    expect(valid).toBe(false);
+  });
+  it("should fail if step is missing id", () => {
+    const step1 = { ...validJson.levels[0].steps[0], id: undefined };
+    const level1 = { ...validJson.levels[0], steps: [step1] };
+    const json = {
+      ...validJson,
+      levels: [level1],
+    };
+
+    const valid = validateSkeleton(json);
+    expect(valid).toBe(false);
+  });
+  it("should fail if step setup is missing", () => {
+    const step1 = { ...validJson.levels[0].steps[0], setup: undefined };
+    const level1 = { ...validJson.levels[0], steps: [step1] };
+    const json = {
+      ...validJson,
+      levels: [level1],
+    };
+
+    const valid = validateSkeleton(json);
+    expect(valid).toBe(false);
+  });
+  it("should fail if step setup is invalid", () => {
+    const step1 = {
+      ...validJson.levels[0].steps[0],
+      setup: { invalidThing: [] },
+    };
+    const level1 = { ...validJson.levels[0], steps: [step1] };
+    const json = {
+      ...validJson,
+      levels: [level1],
+    };
+
+    const valid = validateSkeleton(json);
+    expect(valid).toBe(false);
+  });
+  it("should not fail if step solution is missing", () => {
+    const step1 = { ...validJson.levels[0].steps[0], solution: undefined };
+    const level1 = { ...validJson.levels[0], steps: [step1] };
+    const json = {
+      ...validJson,
+      levels: [level1],
+    };
+
+    const valid = validateSkeleton(json);
+    expect(valid).toBe(true);
+  });
+  it("should fail if step solution is invalid", () => {
+    const step1 = {
+      ...validJson.levels[0].steps[0],
+      solution: { invalidThing: [] },
+    };
+    const level1 = { ...validJson.levels[0], steps: [step1] };
+    const json = {
+      ...validJson,
+      levels: [level1],
+    };
+
+    const valid = validateSkeleton(json);
+    expect(valid).toBe(false);
+  });
 });

From bc4dd2cb8bd039146bb46cde8cd6f4f88983f1f1 Mon Sep 17 00:00:00 2001
From: shmck <shawn.j.mckay@gmail.com>
Date: Sun, 7 Jun 2020 10:18:18 -0700
Subject: [PATCH 7/9] add skeleton validation

Signed-off-by: shmck <shawn.j.mckay@gmail.com>
---
 src/build.ts        | 28 ++++++++++++++++-------
 src/utils/parse.ts  | 10 ++++----
 tests/parse.test.ts | 56 ++++++++++++++++++++++-----------------------
 3 files changed, 53 insertions(+), 41 deletions(-)

diff --git a/src/build.ts b/src/build.ts
index 8512132..4691e55 100644
--- a/src/build.ts
+++ b/src/build.ts
@@ -5,6 +5,7 @@ import * as util from "util";
 import { parse } from "./utils/parse";
 import { getArg } from "./utils/args";
 import { getCommits, CommitLogObject } from "./utils/commits";
+import skeletonSchema from "./schema/skeleton";
 import tutorialSchema from "./schema/tutorial";
 import { validateSchema } from "./utils/validateSchema";
 import * as T from "../typings/tutorial";
@@ -71,12 +72,11 @@ async function build(args: string[]) {
     return;
   }
 
-  // parse yaml config
-  let config;
+  // parse yaml skeleton config
+  let skeleton;
   try {
-    config = yamlParser.load(_yaml);
-    // TODO: validate yaml
-    if (!config || !config.length) {
+    skeleton = yamlParser.load(_yaml);
+    if (!skeleton || !skeleton.length) {
       throw new Error("Invalid yaml file contents");
     }
   } catch (e) {
@@ -84,12 +84,24 @@ async function build(args: string[]) {
     console.error(e.message);
   }
 
+  // validate skeleton based on skeleton json schema
+  try {
+    const valid = validateSchema(skeletonSchema, skeleton);
+    if (!valid) {
+      console.error("Tutorial validation failed. See above to see what to fix");
+      return;
+    }
+  } catch (e) {
+    console.error("Error validating tutorial schema:");
+    console.error(e.message);
+  }
+
   // load git commits to use in parse step
   let commits: CommitLogObject;
   try {
     commits = await getCommits({
       localDir: localPath,
-      codeBranch: config.config.repo.branch,
+      codeBranch: skeleton.config.repo.branch,
     });
   } catch (e) {
     console.error("Error loading commits:");
@@ -102,7 +114,7 @@ async function build(args: string[]) {
   try {
     tutorial = await parse({
       text: _markdown,
-      config,
+      skeleton,
       commits,
     });
   } catch (e) {
@@ -111,7 +123,7 @@ async function build(args: string[]) {
     return;
   }
 
-  // validate tutorial based on json schema
+  // validate tutorial based on tutorial json schema
   try {
     const valid = validateSchema(tutorialSchema, tutorial);
     if (!valid) {
diff --git a/src/utils/parse.ts b/src/utils/parse.ts
index e796be1..a6db41f 100644
--- a/src/utils/parse.ts
+++ b/src/utils/parse.ts
@@ -90,7 +90,7 @@ export function parseMdContent(md: string): TutorialFrame | never {
 
 type ParseParams = {
   text: string;
-  config: Partial<T.Tutorial | any>;
+  skeleton: Partial<T.Tutorial | any>;
   commits: CommitLogObject;
 };
 
@@ -98,9 +98,9 @@ export function parse(params: ParseParams): any {
   const mdContent: TutorialFrame = parseMdContent(params.text);
 
   const parsed: Partial<T.Tutorial> = {
-    version: params.config.version,
+    version: params.skeleton.version,
     summary: mdContent.summary,
-    config: params.config.config || {},
+    config: params.skeleton.config || {},
     levels: [],
   };
 
@@ -114,8 +114,8 @@ export function parse(params: ParseParams): any {
   }
 
   // merge content and tutorial
-  if (params.config.levels && params.config.levels.length) {
-    parsed.levels = params.config.levels
+  if (params.skeleton.levels && params.skeleton.levels.length) {
+    parsed.levels = params.skeleton.levels
       .map((level: T.Level, levelIndex: number) => {
         const levelContent = mdContent.levels[level.id];
 
diff --git a/tests/parse.test.ts b/tests/parse.test.ts
index ea84002..767b70c 100644
--- a/tests/parse.test.ts
+++ b/tests/parse.test.ts
@@ -9,10 +9,10 @@ Short description to be shown as a tutorial's subtitle.
 
 `;
 
-    const config = { version: "0.1.0" };
+    const skeleton = { version: "0.1.0" };
     const result = parse({
       text: md,
-      config,
+      skeleton,
       commits: {},
     });
     const expected = {
@@ -37,13 +37,13 @@ Description.
 Some text
 `;
 
-    const config = {
+    const skeleton = {
       levels: [{ id: "L1" }],
     };
 
     const result = parse({
       text: md,
-      config,
+      skeleton,
       commits: {},
     });
     const expected = {
@@ -73,7 +73,7 @@ Description.
 Some text
 `;
 
-    const config = {
+    const skeleton = {
       levels: [
         {
           id: "L1",
@@ -85,7 +85,7 @@ Some text
     };
     const result = parse({
       text: md,
-      config,
+      skeleton,
       commits: {},
     });
     const expected = {
@@ -115,10 +115,10 @@ Description.
 Some text that becomes the summary
 `;
 
-    const config = { levels: [{ id: "L1" }] };
+    const skeleton = { levels: [{ id: "L1" }] };
     const result = parse({
       text: md,
-      config,
+      skeleton,
       commits: {},
     });
     const expected = {
@@ -145,10 +145,10 @@ Description.
 Some text that becomes the summary and goes beyond the maximum length of 80 so that it gets truncated at the end
 `;
 
-    const config = { levels: [{ id: "L1" }] };
+    const skeleton = { levels: [{ id: "L1" }] };
     const result = parse({
       text: md,
-      config,
+      skeleton,
       commits: {},
     });
     const expected = {
@@ -180,10 +180,10 @@ Second line
 Third line
 `;
 
-    const config = { levels: [{ id: "L1" }] };
+    const skeleton = { levels: [{ id: "L1" }] };
     const result = parse({
       text: md,
-      config,
+      skeleton,
       commits: {},
     });
     const expected = {
@@ -215,7 +215,7 @@ First line
 
 The first step
 `;
-    const config = {
+    const skeleton = {
       levels: [
         {
           id: "L1",
@@ -229,7 +229,7 @@ The first step
     };
     const result = parse({
       text: md,
-      config,
+      skeleton,
       commits: {
         L1S1Q: ["abcdefg1"],
       },
@@ -271,7 +271,7 @@ First line
 
 The first step
 `;
-    const config = {
+    const skeleton = {
       levels: [
         {
           id: "L1",
@@ -285,7 +285,7 @@ The first step
     };
     const result = parse({
       text: md,
-      config,
+      skeleton,
       commits: {
         L1S1Q: ["abcdefg1", "123456789"],
       },
@@ -327,7 +327,7 @@ First line
 
 The first step
 `;
-    const config = {
+    const skeleton = {
       levels: [
         {
           id: "L1",
@@ -336,7 +336,7 @@ The first step
     };
     const result = parse({
       text: md,
-      config,
+      skeleton,
       commits: {
         L1: ["abcdefg1"],
       },
@@ -372,7 +372,7 @@ First line
 
 The first step
 `;
-    const config = {
+    const skeleton = {
       levels: [
         {
           id: "L1",
@@ -397,7 +397,7 @@ The first step
     };
     const result = parse({
       text: md,
-      config,
+      skeleton,
       commits: {
         L1S1Q: ["abcdefg1", "123456789"],
         L1S1A: ["1gfedcba", "987654321"],
@@ -462,7 +462,7 @@ Second level content.
 
 The third step
 `;
-    const config = {
+    const skeleton = {
       levels: [
         {
           id: "L1",
@@ -522,7 +522,7 @@ The third step
     };
     const result = parse({
       text: md,
-      config,
+      skeleton,
       commits: {
         L1S1Q: ["abcdef1", "123456789"],
         L1S1A: ["1fedcba", "987654321"],
@@ -623,7 +623,7 @@ First level content.
 The first step
 
 `;
-    const config = {
+    const skeleton = {
       levels: [
         {
           id: "L1",
@@ -637,7 +637,7 @@ The first step
     };
     const result = parse({
       text: md,
-      config,
+      skeleton,
       commits: {
         L1S1Q: ["abcdef1", "123456789"],
       },
@@ -674,7 +674,7 @@ The first step
 Description.
 `;
 
-    const config = {
+    const skeleton = {
       config: {
         testRunner: {
           command: "./node_modules/.bin/mocha",
@@ -704,7 +704,7 @@ Description.
     };
     const result = parse({
       text: md,
-      config,
+      skeleton,
       commits: {},
     });
     const expected = {
@@ -747,7 +747,7 @@ Description.
 Description.
 `;
 
-    const config = {
+    const skeleton = {
       config: {
         testRunner: {
           command: "./node_modules/.bin/mocha",
@@ -774,7 +774,7 @@ Description.
     };
     const result = parse({
       text: md,
-      config,
+      skeleton,
       commits: {
         INIT: ["abcdef1", "123456789"],
       },

From 016d4d660327aecd52a32d3eee626077c47cd689 Mon Sep 17 00:00:00 2001
From: shmck <shawn.j.mckay@gmail.com>
Date: Sun, 7 Jun 2020 10:19:00 -0700
Subject: [PATCH 8/9] fix build issue

Signed-off-by: shmck <shawn.j.mckay@gmail.com>
---
 src/validate.ts | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/validate.ts b/src/validate.ts
index 7106e4a..4da3048 100644
--- a/src/validate.ts
+++ b/src/validate.ts
@@ -18,18 +18,18 @@ async function validate(args: string[]) {
 
   // -y --yaml - default coderoad-config.yml
   const options = {
-    yaml: getArg(args, { name: "yaml", alias: "y" }) || "coderoad.yaml";
-  }
+    yaml: getArg(args, { name: "yaml", alias: "y" }) || "coderoad.yaml",
+  };
 
   const _yaml = await read(path.join(localDir, options.yaml), "utf8");
 
   // parse yaml config
   let config;
   try {
-    config = yamlParser.load(_yaml)
+    config = yamlParser.load(_yaml);
     // TODO: validate yaml
     if (!config || !config.length) {
-      throw new Error('Invalid yaml file contents')
+      throw new Error("Invalid yaml file contents");
     }
   } catch (e) {
     console.error("Error parsing yaml");

From 30985e4177a590c5b54a7ed10d7dc01da8c30cde Mon Sep 17 00:00:00 2001
From: shmck <shawn.j.mckay@gmail.com>
Date: Sun, 7 Jun 2020 10:28:03 -0700
Subject: [PATCH 9/9] exit early if yaml parsing fails

Signed-off-by: shmck <shawn.j.mckay@gmail.com>
---
 src/build.ts | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/build.ts b/src/build.ts
index 4691e55..b129b52 100644
--- a/src/build.ts
+++ b/src/build.ts
@@ -76,12 +76,13 @@ async function build(args: string[]) {
   let skeleton;
   try {
     skeleton = yamlParser.load(_yaml);
-    if (!skeleton || !skeleton.length) {
-      throw new Error("Invalid yaml file contents");
+    if (!skeleton || !Object.keys(skeleton).length) {
+      throw new Error(`Skeleton at "${options.yaml}" is invalid`);
     }
   } catch (e) {
     console.error("Error parsing yaml");
     console.error(e.message);
+    return;
   }
 
   // validate skeleton based on skeleton json schema