merge: #3308

3308: feat(si-generator): Adds si-generator r=stack72 a=adamhjk

This adds a prototype of `si-generator`, a deno based cli that generates assets. Initially it only works for AWS.

You can use it by calling:

```
deno run --allow-run ./main.ts s3api create-bucket
```

You can also compile a standlone binary with:

```
deno task compile
```

And run the (very minimal) test suite with:

```
deno task test
```

Co-authored-by: Adam Jacob <adam@systeminit.com>

Author: si-bors-ng[bot]

bin/si-generator/.gitignore

@@ -0,0 +1 @@
+out

bin/si-generator/README.md

@@ -0,0 +1,13 @@
+# This is a spike
+
+It doesn't have buck2 support, we don't actually use it much yet.
+
+But it works!
+
+# To use
+
+```
+$ deno run --allow-run main.ts asset s3api create-bucket
+```
+
+Would print an asset definition function for the s3api create-bucket call.

bin/si-generator/deno.json

@@ -0,0 +1,7 @@
+{
+  "tasks": {
+    "test": "deno test --allow-run",
+    "dev": "deno run --watch main.ts",
+    "compile": "deno compile --allow-run --output ./out/si-generate main.ts"
+  }
+}

bin/si-generator/deno.lock

@@ -0,0 +1,5 @@
+import { run } from "./src/run.ts";
+
+if (import.meta.main) {
+  await run();
+}

bin/si-generator/main.ts

@@ -0,0 +1,49 @@
+import { assertEquals } from "https://deno.land/std@0.215.0/assert/mod.ts";
+import { awsGenerate } from "./src/asset_generator.ts";
+import { Prop } from "./src/props.ts";
+
+Deno.test(function awsServiceProps() {
+  const correctProps: Array<Prop> = [
+    { kind: "string", name: "KeyName", variableName: "keyNameProp" },
+    { kind: "boolean", name: "DryRun", variableName: "dryRunProp" },
+    { kind: "string", name: "KeyType", variableName: "keyTypeProp" },
+    {
+      kind: "array",
+      name: "TagSpecifications",
+      variableName: "tagSpecificationsProp",
+      entry: {
+        kind: "object",
+        name: "TagSpecificationsChild",
+        variableName: "tagSpecificationsChildProp",
+        children: [
+          {
+            kind: "string",
+            name: "ResourceType",
+            variableName: "resourceTypeProp",
+          },
+          {
+            kind: "array",
+            name: "Tags",
+            variableName: "tagsProp",
+            entry: {
+              kind: "object",
+              name: "TagsChild",
+              variableName: "tagsChildProp",
+              children: [
+                { kind: "string", name: "Key", variableName: "keyProp" },
+                {
+                  kind: "string",
+                  name: "Value",
+                  variableName: "valueProp",
+                },
+              ],
+            },
+          },
+        ],
+      },
+    },
+    { kind: "string", name: "KeyFormat", variableName: "keyFormatProp" },
+  ];
+  const props = awsGenerate("ec2", "create-key-pair");
+  assertEquals(props, correctProps);
+});

bin/si-generator/main_test.ts

@@ -0,0 +1,109 @@
+import { Prop, PropParent } from "./props.ts";
+import { camelCase } from "https://deno.land/x/case/mod.ts";
+import { singular } from "https://deno.land/x/deno_plural/mod.ts";
+
+type AwsScaffold = Record<string, unknown>;
+
+export function awsGenerate(
+  awsService: string,
+  awsCommand: string,
+): Array<Prop> {
+  const scaffold = getAwsCliScaffold(awsService, awsCommand);
+  const props = propsFromScaffold(scaffold, []);
+  return props;
+}
+
+function getAwsCliScaffold(
+  awsService: string,
+  awsCommand: string,
+): AwsScaffold {
+  const command = new Deno.Command("aws", {
+    args: [awsService, awsCommand, "--generate-cli-skeleton"],
+    stdin: "null",
+    stdout: "piped",
+    stderr: "piped",
+  });
+  const { code, stdout: rawStdout, stderr: rawStderr } = command.outputSync();
+  const stdout = new TextDecoder().decode(rawStdout);
+  const stderr = new TextDecoder().decode(rawStderr);
+
+  if (code !== 0) {
+    console.error(`AWS cli failed with exit code: ${code}`);
+    console.error(`STDOUT:\n\n${stdout.toLocaleString()}`);
+    console.error(`STDERR:\n\n${stderr.toLocaleString()}`);
+    throw new Error("aws cli command failed");
+  }
+  const result = JSON.parse(stdout);
+  return result;
+}
+
+function propsFromScaffold(
+  scaffold: AwsScaffold,
+  props: Array<Prop>,
+  parent?: PropParent,
+): Array<Prop> {
+  for (let [key, value] of Object.entries(scaffold)) {
+    if (
+      key == "KeyName" && parent?.kind == "object" &&
+      parent?.children.length == 0
+    ) {
+      // @ts-ignore we know you can't do this officialy, but unofficialy, suck
+      // it.
+      parent.kind = "map";
+      key = singular(parent.name);
+    }
+    let prop: Prop | undefined;
+    if (typeof value === "string") {
+      prop = {
+        kind: "string",
+        name: key,
+        variableName: camelCase(`${key}Prop`),
+      };
+    } else if (typeof value === "number") {
+      prop = {
+        kind: "number",
+        name: key,
+        variableName: camelCase(`${key}Prop`),
+      };
+    } else if (typeof value === "boolean") {
+      prop = {
+        kind: "boolean",
+        name: key,
+        variableName: camelCase(`${key}Prop`),
+      };
+    } else if (Array.isArray(value)) {
+      prop = {
+        kind: "array",
+        name: key,
+        variableName: camelCase(`${key}Prop`),
+      };
+      const childObject: AwsScaffold = {};
+      childObject[`${key}Child`] = value[0];
+      propsFromScaffold(childObject, props, prop);
+    } else if (value == null) {
+      // Sometimes the default value is null, and not the empty string. This
+      // seems like a reasonable default, even if it is going to be weird.
+      prop = {
+        kind: "string",
+        name: key,
+        variableName: camelCase(`${key}Prop`),
+      };
+    } else if (typeof value === "object") {
+      prop = {
+        kind: "object",
+        name: key,
+        variableName: camelCase(`${key}Prop`),
+        children: [],
+      };
+      propsFromScaffold(value as AwsScaffold, props, prop);
+    }
+    if (prop && parent?.kind == "object") {
+      parent.children.push(prop);
+    } else if (prop && parent?.kind == "array" || parent?.kind == "map") {
+      parent.entry = prop;
+    } else if (prop && !parent) {
+      props.push(prop);
+    }
+  }
+  return props;
+}

bin/si-generator/src/asset_generator.ts

@@ -0,0 +1,42 @@
+export interface PropBase {
+  kind: string;
+  name: string;
+  variableName: string;
+}
+
+export interface PropString extends PropBase {
+  kind: "string";
+}
+
+export interface PropNumber extends PropBase {
+  kind: "number";
+}
+
+export interface PropBoolean extends PropBase {
+  kind: "boolean";
+}
+
+export interface PropObject extends PropBase {
+  kind: "object";
+  children: Array<Prop>;
+}
+
+export interface PropMap extends PropBase {
+  kind: "map";
+  entry?: Prop;
+}
+
+export interface PropArray extends PropBase {
+  kind: "array";
+  entry?: Prop;
+}
+
+export type Prop =
+  | PropString
+  | PropNumber
+  | PropObject
+  | PropArray
+  | PropBoolean
+  | PropMap;
+
+export type PropParent = PropObject | PropArray | PropMap;

bin/si-generator/src/props.ts

@@ -0,0 +1,49 @@
+import { Eta } from "https://deno.land/x/eta@v3.2.0/src/index.ts";
+import { Prop } from "./props.ts";
+import { partial as assetMainPartial } from "./templates/assetMain.ts";
+import { partial as arrayPartial } from "./templates/array.ts";
+import { partial as booleanPartial } from "./templates/boolean.ts";
+import { partial as mapPartial } from "./templates/map.ts";
+import { partial as numberPartial } from "./templates/number.ts";
+import { partial as objectPartial } from "./templates/object.ts";
+import { partial as stringPartial } from "./templates/string.ts";
+import { partial as renderPropPartial } from "./templates/renderProp.ts";
+
+type RenderProvider = "aws";
+
+export async function renderAsset(props: Array<Prop>, provider: RenderProvider): Promise<string> {
+  const eta = new Eta({
+    debug: true,
+    autoEscape: false,
+  });
+  eta.loadTemplate("@assetMain", assetMainPartial);
+  eta.loadTemplate("@arrayPartial", arrayPartial);
+  eta.loadTemplate("@booleanPartial", booleanPartial);
+  eta.loadTemplate("@mapPartial", mapPartial);
+  eta.loadTemplate("@numberPartial", numberPartial);
+  eta.loadTemplate("@objectPartial", objectPartial);
+  eta.loadTemplate("@stringPartial", stringPartial);
+  eta.loadTemplate("@renderPropPartial", renderPropPartial);
+  const assetDefinition = eta.render("@assetMain", { props, provider });
+
+  const command = new Deno.Command("deno", {
+    args: ["fmt", "-"],
+    stdin: "piped",
+    stdout: "piped",
+    stderr: "piped",
+  });
+  const running = command.spawn();
+  const writer = running.stdin.getWriter();
+  await writer.write(new TextEncoder().encode(assetDefinition));
+  writer.releaseLock();
+  await running.stdin.close();
+
+  const n = await running.stdout.getReader().read();
+  const stdout = new TextDecoder().decode(n.value);
+  const result = await running.status;
+  if (result.success) {
+    return stdout;
+  } else {
+    return assetDefinition;
+  }
+}

bin/si-generator/src/render.ts

@@ -0,0 +1,25 @@
+import { Command } from "https://deno.land/x/cliffy@v1.0.0-rc.3/command/mod.ts";
+import { awsGenerate } from "./asset_generator.ts";
+import { renderAsset } from "./render.ts";
+
+export async function run() {
+  const command = new Command()
+    .name("si-generator")
+    .version("0.1.0")
+    .description(
+      "Generate Assets and code for System Initiative",
+    )
+    .action(() => {
+      command.showHelp();
+      Deno.exit(1);
+    })
+    .command("asset")
+    .description("generate an asset definition from an aws cli skeleton")
+    .arguments("<awsService:string> <awsCommand:string>")
+    .action(async (_options, awsService, awsCommand) => {
+      const props = awsGenerate(awsService, awsCommand);
+      const result = await renderAsset(props, "aws");
+      console.log(result);
+    });
+  const _result = await command.parse(Deno.args);
+}