Deploy with Manifests from URLs (#251)

* added functionality, need to add/modify existing tests

* added tests

* updated readme

* prettier

src/run.ts

@@ -5,7 +5,7 @@ import {promote} from './actions/promote'
 import {reject} from './actions/reject'
 import {Action, parseAction} from './types/action'
 import {parseDeploymentStrategy} from './types/deploymentStrategy'
-import {getFilesFromDirectories} from './utilities/fileUtils'
+import {getFilesFromDirectoriesAndURLs} from './utilities/fileUtils'
 import {PrivateKubectl} from './types/privatekubectl'
 
 export async function run() {
@@ -26,7 +26,9 @@ export async function run() {
       .map((manifest) => manifest.trim()) // remove surrounding whitespace
       .filter((manifest) => manifest.length > 0) // remove any blanks
 
-   const fullManifestFilePaths = getFilesFromDirectories(manifestFilePaths)
+   const fullManifestFilePaths = await getFilesFromDirectoriesAndURLs(
+      manifestFilePaths
+   )
    const kubectlPath = await getKubectlPath()
    const namespace = core.getInput('namespace') || 'default'
    const isPrivateCluster =

src/types/errorable.ts

@@ -0,0 +1,48 @@
+export interface Succeeded<T> {
+   readonly succeeded: true
+   readonly result: T
+}
+
+export interface Failed {
+   readonly succeeded: false
+   readonly error: string
+}
+
+export type Errorable<T> = Succeeded<T> | Failed
+
+export function succeeded<T>(e: Errorable<T>): e is Succeeded<T> {
+   return e.succeeded
+}
+
+export function failed<T>(e: Errorable<T>): e is Failed {
+   return !e.succeeded
+}
+
+export function map<T, U>(e: Errorable<T>, fn: (t: T) => U): Errorable<U> {
+   if (failed(e)) {
+      return {succeeded: false, error: e.error}
+   }
+   return {succeeded: true, result: fn(e.result)}
+}
+
+export function combine<T>(es: Errorable<T>[]): Errorable<T[]> {
+   const failures = es.filter(failed)
+   if (failures.length > 0) {
+      return {
+         succeeded: false,
+         error: failures.map((f) => f.error).join('\n')
+      }
+   }
+
+   return {
+      succeeded: true,
+      result: es.map((e) => (e as Succeeded<T>).result)
+   }
+}
+
+export function getErrorMessage(error: unknown) {
+   if (error instanceof Error) {
+      return error.message
+   }
+   return String(error)
+}

src/utilities/fileUtils.test.ts

@@ -1,11 +1,45 @@
-import {getFilesFromDirectories} from './fileUtils'
+import {
+   getFilesFromDirectoriesAndURLs,
+   getTempDirectory,
+   urlFileKind,
+   writeYamlFromURLToFile
+} from './fileUtils'
 
+import * as yaml from 'js-yaml'
+import * as fs from 'fs'
 import * as path from 'path'
+import {succeeded} from '../types/errorable'
 
+const sampleYamlUrl =
+   'https://raw.githubusercontent.com/kubernetes/website/main/content/en/examples/controllers/nginx-deployment.yaml'
 describe('File utils', () => {
-   it('detects files in nested directories and ignores non-manifest files and empty dirs', () => {
+   test('correctly parses a yaml file from a URL', async () => {
+      const tempFile = await writeYamlFromURLToFile(sampleYamlUrl, 0)
+      const fileContents = fs.readFileSync(tempFile).toString()
+      const inputObjects = yaml.safeLoadAll(fileContents)
+      expect(inputObjects).toHaveLength(1)
+
+      for (const obj of inputObjects) {
+         expect(obj.metadata.name).toBe('nginx-deployment')
+         expect(obj.kind).toBe('Deployment')
+      }
+   })
+
+   it('fails when a bad URL is given among other files', async () => {
+      const badUrl = 'https://www.github.com'
+
       const testPath = path.join('test', 'unit', 'manifests')
-      const testSearch: string[] = getFilesFromDirectories([testPath])
+      await expect(
+         getFilesFromDirectoriesAndURLs([testPath, badUrl])
+      ).rejects.toThrow()
+   })
+
+   it('detects files in nested directories and ignores non-manifest files and empty dirs', async () => {
+      const testPath = path.join('test', 'unit', 'manifests')
+      const testSearch: string[] = await getFilesFromDirectoriesAndURLs([
+         testPath,
+         sampleYamlUrl
+      ])
 
       const expectedManifests = [
          'test/unit/manifests/manifest_test_dir/another_layer/deep-ingress.yaml',
@@ -17,13 +51,18 @@ describe('File utils', () => {
       ]
 
       // is there a more efficient way to test equality w random order?
-      expect(testSearch).toHaveLength(7)
+      expect(testSearch).toHaveLength(8)
       expectedManifests.forEach((fileName) => {
-         expect(testSearch).toContain(fileName)
+         if (fileName.startsWith('test/unit')) {
+            expect(testSearch).toContain(fileName)
+         } else {
+            expect(fileName.includes(urlFileKind)).toBe(true)
+            expect(fileName.startsWith(getTempDirectory()))
+         }
       })
    })
 
-   it('crashes when an invalid file is provided', () => {
+   it('crashes when an invalid file is provided', async () => {
       const badPath = path.join('test', 'unit', 'manifests', 'nonexistent.yaml')
       const goodPath = path.join(
          'test',
@@ -32,12 +71,12 @@ describe('File utils', () => {
          'manifest_test_dir'
       )
 
-      expect(() => {
-         getFilesFromDirectories([badPath, goodPath])
-      }).toThrowError()
+      expect(
+         getFilesFromDirectoriesAndURLs([badPath, goodPath])
+      ).rejects.toThrowError()
    })
 
-   it("doesn't duplicate files when nested dir included", () => {
+   it("doesn't duplicate files when nested dir included", async () => {
       const outerPath = path.join('test', 'unit', 'manifests')
       const fileAtOuter = path.join(
          'test',
@@ -53,11 +92,16 @@ describe('File utils', () => {
       )
 
       expect(
-         getFilesFromDirectories([outerPath, fileAtOuter, innerPath])
+         await getFilesFromDirectoriesAndURLs([
+            outerPath,
+            fileAtOuter,
+            innerPath
+         ])
       ).toHaveLength(7)
    })
-})
 
-// files that don't exist / nested files that don't exist / something else with non-manifest
-// lots of combinations of pointing to a directory and non yaml/yaml file
-// similarly named files in different folders
+   it('throws an error for an invalid URL', async () => {
+      const badUrl = 'https://www.github.com'
+      await expect(writeYamlFromURLToFile(badUrl, 0)).rejects.toBeTruthy()
+   })
+})

src/utilities/fileUtils.ts

@@ -1,8 +1,15 @@
 import * as fs from 'fs'
+import * as https from 'https'
 import * as path from 'path'
 import * as core from '@actions/core'
 import * as os from 'os'
+import * as yaml from 'js-yaml'
+import {Errorable, succeeded, failed, Failed} from '../types/errorable'
 import {getCurrentTime} from './timeUtils'
+import {isHttpUrl} from './githubUtils'
+import {K8sObject} from '../types/k8sObject'
+
+export const urlFileKind = 'urlfile'
 
 export function getTempDirectory(): string {
    return process.env['runner.tempDirectory'] || os.tmpdir()
@@ -62,12 +69,27 @@ function getManifestFileName(kind: string, name: string) {
    return path.join(tempDirectory, path.basename(filePath))
 }
 
-export function getFilesFromDirectories(filePaths: string[]): string[] {
+export async function getFilesFromDirectoriesAndURLs(
+   filePaths: string[]
+): Promise<string[]> {
    const fullPathSet: Set<string> = new Set<string>()
 
-   filePaths.forEach((fileName) => {
+   let fileCounter = 0
+   for (const fileName of filePaths) {
       try {
-         if (fs.lstatSync(fileName).isDirectory()) {
+         if (isHttpUrl(fileName)) {
+            try {
+               const tempFilePath: string = await writeYamlFromURLToFile(
+                  fileName,
+                  fileCounter++
+               )
+               fullPathSet.add(tempFilePath)
+            } catch (e) {
+               throw Error(
+                  `encountered error trying to pull YAML from URL ${fileName}: ${e}`
+               )
+            }
+         } else if (fs.lstatSync(fileName).isDirectory()) {
             recurisveManifestGetter(fileName).forEach((file) => {
                fullPathSet.add(file)
             })
@@ -86,9 +108,86 @@ export function getFilesFromDirectories(filePaths: string[]): string[] {
             `Exception occurred while reading the file ${fileName}: ${ex}`
          )
       }
-   })
+   }
 
-   return Array.from(fullPathSet)
+   const arr = Array.from(fullPathSet)
+   return arr
+}
+
+export async function writeYamlFromURLToFile(
+   url: string,
+   fileNumber: number
+): Promise<string> {
+   return new Promise((resolve, reject) => {
+      https
+         .get(url, async (response) => {
+            const code = response.statusCode ?? 0
+            if (code >= 400) {
+               reject(
+                  Error(
+                     `received response status ${response.statusMessage} from url ${url}`
+                  )
+               )
+            }
+
+            const targetPath = getManifestFileName(
+               urlFileKind,
+               fileNumber.toString()
+            )
+            // save the file to disk
+            const fileWriter = fs
+               .createWriteStream(targetPath)
+               .on('finish', () => {
+                  const verification = verifyYaml(targetPath, url)
+                  if (succeeded(verification)) {
+                     core.debug(
+                        `outputting YAML contents from ${url} to ${targetPath}: ${JSON.stringify(
+                           verification.result
+                        )}`
+                     )
+                     resolve(targetPath)
+                  } else {
+                     reject(verification.error)
+                  }
+               })
+
+            response.pipe(fileWriter)
+         })
+         .on('error', (error) => {
+            reject(error)
+         })
+   })
+}
+
+function verifyYaml(filepath: string, url: string): Errorable<K8sObject[]> {
+   const fileContents = fs.readFileSync(filepath).toString()
+   let inputObjects
+   try {
+      inputObjects = yaml.safeLoadAll(fileContents)
+   } catch (e) {
+      return {
+         succeeded: false,
+         error: `failed to parse manifest from url ${url}: ${e}`
+      }
+   }
+
+   if (!inputObjects || inputObjects.length == 0) {
+      return {
+         succeeded: false,
+         error: `failed to parse manifest from url ${url}: no objects detected in manifest`
+      }
+   }
+
+   for (const obj of inputObjects) {
+      if (!obj.kind || !obj.apiVersion || !obj.metadata) {
+         return {
+            succeeded: false,
+            error: `failed to parse manifest from ${url}: missing fields`
+         }
+      }
+   }
+
+   return {succeeded: true, result: inputObjects}
 }
 
 function recurisveManifestGetter(dirName: string): string[] {