exhaust: added exhaust function, with stub for exhaust-csv-export and…

.commitlintrc.js

@@ -30,7 +30,9 @@ module.exports = {
         'examples',
         '.github',
         '.husky',
-        'scripts'
+        'scripts',
+        'models',
+        'plugins'
       ]
     ],
     'scope-empty': [

examples/impls/functional/sci-e-exhaust-csv.yml

@@ -0,0 +1,22 @@
+name: sci-e-demo
+description:
+tags:
+initialize:
+  plugins:
+    'sci-e':
+      model: SciE
+      path: '@grnsft/if-models'
+exhaust:
+  pipeline: ['yaml', 'csv', 'grafana']
+  basepath: ''
+tree:
+  children:
+    child:
+      pipeline:
+        - sci-e
+      config:
+        sci-e:
+      inputs:
+        - timestamp: 2023-08-06T00:00
+          duration: 3600
+          energy-cpu: 0.001

examples/impls/functional/time-sync-exhaust-csv-export.yml

@@ -0,0 +1,55 @@
+name: exhaust-csv-export demo
+description:
+tags:
+initialize:
+  plugins:
+    'time-sync':
+      method: TimeSync
+      path: 'builtin'
+      global-config:
+        start-time: '2023-12-12T00:00:00.000Z'
+        end-time: '2023-12-12T00:01:00.000Z'
+        interval: 5
+        allow-padding: true
+  outputs:
+    'csv':
+      output-path: 'C:\dev\demo-exhaust-csv-export.csv'
+tree:
+  children:
+    child1:
+      pipeline:
+        - time-sync
+      config:
+      inputs:
+        - timestamp: '2023-12-12T00:00:00.000Z'
+          duration: 1
+          energy-cpu: 0.001
+          custom-metric: 0.001
+        - timestamp: '2023-12-12T00:00:01.000Z'
+          duration: 5
+          energy-cpu: 0.001
+        - timestamp: '2023-12-12T00:00:06.000Z'
+          duration: 7
+          energy-cpu: 0.001
+        - timestamp: '2023-12-12T00:00:13.000Z'
+          duration: 30
+          energy-cpu: 0.001
+          custom-metric: 0.002
+    child2:
+      pipeline:
+        - time-sync
+      config:
+      inputs:
+        - timestamp: '2023-12-12T00:00:00.000Z'
+          duration: 1
+          energy-cpu: 0.001
+        - timestamp: '2023-12-12T00:00:01.000Z'
+          duration: 5
+          energy-cpu: 0.001
+          custom-metric: 0.003
+        - timestamp: '2023-12-12T00:00:06.000Z'
+          duration: 7
+          energy-cpu: 0.001
+        - timestamp: '2023-12-12T00:00:13.000Z'
+          duration: 30
+          energy-cpu: 0.001

src/config/strings.ts

@@ -45,4 +45,6 @@ https://github.com/Green-Software-Foundation/if/issues/new?assignees=&labels=fee
   INVALID_GROUP_BY: (type: string) => `Invalid group ${type}.`,
   REJECTING_OVERRIDE: (param: ManifestParameter) =>
     `Rejecting overriding of canonical parameter: ${param.name}.`,
+  INVALID_EXHAUST_PLUGIN: (pluginName: string) =>
+    `Invalid exhaust plugin: ${pluginName}.`,
 };

src/index.ts

@@ -1,6 +1,7 @@
 #!/usr/bin/env node
 import {aggregate} from './lib/aggregate';
 import {compute} from './lib/compute';
+import {exhaust} from './lib/exhaust';
 import {initalize} from './lib/initialize';
 import {load} from './lib/load';
 import {parameterize} from './lib/parameterize';
@@ -9,7 +10,6 @@ import {parseArgs} from './util/args';
 import {ERRORS} from './util/errors';
 import {andHandle} from './util/helpers';
 import {logger} from './util/logger';
-import {saveYamlFileAs} from './util/yaml';
 
 import {STRINGS} from './config';
 
@@ -29,18 +29,7 @@ const impactEngine = async () => {
     const plugins = await initalize(context.initialize.plugins);
     const computedTree = await compute(tree, {context, plugins});
     const aggregatedTree = aggregate(computedTree, context.aggregation);
-
-    const outputFile = {
-      ...context,
-      tree: aggregatedTree,
-    };
-
-    if (!outputPath) {
-      logger.info(JSON.stringify(outputFile, null, 2));
-      return;
-    }
-
-    await saveYamlFileAs(outputFile, outputPath);
+    exhaust(aggregatedTree, context, outputPath);
 
     return;
   }

src/lib/exhaust.ts

@@ -0,0 +1,55 @@
+/**
+ * @todo This is temporary solution, will be refactored to support dynamic plugins.
+ */
+import {ExportCsv} from '../models/export-csv';
+import {ExportLog} from '../models/export-log';
+import {ExportYaml} from '../models/export-yaml';
+
+import {ERRORS} from '../util/errors';
+
+import {STRINGS} from '../config';
+
+import {ExhaustPluginInterface} from '../types/exhaust-plugin-interface';
+import {Context} from '../types/manifest';
+
+const {ModuleInitializationError} = ERRORS;
+const {INVALID_EXHAUST_PLUGIN} = STRINGS;
+
+/**
+ * Initialize exhaust plugins based on the provided config
+ */
+const initializeExhaustPlugins = (plugins: string[]) =>
+  plugins.map(initializeExhaustPlugin);
+
+/**
+ * factory method for exhaust plugins
+ */
+const initializeExhaustPlugin = (name: string): ExhaustPluginInterface => {
+  switch (name) {
+    case 'yaml':
+      return ExportYaml();
+    case 'csv':
+      return ExportCsv();
+    case 'log':
+      return ExportLog();
+    default:
+      throw new ModuleInitializationError(INVALID_EXHAUST_PLUGIN(name));
+  }
+};
+
+/**
+ * Output manager - Exhaust.
+ * Grabs output plugins from context, executes every.
+ */
+export const exhaust = (tree: any, context: Context, outputPath?: string) => {
+  const outputPlugins = context.initialize.outputs;
+
+  if (!outputPlugins) {
+    ExportLog().execute(tree, context);
+
+    return;
+  }
+
+  const exhaustPlugins = initializeExhaustPlugins(outputPlugins);
+  exhaustPlugins.forEach(plugin => plugin.execute(tree, context, outputPath));
+};

src/models/export-csv.ts

@@ -0,0 +1,155 @@
+import * as fs from 'fs/promises';
+
+import {ERRORS} from '../util/errors';
+
+import {ExhaustPluginInterface} from '../types/exhaust-plugin-interface';
+import {Context} from '../types/manifest';
+
+const {WriteFileError, CliInputError} = ERRORS;
+
+export const ExportCsv = (): ExhaustPluginInterface => {
+  /**
+   * handle a tree leaf, where there are no child nodes, by adding it as key->value pair to the flat map
+   * and capturing key as a header
+   */
+  const handleLeafValue = (
+    value: any,
+    fullPath: string,
+    key: any,
+    flatMap: {[key: string]: any},
+    headers: Set<string>
+  ) => {
+    if (fullPath.includes('outputs')) {
+      headers.add(key);
+      flatMap[fullPath] = value;
+    }
+  };
+
+  /**
+   * handle a tree node, recursively traverse the children and append their results to the flat map and captured headers
+   */
+  const handleNodeValue = (
+    value: any,
+    fullPath: string,
+    flatMap: Record<string, any>,
+    headers: Set<string>
+  ) => {
+    const [subFlatMap, subHeaders] = extractFlatMapAndHeaders(value, fullPath);
+
+    if (Object.keys(subFlatMap).length > 0) {
+      Object.entries(subFlatMap).forEach(([subKey, value]) => {
+        flatMap[subKey] = value;
+      });
+
+      subHeaders.forEach(subHeader => {
+        headers.add(subHeader);
+      });
+    }
+  };
+
+  /**
+   * Handles a key at the top level of the tree
+   */
+  const handleKey = (
+    value: any,
+    key: any,
+    prefix: string,
+    flatMap: Record<string, any>,
+    headers: Set<string>
+  ) => {
+    const fullPath = prefix ? `${prefix}.${key}` : key;
+
+    if (value !== null && typeof value === 'object') {
+      return handleNodeValue(value, fullPath, flatMap, headers);
+    }
+
+    return handleLeafValue(value, fullPath, key, flatMap, headers);
+  };
+
+  /**
+   * qrecursively extract a flat map and headers from the hierarcial tree
+   */
+  const extractFlatMapAndHeaders = (
+    tree: any,
+    prefix = ''
+  ): [Record<string, any>, Set<string>] => {
+    const headers: Set<string> = new Set();
+    const flatMap: Record<string, any> = [];
+
+    for (const key in tree) {
+      if (key in tree) {
+        handleKey(tree[key], key, prefix, flatMap, headers);
+      }
+    }
+
+    return [flatMap, headers];
+  };
+
+  /**
+   * extract the id of the key, that is removing the last token (which is the index).
+   * in this manner, multiple keys that identical besides their index share the same id.
+   */
+  const extractIdHelper = (key: string): string => {
+    const parts = key.split('.');
+    parts.pop();
+
+    return parts.join('.');
+  };
+
+  /**
+   * generate a CSV formatted string based on a flat key->value map, headers and ids
+   */
+  const getCsvString = (
+    map: {[key: string]: any},
+    headers: Set<string>,
+    ids: Set<string>
+  ): string => {
+    const csvRows: string[] = [];
+    csvRows.push(['id', ...headers].join(','));
+
+    ids.forEach(id => {
+      const rowData = [id];
+
+      headers.forEach(header => {
+        const value = map[`${id}.${header}`] ?? '';
+        rowData.push(value.toString());
+      });
+      csvRows.push(rowData.join(','));
+    });
+
+    return csvRows.join('\n');
+  };
+
+  /**
+   * write the given string content to a file at the provided path
+   */
+  const writeOutputFile = async (content: string, outputPath: string) => {
+    try {
+      await fs.writeFile(outputPath, content);
+    } catch (error) {
+      throw new WriteFileError(
+        `Failed to write CSV to ${outputPath}: ${error}`
+      );
+    }
+  };
+
+  /**
+   * export the provided tree content to a CSV file, represented in a flat structure
+   */
+  const execute = async (tree: any, _context: Context, outputPath: string) => {
+    if (!outputPath) {
+      throw new CliInputError('Output path is required.');
+    }
+
+    const [extractredFlatMap, extractedHeaders] =
+      extractFlatMapAndHeaders(tree);
+    const ids = new Set(
+      Object.keys(extractredFlatMap).map(key => extractIdHelper(key))
+    );
+    const csvString = getCsvString(extractredFlatMap, extractedHeaders, ids);
+
+    writeOutputFile(csvString, outputPath);
+  };
+
+  return {execute};
+};

src/models/export-log.ts

@@ -0,0 +1,17 @@
+import {Context} from '../types/manifest';
+
+export const ExportLog = () => {
+  /**
+   * Logs output manifest in console.
+   */
+  const execute = async (tree: any, context: Context) => {
+    const outputFile = {
+      ...context,
+      tree,
+    };
+
+    console.log(JSON.stringify(outputFile, null, 2));
+  };
+
+  return {execute};
+};

src/models/export-yaml.ts

@@ -0,0 +1,27 @@
+import {saveYamlFileAs} from '../util/yaml';
+
+import {ERRORS} from '../util/errors';
+
+import {Context} from '../types/manifest';
+
+const {CliInputError} = ERRORS;
+
+export const ExportYaml = () => {
+  /**
+   * Saves output file in YAML format.
+   */
+  const execute = async (tree: any, context: Context, outputPath: string) => {
+    if (!outputPath) {
+      throw new CliInputError('Output path is required.');
+    }
+
+    const outputFile = {
+      ...context,
+      tree,
+    };
+
+    await saveYamlFileAs(outputFile, outputPath);
+  };
+
+  return {execute};
+};

src/models/index.ts

@@ -1,2 +1,3 @@
 export {GroupBy} from './group-by';
 export {TimeSync} from './time-sync';
+export {ExportCsv as ExhaustExportCsv} from './export-csv';