[heft-config-file] Add precompiled JSON schema validator support

apps/heft/config/heft.json

@@ -22,6 +22,22 @@
           }
         },
 
+        "precompile-json-schemas": {
+          "taskDependencies": ["typescript"],
+          "taskPlugin": {
+            "pluginPackage": "@rushstack/heft",
+            "pluginName": "run-script-plugin",
+            "options": {
+              "scriptPath": "lib-commonjs/plugins/PrecompileJsonSchemasPlugin.js",
+              "scriptOptions": {
+                "sourceFolder": "src/schemas",
+                "destinationFolders": ["lib-commonjs/schemas"],
+                "esmDestinationFolders": ["lib-esm/schemas"]
+              }
+            }
+          }
+        },
+
         "copy-legacy-compatibility-start-js": {
           "taskDependencies": ["typescript"],
           "taskPlugin": {

apps/heft/heft-plugin.json

@@ -26,6 +26,11 @@
         }
       ]
     },
+    {
+      "pluginName": "precompile-json-schemas-plugin",
+      "entryPoint": "./lib-commonjs/plugins/PrecompileJsonSchemasPlugin",
+      "optionsSchema": "./lib-commonjs/schemas/precompile-json-schemas-options.schema.json"
+    },
     {
       "pluginName": "run-script-plugin",
       "entryPoint": "./lib-commonjs/plugins/RunScriptPlugin",

apps/heft/src/configuration/HeftPluginConfiguration.ts

@@ -11,7 +11,7 @@ import {
   type IHeftTaskPluginDefinitionJson
 } from './HeftPluginDefinition';
 import type { IHeftConfigurationJsonPluginSpecifier } from '../utilities/CoreConfigFiles';
-import heftPluginSchema from '../schemas/heft-plugin.schema.json';
+import heftPluginSchemaValidator from '../schemas/heft-plugin.validator.js';
 
 export interface IHeftPluginConfigurationJson {
   lifecyclePlugins?: IHeftLifecyclePluginDefinitionJson[];
@@ -20,7 +20,10 @@ export interface IHeftPluginConfigurationJson {
 
 const HEFT_PLUGIN_CONFIGURATION_FILENAME: 'heft-plugin.json' = 'heft-plugin.json';
 
-const _jsonSchema: JsonSchema = JsonSchema.fromLoadedObject(heftPluginSchema);
+const _jsonSchema: JsonSchema = JsonSchema.fromCompiledValidator(
+  heftPluginSchemaValidator,
+  'heft-plugin.schema.json'
+);
 const _pluginConfigurationPromises: Map<string, Promise<HeftPluginConfiguration>> = new Map();
 
 /**

apps/heft/src/configuration/HeftPluginDefinition.ts

@@ -3,8 +3,13 @@
 
 import * as path from 'node:path';
 
-import { InternalError, JsonSchema } from '@rushstack/node-core-library';
+import { InternalError, JsonSchema, type IJsonSchemaCompiledValidator } from '@rushstack/node-core-library';
 
+import copyFilesValidator from '../schemas/copy-files-options.validator.js';
+import deleteFilesValidator from '../schemas/delete-files-options.validator.js';
+import precompileSchemasValidator from '../schemas/precompile-json-schemas-options.validator.js';
+import runScriptValidator from '../schemas/run-script-options.validator.js';
+import setEnvironmentVariablesValidator from '../schemas/set-environment-variables-plugin.validator.js';
 import type { IHeftPlugin } from '../pluginFramework/IHeftPlugin';
 import type { IScopedLogger } from '../pluginFramework/logging/ScopedLogger';
 import type { HeftLifecycleSession } from '../pluginFramework/HeftLifecycleSession';
@@ -192,6 +197,14 @@ export interface IHeftPluginDefinitionOptions {
   packageRoot: string;
 }
 
+const BUILT_IN_OPTIONS_VALIDATORS: ReadonlyMap<string, IJsonSchemaCompiledValidator> = new Map([
+  ['copy-files-options.schema.json', copyFilesValidator],
+  ['delete-files-options.schema.json', deleteFilesValidator],
+  ['precompile-json-schemas-options.schema.json', precompileSchemasValidator],
+  ['run-script-options.schema.json', runScriptValidator],
+  ['set-environment-variables-plugin.schema.json', setEnvironmentVariablesValidator]
+]);
+
 export abstract class HeftPluginDefinitionBase {
   #heftPluginDefinitionJson: IHeftPluginDefinitionJson;
   #pluginPackageName: string;
@@ -222,7 +235,14 @@ export abstract class HeftPluginDefinitionBase {
         options.packageRoot,
         options.heftPluginDefinitionJson.optionsSchema
       );
-      this.#optionsSchema = JsonSchema.fromFile(resolvedSchemaPath);
+      const builtInValidator: IJsonSchemaCompiledValidator | undefined =
+        options.packageName === '@rushstack/heft' &&
+        path.dirname(resolvedSchemaPath) === path.resolve(options.packageRoot, 'lib-commonjs/schemas')
+          ? BUILT_IN_OPTIONS_VALIDATORS.get(path.basename(resolvedSchemaPath))
+          : undefined;
+      this.#optionsSchema = builtInValidator
+        ? JsonSchema.fromCompiledValidator(builtInValidator, path.basename(resolvedSchemaPath))
+        : JsonSchema.fromFile(resolvedSchemaPath);
     }
   }
 

apps/heft/src/configuration/test/PrecompiledSchemas.test.ts

@@ -0,0 +1,96 @@
+// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license.
+// See LICENSE in the project root for license information.
+
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+
+import { JsonSchema, type IJsonSchemaCompiledValidator } from '@rushstack/node-core-library';
+
+import anythingValidator from '../../schemas/anything.validator.js';
+import copyFilesValidator from '../../schemas/copy-files-options.validator.js';
+import deleteFilesValidator from '../../schemas/delete-files-options.validator.js';
+import legacyValidator from '../../schemas/heft-legacy.validator.js';
+import heftPluginValidator from '../../schemas/heft-plugin.validator.js';
+import heftValidator from '../../schemas/heft.validator.js';
+import nodeServiceValidator from '../../schemas/node-service.validator.js';
+import precompileValidator from '../../schemas/precompile-json-schemas-options.validator.js';
+import runScriptValidator from '../../schemas/run-script-options.validator.js';
+import setEnvironmentValidator from '../../schemas/set-environment-variables-plugin.validator.js';
+import { HeftPluginConfiguration } from '../HeftPluginConfiguration';
+
+describe('Heft built-in schemas', () => {
+  const packageRoot: string = path.resolve(__dirname, '../../..');
+  const schemaFolder: string = path.join(packageRoot, 'lib-commonjs/schemas');
+  const validators: ReadonlyMap<string, IJsonSchemaCompiledValidator> = new Map([
+    ['anything.schema.json', anythingValidator],
+    ['copy-files-options.schema.json', copyFilesValidator],
+    ['delete-files-options.schema.json', deleteFilesValidator],
+    ['heft-legacy.schema.json', legacyValidator],
+    ['heft-plugin.schema.json', heftPluginValidator],
+    ['heft.schema.json', heftValidator],
+    ['node-service.schema.json', nodeServiceValidator],
+    ['precompile-json-schemas-options.schema.json', precompileValidator],
+    ['run-script-options.schema.json', runScriptValidator],
+    ['set-environment-variables-plugin.schema.json', setEnvironmentValidator]
+  ]);
+
+  it('publishes a precompiled validator for every JSON schema', () => {
+    const schemaFiles: string[] = fs
+      .readdirSync(schemaFolder)
+      .filter((fileName: string) => fileName.endsWith('.schema.json'));
+    expect(schemaFiles.length).toBeGreaterThan(0);
+    expect(schemaFiles.length).toBe(validators.size);
+
+    let esmImportCount: number = 0;
+    for (const schemaFile of schemaFiles) {
+      const schemaPath: string = path.join(schemaFolder, schemaFile);
+      const validatorPath: string = schemaPath.replace(/\.schema\.json$/, '.validator.js');
+      expect(fs.existsSync(validatorPath)).toBe(true);
+      const esmValidatorPath: string = path.join(packageRoot, 'lib-esm/schemas', path.basename(validatorPath));
+      const esmCode: string = fs.readFileSync(esmValidatorPath, 'utf8');
+      if (/import .* from "@rushstack\/node-core-library\/lib\/JsonSchemaRuntime"/.test(esmCode)) {
+        esmImportCount++;
+      }
+      expect(esmCode).toMatch(/export default validate\d+;/);
+      expect(esmCode).not.toContain('require(');
+      const validator: IJsonSchemaCompiledValidator = validators.get(schemaFile)!;
+      expect(typeof validator).toBe('function');
+
+      const compiledSchema: JsonSchema = JsonSchema.fromCompiledValidator(validator, schemaFile);
+      const sourceSchema: JsonSchema = JsonSchema.fromFile(schemaPath);
+      for (const example of [{}, { invalid: true }, { taskPlugins: [] }]) {
+        let sourceValid: boolean = true;
+        try {
+          sourceSchema.validateObject(example, 'test.json');
+        } catch {
+          sourceValid = false;
+        }
+        if (sourceValid) {
+          expect(() => compiledSchema.validateObject(example, 'test.json')).not.toThrow();
+        } else {
+          expect(() => compiledSchema.validateObject(example, 'test.json')).toThrow();
+        }
+      }
+    }
+    expect(esmImportCount).toBeGreaterThan(0);
+  });
+
+  it('loads every built-in plugin option schema without compiling a JSON schema', async () => {
+    const fromFileSpy = jest.spyOn(JsonSchema, 'fromFile');
+    try {
+      const plugins: HeftPluginConfiguration = await HeftPluginConfiguration.loadFromPackageAsync(
+        packageRoot,
+        '@rushstack/heft'
+      );
+      const manifest: { taskPlugins: { pluginName: string }[] } = JSON.parse(
+        fs.readFileSync(path.join(packageRoot, 'heft-plugin.json'), 'utf8')
+      );
+      for (const plugin of manifest.taskPlugins) {
+        expect(plugins.tryGetTaskPluginDefinitionByName(plugin.pluginName)).toBeDefined();
+      }
+      expect(fromFileSpy).not.toHaveBeenCalled();
+    } finally {
+      fromFileSpy.mockRestore();
+    }
+  });
+});

apps/heft/src/plugins/PrecompileJsonSchemasPlugin.ts

@@ -0,0 +1,107 @@
+// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license.
+// See LICENSE in the project root for license information.
+
+import * as path from 'node:path';
+
+import { FileSystem, JsonSchema } from '@rushstack/node-core-library';
+
+import type { HeftConfiguration } from '../configuration/HeftConfiguration';
+import type { IHeftTaskPlugin } from '../pluginFramework/IHeftPlugin';
+import type { IHeftTaskSession, IHeftTaskRunHookOptions } from '../pluginFramework/HeftTaskSession';
+import type { IRunScriptOptions } from './RunScriptPlugin';
+
+interface IPrecompileJsonSchemasPluginOptions {
+  sourceFolder: string;
+  destinationFolders?: string[];
+  esmDestinationFolders?: string[];
+}
+
+const PLUGIN_NAME: 'precompile-json-schemas-plugin' = 'precompile-json-schemas-plugin';
+
+async function precompileSchemasAsync(
+  buildFolderPath: string,
+  options: IPrecompileJsonSchemasPluginOptions,
+  runOptions: IHeftTaskRunHookOptions
+): Promise<number> {
+  const sourceFolder: string = path.resolve(buildFolderPath, options.sourceFolder);
+  const schemaPaths: string[] = await runOptions.globAsync('**/*.schema.json', {
+    cwd: sourceFolder,
+    absolute: true
+  });
+
+  for (const schemaPath of schemaPaths) {
+    try {
+      const relativePath: string = path
+        .relative(sourceFolder, schemaPath)
+        .replace(/\.schema\.json$/, '.validator.js');
+      if (options.destinationFolders?.length) {
+        const validatorCode: string = JsonSchema.compileStandaloneCodeFromFile(schemaPath);
+        for (const destinationFolder of options.destinationFolders) {
+          const destinationPath: string = path.resolve(buildFolderPath, destinationFolder, relativePath);
+          await FileSystem.writeFileAsync(destinationPath, validatorCode, { ensureFolderExists: true });
+        }
+      }
+      if (options.esmDestinationFolders?.length) {
+        const esmCode: string = JsonSchema.compileStandaloneCodeFromFile(schemaPath, undefined, {
+          moduleFormat: 'esm'
+        });
+        for (const destinationFolder of options.esmDestinationFolders) {
+          const destinationPath: string = path.resolve(buildFolderPath, destinationFolder, relativePath);
+          await FileSystem.writeFileAsync(destinationPath, esmCode, { ensureFolderExists: true });
+        }
+      }
+    } catch (error) {
+      const message: string = error instanceof Error ? error.message : String(error);
+      const referenceHint: string = /can't resolve reference|missing ref/i.test(message)
+        ? ' External $ref dependencies are not supported by this plugin.'
+        : '';
+      throw new Error(
+        `Failed to precompile "${schemaPath}": ${message}${referenceHint}`,
+        { cause: error }
+      );
+    }
+  }
+  return schemaPaths.length;
+}
+
+// The Heft package builds itself using the previously published version of Heft. Until that
+// version includes this plugin, its run-script-plugin invokes this entry point after TypeScript emits it.
+export async function runAsync(options: IRunScriptOptions): Promise<void> {
+  const { sourceFolder, destinationFolders, esmDestinationFolders } = options.scriptOptions;
+  if (
+    typeof sourceFolder !== 'string' ||
+    (destinationFolders !== undefined &&
+      (!Array.isArray(destinationFolders) ||
+        !destinationFolders.every((folder: unknown) => typeof folder === 'string'))) ||
+    (esmDestinationFolders !== undefined &&
+      (!Array.isArray(esmDestinationFolders) ||
+        !esmDestinationFolders.every((folder: unknown) => typeof folder === 'string'))) ||
+    !(destinationFolders?.length || esmDestinationFolders?.length)
+  ) {
+    throw new Error(
+      'Schema precompilation requires a string sourceFolder and at least one non-empty ' +
+        'destinationFolders or esmDestinationFolders string array.'
+    );
+  }
+  const count: number = await precompileSchemasAsync(
+    options.heftConfiguration.buildFolderPath,
+    { sourceFolder, destinationFolders, esmDestinationFolders },
+    options.runOptions
+  );
+  options.heftTaskSession.logger.terminal.writeLine(`Precompiled ${count} JSON schemas.`);
+}
+
+export default class PrecompileJsonSchemasPlugin
+  implements IHeftTaskPlugin<IPrecompileJsonSchemasPluginOptions>
+{
+  public apply(
+    taskSession: IHeftTaskSession,
+    heftConfiguration: HeftConfiguration,
+    options: IPrecompileJsonSchemasPluginOptions
+  ): void {
+    taskSession.hooks.run.tapPromise(PLUGIN_NAME, async (runOptions: IHeftTaskRunHookOptions) => {
+      const count: number = await precompileSchemasAsync(heftConfiguration.buildFolderPath, options, runOptions);
+      taskSession.logger.terminal.writeLine(`Precompiled ${count} JSON schemas.`);
+    });
+  }
+}

apps/heft/src/schemas/precompile-json-schemas-options.schema.json

@@ -0,0 +1,29 @@
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "title": "Precompile JSON Schemas Plugin Options",
+  "type": "object",
+  "additionalProperties": false,
+  "required": ["sourceFolder"],
+  "anyOf": [
+    { "required": ["destinationFolders"] },
+    { "required": ["esmDestinationFolders"] }
+  ],
+  "properties": {
+    "sourceFolder": {
+      "type": "string",
+      "description": "Folder containing JSON schema files, relative to the project root."
+    },
+    "destinationFolders": {
+      "type": "array",
+      "minItems": 1,
+      "items": { "type": "string" },
+      "description": "Folders for generated CommonJS validator modules, relative to the project root."
+    },
+    "esmDestinationFolders": {
+      "type": "array",
+      "minItems": 1,
+      "items": { "type": "string" },
+      "description": "Folders for generated ESM validator modules, relative to the project root."
+    }
+  }
+}

apps/heft/src/schemas/validator.d.ts

@@ -0,0 +1,4 @@
+declare module '*.validator.js' {
+  const validator: import('@rushstack/node-core-library').IJsonSchemaCompiledValidator;
+  export default validator;
+}

apps/heft/src/utilities/CoreConfigFiles.ts

@@ -15,6 +15,9 @@ import type { IRigConfig } from '@rushstack/rig-package';
 
 import type { IDeleteOperation } from '../plugins/DeleteFilesPlugin';
 import type { INodeServicePluginConfiguration } from '../plugins/NodeServicePlugin';
+import heftSchemaValidator from '../schemas/heft.validator.js';
+import legacySchemaValidator from '../schemas/heft-legacy.validator.js';
+import nodeServiceSchemaValidator from '../schemas/node-service.validator.js';
 import { Constants } from './Constants';
 
 export interface IHeftConfigurationJsonActionReference {
@@ -108,11 +111,11 @@ export class CoreConfigFiles {
         }
       };
 
-      const schemaObject: object = await import('../schemas/heft.schema.json');
       // eslint-disable-next-line require-atomic-updates
       _heftConfigFileLoader = new ProjectConfigurationFile<IHeftConfigurationJson>({
         projectRelativeFilePath: CoreConfigFiles.heftConfigurationProjectRelativeFilePath,
-        jsonSchemaObject: schemaObject,
+        jsonSchemaValidator: heftSchemaValidator,
+        jsonSchemaValidatorName: 'heft.schema.json',
         propertyInheritanceDefaults: {
           array: { inheritanceType: InheritanceType.append },
           object: { inheritanceType: InheritanceType.merge }
@@ -156,11 +159,11 @@ export class CoreConfigFiles {
         // match the legacy schema. We don't need to worry about the resulting object, we just
         // want to see if it parses. We will use the ConfigurationFile class to load it to ensure
         // that we follow the "extends" chain for the entire config file.
-        const legacySchemaObject: object = await import('../schemas/heft-legacy.schema.json');
         const legacyConfigFileLoader: ProjectConfigurationFile<unknown> =
           new ProjectConfigurationFile<unknown>({
             projectRelativeFilePath: CoreConfigFiles.heftConfigurationProjectRelativeFilePath,
-            jsonSchemaObject: legacySchemaObject
+            jsonSchemaValidator: legacySchemaValidator,
+            jsonSchemaValidatorName: 'heft-legacy.schema.json'
           });
         await legacyConfigFileLoader.loadConfigurationFileForProjectAsync(terminal, projectPath, rigConfig);
       } catch (e2) {
@@ -230,11 +233,11 @@ export class CoreConfigFiles {
     rigConfig?: IRigConfig | undefined
   ): Promise<INodeServicePluginConfiguration | undefined> {
     if (!_nodeServiceConfigurationLoader) {
-      const schemaObject: object = await import('../schemas/node-service.schema.json');
       // eslint-disable-next-line require-atomic-updates
       _nodeServiceConfigurationLoader = new ProjectConfigurationFile<INodeServicePluginConfiguration>({
         projectRelativeFilePath: CoreConfigFiles.nodeServiceConfigurationProjectRelativeFilePath,
-        jsonSchemaObject: schemaObject
+        jsonSchemaValidator: nodeServiceSchemaValidator,
+        jsonSchemaValidatorName: 'node-service.schema.json'
       });
     }