Generation scripts now acquire a lock

2026-06-04 21:04:37 +08:00 · 2025-03-10 10:47:10 -07:00 · 2025-03-10 10:47:10 -07:00 · ea291337cb
commit ea291337cb
parent da79285e7f
3 changed files with 126 additions and 95 deletions
--- a/scripts/generate-docs.ts
+++ b/scripts/generate-docs.ts
@ -1,7 +1,7 @@
 import fs from "fs";
 import path from "path";
 import yaml from "yaml";
-import { PLATFORMS, copyFromSrcToDest, processMacros, writeFileSyncIfChanged } from "./utils";
+import { PLATFORMS, copyFromSrcToDest, processMacros, withGeneratorLock, writeFileSyncIfChanged } from "./utils";

 interface DocObject {
  platform?: string;
@ -65,32 +65,36 @@ function processDocObject(obj: any, platforms: string[]): { result: any, validPa
  }
 }

-const docsDir = path.resolve(__dirname, "..", "docs", "fern");
-const templateDir = path.join(docsDir, "docs", "pages-template");
-const ymlTemplatePath = path.join(docsDir, "docs-template.yml");

-for (const platform of ["next", "js", "react", "python"]) {
-  const destDir = path.join(docsDir, 'docs', `pages-${platform}`);

-  const mainYmlContent = fs.readFileSync(ymlTemplatePath, "utf-8");
-  const macroProcessed = processMacros(mainYmlContent, PLATFORMS[platform]);
-  const template = yaml.parse(macroProcessed);
-  const { result: processed, validPaths: processedValidPaths } = processDocObject(template, PLATFORMS[platform]);
-  const output = yaml.stringify(processed);
-  writeFileSyncIfChanged(path.join(docsDir, `${platform}.yml`), output);
+withGeneratorLock(async () => {
+  const docsDir = path.resolve(__dirname, "..", "docs", "fern");
+  const templateDir = path.join(docsDir, "docs", "pages-template");
+  const ymlTemplatePath = path.join(docsDir, "docs-template.yml");

-  // Copy the entire template directory, processing macros for each file
-  copyFromSrcToDest({
-    srcDir: templateDir,
-    destDir,
-    editFn: (relativePath, content) => {
-      return processMacros(content, PLATFORMS[platform]);
-    },
-    filterFn: (relativePath) => {
-      if (relativePath.endsWith('.mdx') && !relativePath.startsWith('snippets')) {
-        return processedValidPaths.includes(relativePath);
+  for (const platform of ["next", "js", "react", "python"]) {
+    const destDir = path.join(docsDir, 'docs', `pages-${platform}`);
+
+    const mainYmlContent = fs.readFileSync(ymlTemplatePath, "utf-8");
+    const macroProcessed = processMacros(mainYmlContent, PLATFORMS[platform]);
+    const template = yaml.parse(macroProcessed);
+    const { result: processed, validPaths: processedValidPaths } = processDocObject(template, PLATFORMS[platform]);
+    const output = yaml.stringify(processed);
+    writeFileSyncIfChanged(path.join(docsDir, `${platform}.yml`), output);
+
+    // Copy the entire template directory, processing macros for each file
+    copyFromSrcToDest({
+      srcDir: templateDir,
+      destDir,
+      editFn: (relativePath, content) => {
+        return processMacros(content, PLATFORMS[platform]);
+      },
+      filterFn: (relativePath) => {
+        if (relativePath.endsWith('.mdx') && !relativePath.startsWith('snippets')) {
+          return processedValidPaths.includes(relativePath);
+        }
+        return true;
      }
-      return true;
-    }
-  });
-}
+    });
+  }
+}).catch(console.error);
--- a/scripts/generate-sdks.ts
+++ b/scripts/generate-sdks.ts
@ -1,6 +1,6 @@
 import fs from "fs";
 import path from "path";
-import { COMMENT_BLOCK, COMMENT_LINE, PLATFORMS, copyFromSrcToDest, processMacros, writeFileSyncIfChanged } from "./utils";
+import { COMMENT_BLOCK, COMMENT_LINE, PLATFORMS, copyFromSrcToDest, processMacros, withGeneratorLock, writeFileSyncIfChanged } from "./utils";

 /**
 * Main function to generate from a template:
@ -123,65 +123,67 @@ function baseEditFn(options: {
 }


-const baseDir = path.resolve(__dirname, "..", "packages");
-const srcDir = path.resolve(baseDir, "template");
+withGeneratorLock(async () => {
+  const baseDir = path.resolve(__dirname, "..", "packages");
+  const srcDir = path.resolve(baseDir, "template");

-// Copy package-template.json to package.json in the template,
-// applying macros and adding a comment field.
-const packageTemplateContent = fs.readFileSync(
-  path.join(srcDir, "package-template.json"),
-  "utf-8"
-);
-const processedPackageJson = processMacros(packageTemplateContent, PLATFORMS["template"]);
-writeFileSyncIfChanged(
-  path.join(srcDir, "package.json"),
-  processPackageJson(processedPackageJson)
-);
+  // Copy package-template.json to package.json in the template,
+  // applying macros and adding a comment field.
+  const packageTemplateContent = fs.readFileSync(
+    path.join(srcDir, "package-template.json"),
+    "utf-8"
+  );
+  const processedPackageJson = processMacros(packageTemplateContent, PLATFORMS["template"]);
+  writeFileSyncIfChanged(
+    path.join(srcDir, "package.json"),
+    processPackageJson(processedPackageJson)
+  );

-generateFromTemplate({
-  src: srcDir,
-  dest: path.resolve(baseDir, "js"),
-  editFn: (relativePath, content) => {
-    return baseEditFn({ relativePath, content, platforms: PLATFORMS["js"] });
-  },
-  filterFn: (relativePath) => {
-    const ignores = [
-      "postcss.config.js",
-      "tailwind.config.js",
-      "quetzal.config.json",
-      "components.json",
-      ".env",
-      ".env.local",
-      "scripts/",
-      "quetzal-translations/",
-      "src/components/",
-      "src/components-page/",
-      "src/generated/",
-      "src/providers/",
-      "src/global.css",
-      "src/global.d.ts",
-    ];
+  generateFromTemplate({
+    src: srcDir,
+    dest: path.resolve(baseDir, "js"),
+    editFn: (relativePath, content) => {
+      return baseEditFn({ relativePath, content, platforms: PLATFORMS["js"] });
+    },
+    filterFn: (relativePath) => {
+      const ignores = [
+        "postcss.config.js",
+        "tailwind.config.js",
+        "quetzal.config.json",
+        "components.json",
+        ".env",
+        ".env.local",
+        "scripts/",
+        "quetzal-translations/",
+        "src/components/",
+        "src/components-page/",
+        "src/generated/",
+        "src/providers/",
+        "src/global.css",
+        "src/global.d.ts",
+      ];

-    if (ignores.some((ignorePath) => relativePath.startsWith(ignorePath)) || relativePath.endsWith(".tsx")) {
-      return false;
-    } else {
-      return true;
-    }
-  },
-});
+      if (ignores.some((ignorePath) => relativePath.startsWith(ignorePath)) || relativePath.endsWith(".tsx")) {
+        return false;
+      } else {
+        return true;
+      }
+    },
+  });

-generateFromTemplate({
-  src: srcDir,
-  dest: path.resolve(baseDir, "stack"),
-  editFn: (relativePath, content) => {
-    return baseEditFn({ relativePath, content, platforms: PLATFORMS["next"] });
-  },
-});
+  generateFromTemplate({
+    src: srcDir,
+    dest: path.resolve(baseDir, "stack"),
+    editFn: (relativePath, content) => {
+      return baseEditFn({ relativePath, content, platforms: PLATFORMS["next"] });
+    },
+  });

-generateFromTemplate({
-  src: srcDir,
-  dest: path.resolve(baseDir, "react"),
-  editFn: (relativePath, content) => {
-    return baseEditFn({ relativePath, content, platforms: PLATFORMS["react"] });
-  },
-});
+  generateFromTemplate({
+    src: srcDir,
+    dest: path.resolve(baseDir, "react"),
+    editFn: (relativePath, content) => {
+      return baseEditFn({ relativePath, content, platforms: PLATFORMS["react"] });
+    },
+  });
+}).catch(console.error);
--- a/scripts/utils.ts
+++ b/scripts/utils.ts
@ -13,6 +13,36 @@ export const PLATFORMS = {
  "python": ['python', 'python-like'],
 }

+export const withGeneratorLock = async <T>(fn: () => Promise<T>) => {
+  const lockFilePath = path.resolve(__dirname, "../generator-lock-file.untracked.lock");
+  while (true) {
+    try {
+      fs.writeFileSync(lockFilePath, Date.now().toString(), { flag: 'wx' });
+      break;
+    } catch (e) {
+      if ("code" in e && e.code === "EEXIST") {
+        const millis = +fs.readFileSync(lockFilePath, 'utf-8');
+        if (Date.now() - millis > 5 * 60 * 1000) {
+          console.warn(`Generator lock file ${lockFilePath} exists, but is older than 5 minutes. Assuming it's stale and deleting.`);
+          fs.unlinkSync(lockFilePath);  // TODO: this should be done atomically
+          await new Promise((resolve) => setTimeout(resolve, 5000));
+          continue;
+        } else {
+          console.log(`Generator lock file ${lockFilePath} exists. Waiting for it to be released...`);
+          await new Promise((resolve) => setTimeout(resolve, 2000 * Math.random()));
+          continue;
+        }
+      } else {
+        throw e;
+      }
+    }
+  }
+  try {
+    return await fn();
+  } finally {
+    fs.unlinkSync(lockFilePath);
+  }
+}

 export function processMacros(content: string, platforms: string[]): string {
  const lines = content.split('\n');
@ -242,18 +272,13 @@ export function processMacros(content: string, platforms: string[]): string {
 }
 
 export function writeFileSyncIfChanged(path: string, content: string | Buffer): void {
+  if (typeof content === 'string') {
+    content = Buffer.from(content);
+  }
  if (fs.existsSync(path)) {
-    const existingContent = fs.readFileSync(path);
-    if (Buffer.isBuffer(content)) {
-      // For binary files, compare buffers
-      if (Buffer.compare(existingContent, content) === 0) {
-        return;
-      }
-    } else {
-      // For text files, compare strings
-      if (existingContent.toString('utf-8') === content) {
-        return;
-      }
+    const existingContent = fs.readFileSync(path, { encoding: null });
+    if (Buffer.compare(existingContent, content) === 0) {
+      return;
    }
  }
  fs.writeFileSync(path, content);