thefoldwithin-earth/tools/generate-index.mjs

#!/usr/bin/env node
import { promises as fs } from "fs";
import path from "path";
import pdf from "pdf-parse";

const ROOT = "public";
const OUT = path.join(ROOT, "index.json");
const EXCERPT_LENGTH = 400;

function dateFromName(name) {
  const m = name.match(/^(\d{4}-\d{2}-\d{2})/);
  return m ? new Date(m[0]).getTime() : null;
}

async function readHead(abs, full = false) {
  const fh = await fs.open(abs, "r");
  const size = full ? await fs.stat(abs).then(s => Math.min(s.size, EXCERPT_LENGTH * 2)) : 64 * 1024;
  const buf = Buffer.alloc(size);
  const { bytesRead } = await fh.read(buf, 0, size, 0);
  await fh.close();
  return buf.slice(0, bytesRead).toString("utf8");
}

function parseTitle(raw, ext) {
  if (ext === ".md") return raw.match(/^\s*#\s+(.+?)\s*$/m)?.[1].trim();
  if (ext === ".html") return raw.match(/<title[^>]*>([^<]+)<\/title>/i)?.[1].trim();
  return null;
}

function extractExcerpt(raw, ext) {
  if (ext === ".md") raw = raw.replace(/^#.*\n/, '').trim();
  if (ext === ".html") raw = raw.replace(/<head>[\s\S]*<\/head>/i, '').replace(/<[^>]+>/g, ' ').trim();
  return raw.replace(/\s+/g, ' ').slice(0, EXCERPT_LENGTH);
}

function extractTags(raw, ext, pdfData) {
  let tags = [];
  if (ext === ".md") {
    const m = raw.match(/^\s*tags:\s*(.+)$/im);
    if (m) tags = m[1].split(',').map(t => t.trim().toLowerCase());
  } else if (ext === ".html") {
    const m = raw.match(/<meta\s+name="keywords"\s+content="([^"]+)"/i);
    if (m) tags = m[1].split(',').map(t => t.trim().toLowerCase());
  } else if (ext === ".pdf" && pdfData?.info?.Subject) {
    tags = pdfData.info.Subject.split(',').map(t => t.trim().toLowerCase());
  }
  return tags;
}

async function collectFiles(relBase = "", flat = []) {
  const abs = path.join(ROOT, relBase);
  const entries = await fs.readdir(abs, { withFileTypes: true });

  for (const e of entries) {
    if (e.name.startsWith(".")) continue;
    const rel = path.posix.join(relBase, e.name);
    const absPath = path.join(ROOT, rel);
    if (e.isDirectory()) {
      await collectFiles(rel, flat);
      continue;
    }

    const ext = path.posix.extname(e.name).toLowerCase();
    if (![".md", ".html", ".pdf"].includes(ext)) continue;

    const st = await fs.stat(absPath);
    let raw, pdfData, title;
    if (ext === ".pdf") {
      const buffer = await fs.readFile(absPath);
      pdfData = await pdf(buffer);
      raw = pdfData.text;
      title = pdfData.info.Title || e.name.replace(/\.pdf$/, "").trim();
    } else {
      raw = await readHead(absPath, true);
      title = parseTitle(raw, ext) || e.name.replace(new RegExp(`\\${ext}$`), "").trim();
    }

    const ctime = st.birthtimeMs || st.mtimeMs || dateFromName(e.name) || st.mtimeMs;
    const mtime = dateFromName(e.name) ?? st.mtimeMs;
    const baseName = e.name.toLowerCase();

    flat.push({
      type: "file",
      name: e.name,
      title,
      path: rel,
      ext,
      ctime,
      mtime,
      excerpt: extractExcerpt(raw, ext),
      tags: extractTags(raw, ext, pdfData),
      isIndex: baseName.startsWith("index."),
      isPinned: baseName.startsWith("pinned.")
    });
  }
  return flat;
}

(async () => {
  try {
    const flat = await collectFiles();
    // ONLY non-index files → dropdown
    const sections = [...new Set(flat.filter(f => !f.isIndex).map(f => f.path.split("/")[0]))].sort();
    const allTags = [...new Set(flat.flatMap(f => f.tags))].sort();
    await fs.writeFile(OUT, JSON.stringify({ flat, sections, tags: allTags }, null, 2));
    console.log(`index.json built: ${flat.length} files, ${sections.length} sections, ${allTags.length} tags.`);
  } catch (e) {
    console.error("Build failed:", e);
    process.exit(1);
  }
})();
Update generate-index.mjs 2025-11-08 11:23:21 -06:00			`#!/usr/bin/env node`
Update generate-index.mjs 2025-11-08 11:29:10 -06:00			`import { promises as fs } from "fs";`
Create generate-index.mjs 2025-11-08 09:05:04 -06:00			`import path from "path";`
Update generate-index.mjs 2025-11-08 15:47:51 -06:00			`import pdf from "pdf-parse";`
Update generate-index.mjs 2025-11-08 14:34:15 -06:00
Update generate-index.mjs 2025-11-08 15:24:49 -06:00			`const ROOT = "public";`
Update generate-index.mjs 2025-11-08 11:23:21 -06:00			`const OUT = path.join(ROOT, "index.json");`
Update generate-index.mjs 2025-11-08 18:21:53 -06:00			`const EXCERPT_LENGTH = 400;`
Update generate-index.mjs 2025-11-08 10:37:54 -06:00
Update generate-index.mjs 2025-11-08 11:29:10 -06:00			`function dateFromName(name) {`
			`const m = name.match(/^(\d{4}-\d{2}-\d{2})/);`
			`return m ? new Date(m[0]).getTime() : null;`
Update generate-index.mjs 2025-11-08 10:37:54 -06:00			`}`
Update generate-index.mjs 2025-11-08 15:24:49 -06:00
Update generate-index.mjs 2025-11-08 18:21:53 -06:00			`async function readHead(abs, full = false) {`
Update generate-index.mjs 2025-11-08 11:29:10 -06:00			`const fh = await fs.open(abs, "r");`
Update generate-index.mjs 2025-11-08 18:21:53 -06:00			`const size = full ? await fs.stat(abs).then(s => Math.min(s.size, EXCERPT_LENGTH * 2)) : 64 * 1024;`
			`const buf = Buffer.alloc(size);`
			`const { bytesRead } = await fh.read(buf, 0, size, 0);`
Update generate-index.mjs 2025-11-08 10:37:54 -06:00			`await fh.close();`
Update generate-index.mjs 2025-11-08 11:29:10 -06:00			`return buf.slice(0, bytesRead).toString("utf8");`
Update generate-index.mjs 2025-11-08 10:37:54 -06:00			`}`
Update generate-index.mjs 2025-11-08 15:24:49 -06:00
Update generate-index.mjs 2025-11-08 11:29:10 -06:00			`function parseTitle(raw, ext) {`
Update generate-index.mjs 2025-11-08 15:24:49 -06:00			`if (ext === ".md") return raw.match(/^\s#\s+(.+?)\s$/m)?.[1].trim();`
			`if (ext === ".html") return raw.match(/<title[^>]*>([^<]+)<\/title>/i)?.[1].trim();`
Update generate-index.mjs 2025-11-08 10:37:54 -06:00			`return null;`
			`}`

Update generate-index.mjs 2025-11-08 16:07:52 -06:00			`function extractExcerpt(raw, ext) {`
Update generate-index.mjs 2025-11-08 18:21:53 -06:00			`if (ext === ".md") raw = raw.replace(/^#.*\n/, '').trim();`
			`if (ext === ".html") raw = raw.replace(/<head>[\s\S]*<\/head>/i, '').replace(/<[^>]+>/g, ' ').trim();`
Update generate-index.mjs 2025-11-08 16:07:52 -06:00			`return raw.replace(/\s+/g, ' ').slice(0, EXCERPT_LENGTH);`
			`}`

Update generate-index.mjs 2025-11-08 18:21:53 -06:00			`function extractTags(raw, ext, pdfData) {`
Update generate-index.mjs 2025-11-08 16:07:52 -06:00			`let tags = [];`
			`if (ext === ".md") {`
Update generate-index.mjs 2025-11-08 18:21:53 -06:00			`const m = raw.match(/^\stags:\s(.+)$/im);`
			`if (m) tags = m[1].split(',').map(t => t.trim().toLowerCase());`
Update generate-index.mjs 2025-11-08 16:07:52 -06:00			`} else if (ext === ".html") {`
Update generate-index.mjs 2025-11-08 18:21:53 -06:00			`const m = raw.match(/<meta\s+name="keywords"\s+content="([^"]+)"/i);`
			`if (m) tags = m[1].split(',').map(t => t.trim().toLowerCase());`
Update generate-index.mjs 2025-11-08 16:07:52 -06:00			`} else if (ext === ".pdf" && pdfData?.info?.Subject) {`
			`tags = pdfData.info.Subject.split(',').map(t => t.trim().toLowerCase());`
			`}`
			`return tags;`
			`}`

Update generate-index.mjs 2025-11-08 15:34:32 -06:00			`async function collectFiles(relBase = "", flat = []) {`
Update generate-index.mjs 2025-11-08 11:30:41 -06:00			`const abs = path.join(ROOT, relBase);`
			`const entries = await fs.readdir(abs, { withFileTypes: true });`
Update generate-index.mjs 2025-11-08 14:34:15 -06:00
Update generate-index.mjs 2025-11-08 11:30:41 -06:00			`for (const e of entries) {`
Update generate-index.mjs 2025-11-08 14:40:38 -06:00			`if (e.name.startsWith(".")) continue;`
Update generate-index.mjs 2025-11-08 11:30:41 -06:00			`const rel = path.posix.join(relBase, e.name);`
			`const absPath = path.join(ROOT, rel);`
			`if (e.isDirectory()) {`
Update generate-index.mjs 2025-11-08 15:34:32 -06:00			`await collectFiles(rel, flat);`
Update generate-index.mjs 2025-11-08 11:30:41 -06:00			`continue;`
			`}`
Update generate-index.mjs 2025-11-08 14:34:15 -06:00
Update generate-index.mjs 2025-11-08 11:30:41 -06:00			`const ext = path.posix.extname(e.name).toLowerCase();`
Update generate-index.mjs 2025-11-08 18:21:53 -06:00			`if (![".md", ".html", ".pdf"].includes(ext)) continue;`

Update generate-index.mjs 2025-11-08 11:30:41 -06:00			`const st = await fs.stat(absPath);`
Update generate-index.mjs 2025-11-08 16:07:52 -06:00			`let raw, pdfData, title;`
Update generate-index.mjs 2025-11-08 15:47:51 -06:00			`if (ext === ".pdf") {`
			`const buffer = await fs.readFile(absPath);`
Update generate-index.mjs 2025-11-08 16:07:52 -06:00			`pdfData = await pdf(buffer);`
			`raw = pdfData.text;`
Update generate-index.mjs 2025-11-08 15:47:51 -06:00			`title = pdfData.info.Title \|\| e.name.replace(/\.pdf$/, "").trim();`
			`} else {`
Update generate-index.mjs 2025-11-08 16:07:52 -06:00			`raw = await readHead(absPath, true);`
Update generate-index.mjs 2025-11-08 15:47:51 -06:00			title = parseTitle(raw, ext) \|\| e.name.replace(new RegExp(`\\${ext}$`), "").trim();
			`}`
Update generate-index.mjs 2025-11-08 18:21:53 -06:00
Update generate-index.mjs 2025-11-08 19:15:52 -06:00			`const ctime = st.birthtimeMs \|\| st.mtimeMs \|\| dateFromName(e.name) \|\| st.mtimeMs;`
Update generate-index.mjs 2025-11-08 11:30:41 -06:00			`const mtime = dateFromName(e.name) ?? st.mtimeMs;`
Update generate-index.mjs 2025-11-08 19:15:52 -06:00			`const baseName = e.name.toLowerCase();`
Update generate-index.mjs 2025-11-08 15:24:49 -06:00
Update generate-index.mjs 2025-11-08 15:34:32 -06:00			`flat.push({`
Update generate-index.mjs 2025-11-08 11:30:41 -06:00			`type: "file",`
			`name: e.name,`
			`title,`
			`path: rel,`
			`ext,`
Update generate-index.mjs 2025-11-08 19:15:52 -06:00			`ctime,`
			`mtime,`
Update generate-index.mjs 2025-11-08 18:21:53 -06:00			`excerpt: extractExcerpt(raw, ext),`
			`tags: extractTags(raw, ext, pdfData),`
Update generate-index.mjs 2025-11-08 19:22:15 -06:00			`isIndex: baseName.startsWith("index."),`
Update generate-index.mjs 2025-11-08 19:15:52 -06:00			`isPinned: baseName.startsWith("pinned.")`
Update generate-index.mjs 2025-11-08 11:30:41 -06:00			`});`
			`}`
Update generate-index.mjs 2025-11-08 15:34:32 -06:00			`return flat;`
Update generate-index.mjs 2025-11-08 11:30:41 -06:00			`}`

			`(async () => {`
			`try {`
Update generate-index.mjs 2025-11-08 15:34:32 -06:00			`const flat = await collectFiles();`
Update generate-index.mjs 2025-11-08 19:26:48 -06:00			`// ONLY non-index files → dropdown`
Update generate-index.mjs 2025-11-08 18:41:51 -06:00			`const sections = [...new Set(flat.filter(f => !f.isIndex).map(f => f.path.split("/")[0]))].sort();`
Update generate-index.mjs 2025-11-08 16:07:52 -06:00			`const allTags = [...new Set(flat.flatMap(f => f.tags))].sort();`
Update generate-index.mjs 2025-11-08 18:09:56 -06:00			`await fs.writeFile(OUT, JSON.stringify({ flat, sections, tags: allTags }, null, 2));`
			console.log(`index.json built: ${flat.length} files, ${sections.length} sections, ${allTags.length} tags.`);
Update generate-index.mjs 2025-11-08 11:30:41 -06:00			`} catch (e) {`
Update generate-index.mjs 2025-11-08 15:24:49 -06:00			`console.error("Build failed:", e);`
Update generate-index.mjs 2025-11-08 11:30:41 -06:00			`process.exit(1);`
			`}`
			`})();`