fresh start
This commit is contained in:
Mark Randall Havens
parent
62c3e2d368
commit
7f86647175
570 changed files with 4895 additions and 866 deletions
141
tools/build.js
Executable file
141
tools/build.js
Executable file
|
|
@ -0,0 +1,141 @@
|
|||
import fs from 'fs/promises';
|
||||
import path from 'path';
|
||||
import matter from 'gray-matter';
|
||||
import yaml from 'js-yaml';
|
||||
import md from 'markdown-it';
|
||||
import sanitizer from 'markdown-it-sanitizer';
|
||||
import crypto from 'crypto';
|
||||
import nacl from 'tweetnacl';
|
||||
import { validate, detectCycles } from './foldlint.js';
|
||||
|
||||
const mdParser = md().use(sanitizer);
|
||||
|
||||
// Collect files async, deterministic sort
|
||||
async function collectFiles(dir) {
|
||||
let files = [];
|
||||
const entries = await fs.readdir(dir, { withFileTypes: true });
|
||||
entries.sort((a, b) => a.name.localeCompare(b.name));
|
||||
for (const entry of entries) {
|
||||
const fullPath = path.join(dir, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
files = [...files, ...(await collectFiles(fullPath))];
|
||||
} else if (entry.name.endsWith('.md')) {
|
||||
files.push(fullPath);
|
||||
}
|
||||
}
|
||||
return files;
|
||||
}
|
||||
|
||||
// Generate HTML
|
||||
function generateHTML(meta, bodyHtml, exits) {
|
||||
return `
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<title>${meta.title}</title>
|
||||
<link rel="stylesheet" href="/styles.css">
|
||||
<script src="/witness.js"></script>
|
||||
<meta name="description" content="${meta.summary}">
|
||||
</head>
|
||||
<body>
|
||||
<div class="room">
|
||||
<h1>${meta.title}</h1>
|
||||
${bodyHtml}
|
||||
<div class="exits">
|
||||
${exits.map(exit => `<a href="/${exit.to.replace(':', '_').split('@')[0]}.html">${exit.label}</a>`).join('')}
|
||||
</div>
|
||||
<div id="chat"></div>
|
||||
<input id="msg"><button id="send">Send</button>
|
||||
</div>
|
||||
<script>witness.connect('${meta.id}');</script>
|
||||
</body>
|
||||
</html>
|
||||
`;
|
||||
}
|
||||
|
||||
// Canonical hash (exclude id from frontmatter)
|
||||
function canonicalHash(meta, body) {
|
||||
const metaClone = { ...meta };
|
||||
delete metaClone.id; // Exclude id to avoid circularity
|
||||
const frontYaml = yaml.dump(metaClone, { noRefs: true }).trim();
|
||||
const content = frontYaml + '\n' + body.trim();
|
||||
const lines = content.split('\n');
|
||||
const trimmedLines = lines.map(line => line.replace(/\s+$/, ''));
|
||||
const normalized = trimmedLines.join('\n');
|
||||
return crypto.createHash('sha256').update(normalized).digest('hex');
|
||||
}
|
||||
|
||||
// Build
|
||||
async function build() {
|
||||
const atlasDir = path.join(process.cwd(), 'atlas');
|
||||
const distDir = path.join(process.cwd(), 'dist');
|
||||
await fs.mkdir(distDir, { recursive: true });
|
||||
|
||||
const files = await collectFiles(atlasDir);
|
||||
|
||||
// Validate all
|
||||
for (const file of files) {
|
||||
await validate(file);
|
||||
console.log(`Validated: ${file}`);
|
||||
}
|
||||
|
||||
// Build graph
|
||||
const graph = {};
|
||||
const idToFile = {};
|
||||
const slugToId = {};
|
||||
for (const file of files) {
|
||||
const content = await fs.readFile(file, 'utf8');
|
||||
const { data: meta } = matter(content);
|
||||
graph[meta.id] = meta.exits ? meta.exits.map(e => e.to) : [];
|
||||
idToFile[meta.id] = file;
|
||||
slugToId[meta.id.split('@')[0]] = meta.id; // Map kind:slug to full id
|
||||
}
|
||||
|
||||
// Check broken links and cycles
|
||||
Object.keys(graph).forEach(id => {
|
||||
graph[id].forEach(to => {
|
||||
let resolvedTo = to;
|
||||
if (!graph[to] && slugToId[to]) {
|
||||
resolvedTo = slugToId[to]; // Resolve kind:slug to full id
|
||||
}
|
||||
if (!graph[resolvedTo]) {
|
||||
console.error(`Available IDs: ${Object.keys(graph).join(', ')}`);
|
||||
throw new Error(`Broken link: ${to} from ${id}`);
|
||||
}
|
||||
});
|
||||
});
|
||||
detectCycles(graph); // Throws if cycle
|
||||
|
||||
// Generate HTML
|
||||
const buildManifest = { files: [] };
|
||||
for (const file of files) {
|
||||
const content = await fs.readFile(file, 'utf8');
|
||||
const { data: meta, content: body } = matter(content);
|
||||
|
||||
// Hash verify
|
||||
const computed = canonicalHash(meta, body);
|
||||
const idHash = meta.id.split('@sha256:')[1];
|
||||
if (computed !== idHash) throw new Error(`Hash mismatch in ${file}: computed=${computed}, expected=${idHash}`);
|
||||
|
||||
const bodyHtml = mdParser.render(body);
|
||||
const html = generateHTML(meta, bodyHtml, meta.exits || []);
|
||||
|
||||
const slug = meta.id.split(':')[1].split('@')[0];
|
||||
const outPath = path.join(distDir, `${slug}.html`);
|
||||
await fs.writeFile(outPath, html);
|
||||
buildManifest.files.push({ path: outPath, hash: crypto.createHash('sha256').update(html).digest('hex') });
|
||||
}
|
||||
|
||||
// Copy public
|
||||
await fs.cp(path.join(process.cwd(), 'public'), distDir, { recursive: true });
|
||||
|
||||
// Sitemap stub
|
||||
await fs.writeFile(path.join(distDir, 'sitemap.xml'), '<xml>Stub</xml>');
|
||||
|
||||
// Write build manifest
|
||||
await fs.writeFile(path.join(distDir, 'manifest.json'), JSON.stringify(buildManifest));
|
||||
|
||||
console.log('Build complete.');
|
||||
}
|
||||
|
||||
build().catch(console.error);
|
||||
141
tools/foldlint.js
Executable file
141
tools/foldlint.js
Executable file
|
|
@ -0,0 +1,141 @@
|
|||
import fs from 'fs/promises';
|
||||
import path from 'path';
|
||||
import matter from 'gray-matter';
|
||||
import yaml from 'js-yaml';
|
||||
import crypto from 'crypto';
|
||||
import nacl from 'tweetnacl';
|
||||
|
||||
// Error codes
|
||||
const ERRORS = {
|
||||
S001: 'Missing field',
|
||||
S002: 'Invalid type',
|
||||
G001: 'Broken exit',
|
||||
G002: 'Circular link',
|
||||
H001: 'Hash mismatch',
|
||||
V001: 'Spec version unsupported',
|
||||
M001: 'Migration checksum fail',
|
||||
Sig001: 'Signature invalid',
|
||||
V002: 'Unquoted spec_version; quote recommended',
|
||||
Y001: 'Block scalar in exits.to; use inline string',
|
||||
G003: 'Invalid exit hash'
|
||||
};
|
||||
|
||||
// Canonical hash (exclude id from frontmatter)
|
||||
function canonicalHash(meta, body) {
|
||||
const metaClone = { ...meta };
|
||||
delete metaClone.id; // Exclude id to avoid circularity
|
||||
const frontYaml = yaml.dump(metaClone, { noRefs: true, lineWidth: -1 }).trim();
|
||||
const content = frontYaml + '\n' + body.trim();
|
||||
const lines = content.split('\n');
|
||||
const trimmedLines = lines.map(line => line.replace(/\s+$/, ''));
|
||||
const normalized = trimmedLines.join('\n');
|
||||
return crypto.createHash('sha256').update(normalized).digest('hex');
|
||||
}
|
||||
|
||||
// DFS cycle detection
|
||||
function detectCycles(graph) {
|
||||
const visited = new Set();
|
||||
const recStack = new Set();
|
||||
|
||||
function dfs(node) {
|
||||
visited.add(node);
|
||||
recStack.add(node);
|
||||
for (const neighbor of graph[node] || []) {
|
||||
if (!visited.has(neighbor)) {
|
||||
if (dfs(neighbor)) return true;
|
||||
} else if (recStack.has(neighbor)) {
|
||||
return true; // Cycle
|
||||
}
|
||||
}
|
||||
recStack.delete(node);
|
||||
return false;
|
||||
}
|
||||
|
||||
for (const node in graph) {
|
||||
if (!visited.has(node) && dfs(node)) {
|
||||
throw new Error(ERRORS.G002);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Validate async
|
||||
async function validate(file, idToFile = {}) {
|
||||
const content = await fs.readFile(file, 'utf8');
|
||||
const { data: meta, content: body } = matter(content);
|
||||
|
||||
// Required fields
|
||||
const required = ['spec_version', 'id', 'title', 'author', 'date', 'kind', 'medium', 'summary'];
|
||||
required.forEach(field => {
|
||||
if (!meta[field]) throw new Error(`${ERRORS.S001}: ${field}`);
|
||||
});
|
||||
|
||||
// Enums
|
||||
const specVersion = String(meta.spec_version);
|
||||
if (specVersion !== '2.3') {
|
||||
throw new Error(`${ERRORS.V001}: found ${specVersion}`);
|
||||
}
|
||||
if (typeof meta.spec_version !== 'string') {
|
||||
console.warn(`${ERRORS.V002}: ${file} (parsed as ${typeof meta.spec_version}: ${meta.spec_version})`);
|
||||
}
|
||||
if (!['room', 'post', 'artifact'].includes(meta.kind)) throw new Error(`${ERRORS.S002}: kind`);
|
||||
if (!['textual', 'graphical', 'interactive'].includes(meta.medium)) throw new Error(`${ERRORS.S002}: medium`);
|
||||
|
||||
// Check exits.to for block scalars
|
||||
if (meta.exits) {
|
||||
meta.exits.forEach((exit, i) => {
|
||||
if (typeof exit.to === 'string' && exit.to.includes('\n')) {
|
||||
console.warn(`${ERRORS.Y001}: ${file} at exits[${i}].to`);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Hash verify
|
||||
const computed = canonicalHash(meta, body);
|
||||
const idHash = meta.id.split('@sha256:')[1];
|
||||
if (computed !== idHash) throw new Error(`${ERRORS.H001}: computed=${computed}, expected=${idHash}`);
|
||||
|
||||
// Validate exit hashes
|
||||
if (meta.exits && idToFile) {
|
||||
meta.exits.forEach((exit, i) => {
|
||||
const to = exit.to;
|
||||
if (to.includes('@sha256:') && idToFile[to]) {
|
||||
const targetContent = fs.readFileSync(idToFile[to], 'utf8');
|
||||
const targetMeta = matter(targetContent).data;
|
||||
const targetComputed = canonicalHash(targetMeta, matter(targetContent).content);
|
||||
const targetIdHash = to.split('@sha256:')[1];
|
||||
if (targetComputed !== targetIdHash) {
|
||||
throw new Error(`${ERRORS.G003}: Invalid hash in exits[${i}].to=${to} in ${file}`);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Migration if <2.3 (e.g., v2.2)
|
||||
if (specVersion === '2.2') {
|
||||
const transform = (await import('../migrations/v2_2/transform.js')).default;
|
||||
meta = transform(meta, body);
|
||||
const migHash = crypto.createHash('sha256').update(JSON.stringify(meta) + body).digest('hex');
|
||||
if (meta.migration_hash !== migHash) throw new Error(`${ERRORS.M001}: computed=${migHash}`);
|
||||
}
|
||||
|
||||
// Signature verify
|
||||
const rel = path.relative(path.join(process.cwd(), 'atlas'), file).replace(/[\\/]/g, '_');
|
||||
const sigFile = path.join(process.cwd(), 'signatures', rel + '.sig');
|
||||
try {
|
||||
await fs.access(sigFile);
|
||||
const sigContent = await fs.readFile(sigFile, 'utf8');
|
||||
const sig = Buffer.from(sigContent.split(':')[1].trim(), 'hex');
|
||||
const pubKey = Buffer.from('example_pubkey_hex', 'hex'); // Annex for real keys
|
||||
const message = Buffer.from(content);
|
||||
const valid = nacl.sign.detached.verify(message, sig, pubKey);
|
||||
if (!valid) throw new Error(`${ERRORS.Sig001}: ${sigFile}`);
|
||||
} catch {
|
||||
return; // Skip if missing
|
||||
}
|
||||
|
||||
// Append report to jsonl
|
||||
const report = { file, status: 'valid', timestamp: new Date().toISOString() };
|
||||
await fs.appendFile('foldlint.jsonl', JSON.stringify(report) + '\n');
|
||||
}
|
||||
|
||||
export { validate, detectCycles };
|
||||
125
tools/hash.js
Executable file
125
tools/hash.js
Executable file
|
|
@ -0,0 +1,125 @@
|
|||
import fs from 'fs/promises';
|
||||
import path from 'path';
|
||||
import matter from 'gray-matter';
|
||||
import yaml from 'js-yaml';
|
||||
import crypto from 'crypto';
|
||||
|
||||
// Canonical hash (exclude id from frontmatter)
|
||||
function canonicalHash(meta, body) {
|
||||
const metaClone = { ...meta };
|
||||
delete metaClone.id; // Exclude id to avoid circularity
|
||||
const frontYaml = yaml.dump(metaClone, { noRefs: true, lineWidth: -1 }).trim();
|
||||
const content = frontYaml + '\n' + body.trim();
|
||||
const lines = content.split('\n');
|
||||
const trimmedLines = lines.map(line => line.replace(/\s+$/, ''));
|
||||
const normalized = trimmedLines.join('\n');
|
||||
return { hash: crypto.createHash('sha256').update(normalized).digest('hex'), normalized };
|
||||
}
|
||||
|
||||
// Compute canonical hash for a file
|
||||
async function computeHash(file) {
|
||||
const content = await fs.readFile(file, 'utf8');
|
||||
const { data: meta, content: body } = matter(content);
|
||||
// Warn about block scalars in exits.to
|
||||
if (meta.exits) {
|
||||
meta.exits.forEach((exit, i) => {
|
||||
if (typeof exit.to === 'string' && exit.to.includes('\n')) {
|
||||
console.warn(`Block scalar in exits[${i}].to: ${file}; use inline string`);
|
||||
}
|
||||
});
|
||||
}
|
||||
const { hash, normalized } = canonicalHash(meta, body);
|
||||
const expected = meta.id ? meta.id.split('@sha256:')[1] : 'none';
|
||||
console.log(`File: ${file}\nComputed: ${hash}\nExpected: ${expected}\nMatch: ${hash === expected}\nNormalized Content:\n${normalized}\n`);
|
||||
console.log(`To fix, update id to: ${meta.id.split('@sha256:')[0]}@sha256:${hash}`);
|
||||
return { file, computed: hash, expected, match: hash === expected };
|
||||
}
|
||||
|
||||
// Fix hash in file
|
||||
async function fixHash(file) {
|
||||
const content = await fs.readFile(file, 'utf8');
|
||||
const { data: meta, content: body } = matter(content);
|
||||
// Normalize exits.to to inline strings
|
||||
if (meta.exits) {
|
||||
meta.exits = meta.exits.map(exit => ({
|
||||
...exit,
|
||||
to: exit.to.replace(/\n/g, '').trim()
|
||||
}));
|
||||
}
|
||||
const { hash } = canonicalHash(meta, body);
|
||||
const kindSlug = meta.id.split('@sha256:')[0]; // Preserve 'kind:slug'
|
||||
meta.id = kindSlug + '@sha256:' + hash;
|
||||
const newContent = `---\n${yaml.dump(meta, { noRefs: true, lineWidth: -1 }).trim()}\n---\n${body.trim()}`;
|
||||
await fs.writeFile(file, newContent);
|
||||
console.log(`Fixed hash in ${file}: ${hash}`);
|
||||
}
|
||||
|
||||
// Validate mode (check without fixing)
|
||||
async function validateHashes() {
|
||||
const atlasDir = path.join(process.cwd(), 'atlas');
|
||||
const files = [];
|
||||
async function collectFiles(dir) {
|
||||
const entries = await fs.readdir(dir, { withFileTypes: true });
|
||||
entries.sort((a, b) => a.name.localeCompare(b.name));
|
||||
for (const entry of entries) {
|
||||
const fullPath = path.join(dir, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
await collectFiles(fullPath);
|
||||
} else if (entry.name.endsWith('.md')) {
|
||||
files.push(fullPath);
|
||||
}
|
||||
}
|
||||
}
|
||||
await collectFiles(atlasDir);
|
||||
|
||||
const results = [];
|
||||
for (const file of files) {
|
||||
const result = await computeHash(file);
|
||||
results.push(result);
|
||||
}
|
||||
if (results.some(r => !r.match)) {
|
||||
console.error('Hash mismatches detected. Run `node tools/hash.js fix` to update IDs.');
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
// Main
|
||||
async function main() {
|
||||
const mode = process.argv[2] || 'check';
|
||||
if (mode === 'validate') {
|
||||
await validateHashes();
|
||||
return;
|
||||
}
|
||||
|
||||
const atlasDir = path.join(process.cwd(), 'atlas');
|
||||
const files = [];
|
||||
async function collectFiles(dir) {
|
||||
const entries = await fs.readdir(dir, { withFileTypes: true });
|
||||
entries.sort((a, b) => a.name.localeCompare(b.name));
|
||||
for (const entry of entries) {
|
||||
const fullPath = path.join(dir, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
await collectFiles(fullPath);
|
||||
} else if (entry.name.endsWith('.md')) {
|
||||
files.push(fullPath);
|
||||
}
|
||||
}
|
||||
}
|
||||
await collectFiles(atlasDir);
|
||||
|
||||
const results = [];
|
||||
for (const file of files) {
|
||||
if (mode === 'fix') {
|
||||
await fixHash(file);
|
||||
} else {
|
||||
const result = await computeHash(file);
|
||||
results.push(result);
|
||||
}
|
||||
}
|
||||
if (mode === 'check' && results.some(r => !r.match)) {
|
||||
console.error('Hash mismatches detected. Run `node tools/hash.js fix` to update IDs.');
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
main().catch(console.error);
|
||||
44
tools/scribe.js
Executable file
44
tools/scribe.js
Executable file
|
|
@ -0,0 +1,44 @@
|
|||
import fs from 'fs/promises';
|
||||
import path from 'path';
|
||||
import crypto from 'crypto';
|
||||
|
||||
// Daemon: Watch deltas.json, process atomically
|
||||
async function processDeltas() {
|
||||
const deltaFile = path.join(process.cwd(), 'scribe/deltas.json');
|
||||
const auditFile = path.join(process.cwd(), 'scribe/audit.jsonl');
|
||||
const chainFile = path.join(process.cwd(), 'scribe/chain');
|
||||
|
||||
// Guards
|
||||
await fs.mkdir(path.dirname(auditFile), { recursive: true });
|
||||
if (!(await fs.access(auditFile).catch(() => false))) await fs.writeFile(auditFile, '');
|
||||
if (!(await fs.access(chainFile).catch(() => false))) await fs.writeFile(chainFile, '');
|
||||
|
||||
if (await fs.access(deltaFile).catch(() => false)) {
|
||||
const deltas = JSON.parse(await fs.readFile(deltaFile, 'utf8'));
|
||||
const tmpAudit = auditFile + `.tmp.${Date.now()}`;
|
||||
|
||||
for (const delta of deltas) {
|
||||
// Validate clock (stub)
|
||||
if (!delta.clock) continue;
|
||||
|
||||
// Append to tmp
|
||||
await fs.appendFile(tmpAudit, JSON.stringify(delta) + '\n');
|
||||
|
||||
// Idempotent merge (hash check)
|
||||
const existingHash = crypto.createHash('sha256').update(JSON.stringify(delta)).digest('hex');
|
||||
const chainContent = await fs.readFile(chainFile, 'utf8');
|
||||
if (chainContent.includes(existingHash)) continue; // Idempotent skip
|
||||
|
||||
// Append to chain
|
||||
await fs.appendFile(chainFile, existingHash + '\n');
|
||||
}
|
||||
|
||||
// Promote atomic (after full batch)
|
||||
await fs.rename(tmpAudit, auditFile);
|
||||
|
||||
await fs.unlink(deltaFile);
|
||||
}
|
||||
}
|
||||
|
||||
setInterval(async () => await processDeltas(), 1000);
|
||||
console.log('Scribe daemon running...');
|
||||
Loading…
Add table
Add a link
Reference in a new issue