Experiment: Auto-generating an index of pages with my app (using a data loader)

As a means of listing out everything that is currently housed within this app, I decided to experiment with creating a data loader that would parse the page index and information of this very app. The data generated by that data loader is what fuels the home page.

Just to demonstrate the list in action:

const toc = FileAttachment("../data/toc.json").json();

display(html`<li><a href="../">Home</a>`)
for (let i = 0; i < toc.length; i++) {
    if (!toc[i].root) {
        const name = toc[i].section ? `${toc[i].section} ~ ${toc[i].title}` : toc[i].title,
              path = toc[i].file.replace(".md", "").replace(".markdown", "")
        display(html`<li><a href="..${path}">${name}</a>: ${toc[i].description}</li>`);
    }
}

Simplier data loader via walking over Markdown files in /docs

import os, sys, json
import frontmatter

root = "./docs/"
toc = []

for subdir, dirs, files in os.walk(root):
    for file in files:
        file_path = os.path.join(subdir, file)
        if os.path.isfile(file_path) and (file_path[-3:] == ".md" or file_path[-9:] == ".markdown"):
            with open(file_path) as file:
                file_frontmatter = frontmatter.loads(file.read())
                toc.append({
                    **{"file": file_path},
                    **{k: file_frontmatter[k] for k in file_frontmatter.keys()}
                })

json.dump(toc, sys.stdout)

More complicated data loader via parsing Observable Framework's config file

import os, sys, json
import esprima
import frontmatter

# parse observable config file for page structure
config_file = "observablehq.config.ts"
assert os.path.isfile(config_file)
config = esprima.parseModule(open(config_file).read())

def parseArrayExpression(ae):
    return([{p0.key.name: (p0.value.value if p0.value.type == "Literal" else parseArrayExpression(p0.value)) for p0 in p.properties} for p in ae.elements])
pages_config = parseArrayExpression([c for c in config.body[0].declaration.properties if c.key.name == "pages"][0].value)

# get create flat pages list with depth + frontmatter
root = "./docs/"
pages = []

def parsePage(p, depth = 0):
    file = p["path"] + (".md" if os.path.isfile(root + p["path"] + ".md") else ".markdown")
    assert os.path.isfile(root + file)
    file_frontmatter = frontmatter.loads(open(root + file).read())
    return { **p, **{ "depth": depth, "file": file }, **{k: file_frontmatter[k] for k in file_frontmatter.keys()}}

for p0 in pages_config:
    if p0.get('pages'):
        for p1 in p0.get('pages', []):
            pages.append(parsePage({**p1, **{"section": p0.get('name', "")}}, depth = 1))
    else:
        pages.append(parsePage(p0))

# export
json.dump(pages, sys.stdout)