Lint rule probes

#!/usr/bin/env node
/**
 * Probe runner: asserts that every lint rule actually fires, and that every
 * package actually loads it.
 *
 * TWO PASSES, because there are two separate ways a green lint run can be
 * worthless and only the first is the one people think of.
 *
 *   1. SELECTORS  Every line in scripts/probes/*.probe.tsx marked FAIL produces
 *                 a lint error and every line marked OK produces none. Linted
 *                 with a config built from the rule module DIRECTLY, so this
 *                 pass asks one question only: do the selectors match what they
 *                 claim to match.
 *   2. WIRING     The rules are present in each package's RESOLVED config, AND
 *                 that config runs them at error severity. A working selector,
 *                 a config that never loads it, and a config that loads it
 *                 switched off all look identical from outside: all three print
 *                 nothing. This is also the likelier failure of the three,
 *                 because nobody edits a selector and people restructure
 *                 configs constantly.
 *
 * Why pass 1 no longer lints through the project config: doing that only ever
 * tested whichever config resolved at the probe file's own location. In a
 * monorepo with a config per package that is exactly one package, and every
 * other package's rules went untested while the run stayed green. Splitting the
 * two questions is the whole point: one pass owns the selectors, the other owns
 * the wiring, and neither can cover for the other.
 *
 * This is the check that makes the other checks trustworthy. Without it, a rule
 * whose selector matches nothing is indistinguishable from a rule with no
 * violations to find, and you will believe you have coverage that does not
 * exist. Run it in CI and in pre-commit whenever the lint config changes.
 *
 * WHAT IT DOES NOT PROVE. That the rules are the right rules, or that they
 * cover the code anyone actually writes. A probe only tests the lines somebody
 * thought to write in it. A rule with no probe line is invisible to this
 * script exactly as it is invisible to everything else.
 *
 * Usage:  node scripts/check-probes.mjs
 */
import { readFileSync, readdirSync, existsSync } from 'node:fs'
import { join, resolve } from 'node:path'
import { pathToFileURL } from 'node:url'

import { designRules, copyRules } from './eslint-rules.mjs'

// Loaded on first use so the verdict below can be imported and tested without
// ESLint present. The two passes are the part that needs a linter; deciding
// what a resolved config MEANS is arithmetic, and arithmetic should be testable.
const eslintClass = async () => (await import('eslint')).ESLint

// ---------------------------------------------------------------- CONFIG ---

const PROBE_DIR = 'scripts/probes'

/**
 * Every package whose ESLint config is meant to load the house rules, and one
 * REAL file inside it to resolve that config against.
 *
 * `name` is the package directory, used both as the label in the output and as
 * ESLint's cwd, because flat config is looked up from the cwd rather than from
 * the linted file. `probeFile` is ordinary source, not a probe fixture: the
 * question this pass asks is what the config does to normal code in that
 * package.
 *
 * The default is a single-package repo, so this works unchanged outside a
 * monorepo. In a monorepo add one entry per package that lints. A package
 * missing from this list is a package whose rules nobody is checking, which is
 * the exact failure the pass exists to catch.
 */
const PACKAGES = [{ name: '.', probeFile: 'src/app/layout.tsx' }]

/**
 * The severity a wired rule has to carry.
 *
 * 'error' means the rules gate. Set it to 'warn' only if you have deliberately
 * chosen a warning-only rollout, and know that lint then exits 0 on a
 * violation. 'off' is never acceptable: a rule listed in a config that turns it
 * off is the exact state this pass exists to catch, and it is invisible from
 * outside because a disabled rule and a clean file both print nothing.
 */
const REQUIRED_SEVERITY = 'error'

/** ESLint accepts 0/1/2 and 'off'/'warn'/'error'. Normalise to the words. */
const SEVERITIES = { 0: 'off', 1: 'warn', 2: 'error', off: 'off', warn: 'warn', error: 'error' }
export const severityOf = (entry) => {
  const raw = Array.isArray(entry) ? entry[0] : entry
  return SEVERITIES[raw] ?? 'off'
}

/**
 * What a resolved `no-restricted-syntax` entry means: which rules are absent,
 * and whether the ones that are there actually report anything.
 *
 * Pure, and exported, because this is where the pass used to be wrong and a
 * wrong answer here is invisible everywhere else. It read entry.slice(1) and
 * never looked at entry[0], so ['off', ...designRules, ...copyRules] — every
 * selector present, every one disabled — counted as fully wired. That config
 * reports nothing, lint exits 0, and the probe run congratulated it.
 *
 * `problem` is null when the rules are both present and enforcing.
 */
export function wiringVerdict(entry, want, required = REQUIRED_SEVERITY) {
  const got = new Set(
    (Array.isArray(entry) ? entry.slice(1) : []).map((o) => (typeof o === 'string' ? o : o?.selector))
  )
  const missing = want.filter((selector) => !got.has(selector))
  const severity = severityOf(entry)

  if (missing.length) return { missing, severity, problem: 'missing' }
  if (severity !== required) return { missing, severity, problem: severity === 'off' ? 'off' : 'weak' }
  return { missing, severity, problem: null }
}

// --------------------------------------------------------------- HELPERS ---

let failures = 0
let checked = 0

function report(heading, where, lines) {
  failures++
  console.log(`\n  ${heading}   ${where}`)
  for (const line of lines) console.log(`    ${line}`)
}

/** Lines annotated with a trailing // FAIL <tag> or // OK comment. */
function expectationsFor(file) {
  const lines = readFileSync(file, 'utf8').split('\n')
  const expect = new Map() // 1-indexed line -> 'fail' | 'ok'
  lines.forEach((line, i) => {
    // Ignore the explanatory prose at the top of the file: only annotate lines
    // that contain actual code before the marker.
    const m = line.match(/\/\/\s*(FAIL|OK)\b/)
    if (!m) return
    const code = line.slice(0, m.index).trim()
    if (!code || code.startsWith('*') || code.startsWith('//') || code.startsWith('/*')) return
    expect.set(i + 1, m[1] === 'FAIL' ? 'fail' : 'ok')
  })
  return expect
}

function sourceLine(file, line) {
  return readFileSync(file, 'utf8').split('\n')[line - 1].trim()
}

/**
 * The parser this repo already lints TSX with. No new dependency: a repo with a
 * .probe.tsx already has one of these two, and a repo with neither was never
 * linting TSX in the first place.
 */
async function tsxParser() {
  for (const id of ['typescript-eslint', '@typescript-eslint/parser']) {
    try {
      const mod = await import(id)
      const parser = mod.parser || mod.default || mod
      if (parser && (parser.parseForESLint || parser.parse)) return parser
    } catch {
      // Not installed under that name. Try the other one.
    }
  }
  return null
}

// ------------------------------------------------------- 1. the selectors ---

async function checkSelectors(probes, ESLint) {
  /*
   * Built from the rule module directly rather than from the project config.
   * Two reasons. It tests the selectors themselves, unmixed with whatever else
   * a project config turns on. And it tests ALL of them, not only the subset
   * that happens to be configured wherever the probe file sits.
   */
  const parser = await tsxParser()
  if (!parser) {
    console.error(`\n  No TypeScript parser found.`)
    console.error(`  Pass 1 builds its own config, so it needs the parser directly.`)
    console.error(`  Run: pnpm add -D typescript-eslint\n`)
    process.exit(1)
  }

  const eslint = new ESLint({
    // The project config is deliberately not consulted here. Probe files are
    // normally ignored by it, since they are nothing but deliberate violations.
    overrideConfigFile: true,
    overrideConfig: {
      files: ['**/*.tsx', '**/*.ts'],
      languageOptions: {
        parser,
        parserOptions: { ecmaFeatures: { jsx: true }, sourceType: 'module' },
      },
      // The probe file disables rules that this config does not turn on, which
      // is not a finding. It is a probe.
      linterOptions: { reportUnusedDisableDirectives: false },
      rules: { 'no-restricted-syntax': ['error', ...designRules, ...copyRules] },
    },
  })

  for (const file of probes) {
    const expect = expectationsFor(file)
    const [result] = await eslint.lintFiles([file])
    if (!result) {
      console.error(`  Could not lint ${file}. Check that it parses.`)
      process.exit(1)
    }

    const errorsByLine = new Map()
    for (const msg of result.messages) {
      if (msg.severity !== 2) continue
      if (!errorsByLine.has(msg.line)) errorsByLine.set(msg.line, [])
      errorsByLine.get(msg.line).push(msg.ruleId || 'parse-error')
    }

    for (const [line, kind] of expect) {
      checked++
      const got = errorsByLine.get(line) || []

      if (kind === 'fail' && got.length === 0) {
        report('RULE DID NOT FIRE', `${file}:${line}`, [
          sourceLine(file, line),
          `This line is meant to be a violation and the linter said nothing.`,
          `The selector probably matches nothing. Check for the numeric-literal`,
          `trap: esquery regex tests do not apply to numeric values, so match`,
          `on 'raw' rather than 'value'.`,
        ])
      }

      if (kind === 'ok' && got.length > 0) {
        report('FALSE POSITIVE', `${file}:${line}`, [
          sourceLine(file, line),
          `This line is legitimate and the linter flagged it: ${got.join(', ')}`,
          `A rule that fires on correct code gets disabled, and then it protects`,
          `nothing. Narrow the selector.`,
        ])
      }
    }
  }
}

// ----------------------------------------------------------- 2. the wiring ---

async function checkWiring(ESLint) {
  const want = [...designRules, ...copyRules].map((r) => r.selector)

  for (const { name, probeFile } of PACKAGES) {
    const target = join(name, probeFile)
    if (!existsSync(target)) {
      report('CANNOT CHECK WIRING', name, [
        `${target} does not exist, so nothing here resolved a config.`,
        `This is a failure rather than a skip on purpose: a wiring check that`,
        `quietly passes when its subject is missing is the thing being checked.`,
        `fix: point PACKAGES at a real source file in this package.`,
      ])
      continue
    }

    // cwd per package, because flat config is looked up from the cwd. Pointing
    // one ESLint instance at every package would resolve the root config every
    // time and report the same answer for all of them.
    const eslint = new ESLint({ cwd: resolve(name) })
    let config
    try {
      config = await eslint.calculateConfigForFile(resolve(target))
    } catch (e) {
      report('CONFIG WOULD NOT RESOLVE', name, [
        e.message.split('\n')[0],
        `fix: run eslint in ${name} directly and read the error in full.`,
      ])
      continue
    }

    /*
     * Compare the selector fields, not a JSON dump of the resolved entry. A
     * dump re-escapes every backslash, so a selector containing \s never
     * matches the source string it came from and reads as missing. Three real
     * rules were reported as unwired that way, which is the probe harness
     * failing in exactly the manner it exists to catch.
     */
    const { missing, severity, problem } = wiringVerdict(config.rules?.['no-restricted-syntax'], want)

    checked++
    if (problem === 'missing') {
      report('RULES NOT WIRED IN', name, [
        `${missing.length} of ${want.length} house rule(s) are absent from the`,
        `config that resolves for ${target}, so they run nowhere in this package.`,
        `The selectors themselves may be perfect. Nothing loads them.`,
        `First missing: ${String(missing[0]).slice(0, 66)}...`,
        `fix: spread designRules and copyRules into ${join(name, 'eslint.config.mjs')}.`,
      ])
      continue
    }

    // Present but not enforcing. Pass 1 cannot catch this either, because pass 1
    // builds its own config at 'error' on purpose.
    checked++
    if (problem) {
      const dead = problem === 'off'
      report(dead ? 'RULES LOADED BUT SWITCHED OFF' : 'RULES ONLY WARN', name, [
        `All ${want.length} house rule(s) are listed in the config that resolves`,
        `for ${target}, at severity '${severity}' rather than '${REQUIRED_SEVERITY}'.`,
        dead
          ? `Listed and disabled. Every selector matches and nothing is reported,`
          : `A warning does not fail lint, so this is a nudge and not a gate,`,
        dead
          ? `which reads from outside exactly like a clean codebase.`
          : `whatever the setup page says about blocking a commit.`,
        `fix: set the severity to '${REQUIRED_SEVERITY}' in ${join(name, 'eslint.config.mjs')}.`,
        `     A deliberate warning-only rollout sets REQUIRED_SEVERITY here instead,`,
        `     so the choice is written down rather than discovered.`,
      ])
    }
  }
}

// ------------------------------------------------------------------ MAIN ---

async function main() {
  if (!existsSync(PROBE_DIR)) {
    console.error(`No ${PROBE_DIR}. Create it and add at least one *.probe.tsx.`)
    process.exit(1)
  }

  const probes = readdirSync(PROBE_DIR)
    .filter((f) => f.endsWith('.probe.tsx') || f.endsWith('.probe.ts'))
    .map((f) => join(PROBE_DIR, f))
  if (!probes.length) {
    console.error(`No probe files in ${PROBE_DIR}.`)
    process.exit(1)
  }

  const ESLint = await eslintClass()
  await checkSelectors(probes, ESLint)
  await checkWiring(ESLint)

  console.log()
  if (failures) {
    console.log(`  probes: ${failures} of ${checked} expectation(s) not met.\n`)
    process.exit(1)
  }
  console.log(
    `  probes: ok. ${checked} expectation(s) met across ${probes.length} probe file(s)\n` +
      `  and ${PACKAGES.length} package(s). The rules fire, they are loaded, and\n` +
      `  every package runs them at '${REQUIRED_SEVERITY}'.\n`,
  )
}

// Run when invoked, importable when tested. The verdict above is a pure
// function over a resolved config and is worth testing on its own, which needs
// this file to be importable without running the linter.
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
  main().catch((e) => {
    console.error(e)
    process.exit(1)
  })
}