From 330ce5193ec16cb511aa2de446171ac4408896ee Mon Sep 17 00:00:00 2001 From: manifest-updater pipeline seed Date: Tue, 23 Jun 2026 18:04:58 +0000 Subject: [PATCH] Seed/update .ci pipeline from manifest-updater v1.1.0 --- .ci/build-manifest.py | 218 ++++++++++++++++++++++++++++++++++++++++++ .ci/release.sh | 189 ++++++++++++++++++++++++++++++++++++ 2 files changed, 407 insertions(+) create mode 100644 .ci/build-manifest.py create mode 100644 .ci/release.sh diff --git a/.ci/build-manifest.py b/.ci/build-manifest.py new file mode 100644 index 0000000..9926476 --- /dev/null +++ b/.ci/build-manifest.py @@ -0,0 +1,218 @@ +#!/usr/bin/env python3 +"""Build a published update manifest from a plugin repo's base manifest + sections. + +Foundation model: a plugin repo curates a base `manifest.json` (rich, static +fields) + section content; the release pipeline computes the moving parts +(version, download_url, requires/tested from readme, icons, last_updated) and +hands them in as `--computed`. This merges the two and writes the channel manifest. + +SECTIONS (the "View details" modal body) resolve PER SECTION, gap-filling down a +cascade — Manifest-defined > ./sections/.md > auto-generate: + 1. base `manifest.json` `sections` — used verbatim, EXCEPT a section whose body + is a SINGLE line `@path.(md|html|txt)` existing under the repo root is + replaced by that file's converted contents (.md->HTML, .html->as-is, + .txt->escaped text). + 2. `/sections/.md` fills any section not set by (1). + 3. auto-generate (toggle via mu-release.yaml `sections.autogen`, default true): + description <- README.md (fallback: readme.txt `== Description ==`) + changelog <- CHANGELOG.md + ...for description/changelog only, if still unset. + +`sections.strip_title` (mu-release.yaml) removes a leading H1 title from PARSED +markdown only (never inline HTML / .html / .txt): + null | false -> off (default) + true -> strip from every parsed .md section + { "": true|false } -> per-section + +Computed fields always win for release-critical/readme-derived keys; the base +manifest wins for everything else it provides. + +Usage: + build-manifest.py --repo-root . --computed computed.json --out out.json [--config mu-release.yaml] +""" +import argparse +import html +import json +import os +import re +import sys + +try: + import markdown as _md +except ImportError: + sys.exit("build-manifest: python 'markdown' is required (pip install markdown)") + +# Always taken from --computed; never overridable by the curated base manifest. +PROTECTED = { + "version", "slug", "download_url", "last_updated", + "icons", "requires", "tested", "requires_php", +} +SECTIONS_DIR = "sections" +INCLUDE_RE = re.compile(r"^@(?P.+\.(?Pmd|html|txt))$") + + +def md_to_html(text: str) -> str: + return _md.markdown(text, extensions=["extra", "sane_lists", "nl2br"]) + + +def strip_leading_title(md: str) -> str: + """Drop a leading H1 (atx `# Title` or setext `Title\\n===`) from markdown.""" + lines = md.splitlines() + i = 0 + while i < len(lines) and lines[i].strip() == "": + i += 1 + if i < len(lines): + if re.match(r"^#\s+\S", lines[i]): # atx H1 + i += 1 + elif i + 1 < len(lines) and lines[i].strip() and re.match(r"^=+\s*$", lines[i + 1]): + i += 2 # setext H1 + return "\n".join(lines[i:]).lstrip("\n") + + +def should_strip(name: str, strip_cfg) -> bool: + if strip_cfg is True: + return True + if isinstance(strip_cfg, dict): + return bool(strip_cfg.get(name, False)) + return False # None / False / anything else + + +def md_section(text: str, name: str, strip_cfg) -> str: + if should_strip(name, strip_cfg): + text = strip_leading_title(text) + return md_to_html(text) + + +def convert(text: str, ext: str, name: str, strip_cfg) -> str: + if ext == "md": + return md_section(text, name, strip_cfg) + if ext == "html": + return text # already HTML — not parsed, never title-stripped. + paras = [p.strip() for p in re.split(r"\n\s*\n", text) if p.strip()] + return "".join("

" + html.escape(p).replace("\n", "
") + "

" for p in paras) + + +def resolve_in_repo(repo_root: str, rel: str): + root = os.path.realpath(repo_root) + full = os.path.realpath(os.path.join(root, rel)) + if full != root and not full.startswith(root + os.sep): + return None # path traversal — reject. + return full if os.path.isfile(full) else None + + +def read_text(repo_root: str, name: str): + p = os.path.join(repo_root, name) + return open(p, encoding="utf-8").read() if os.path.isfile(p) else None + + +def readme_txt_description(repo_root: str): + """Fallback description source: the `== Description ==` block of readme.txt.""" + text = read_text(repo_root, "readme.txt") + if not text: + return None + m = re.search(r"(?ms)^==\s*Description\s*==\s*\n(.*?)(?=^==\s|\Z)", text) + return (m.group(1).strip() or None) if m else None + + +def build_sections(base: dict, repo_root: str, autogen: bool, strip_cfg) -> dict: + resolved = {} + + # 1. manifest-defined (with @include resolution). + if isinstance(base.get("sections"), dict): + for name, body in base["sections"].items(): + name = str(name) + if isinstance(body, str): + one = body.strip() + m = INCLUDE_RE.match(one) if "\n" not in one else None + if m: + full = resolve_in_repo(repo_root, m.group("path")) + if full: + with open(full, encoding="utf-8") as fh: + resolved[name] = convert(fh.read(), m.group("ext"), name, strip_cfg) + continue + sys.stderr.write( + f"build-manifest: section '{name}' include " + f"'{m.group('path')}' not found — using literal\n" + ) + resolved[name] = body + else: + resolved[name] = str(body) + + # 2. ./sections/.md fills any section not already set. + sdir = os.path.join(repo_root, SECTIONS_DIR) + if os.path.isdir(sdir): + for fn in sorted(os.listdir(sdir)): + if fn.endswith(".md") and fn[:-3] not in resolved: + with open(os.path.join(sdir, fn), encoding="utf-8") as fh: + resolved[fn[:-3]] = md_section(fh.read(), fn[:-3], strip_cfg) + + # 3. auto-generate description (README.md, then readme.txt) + changelog (CHANGELOG.md). + if autogen: + if "description" not in resolved: + raw = read_text(repo_root, "README.md") + if not (raw and raw.strip()): + raw = readme_txt_description(repo_root) + if raw and raw.strip(): + resolved["description"] = md_section(raw, "description", strip_cfg) + if "changelog" not in resolved: + cl = read_text(repo_root, "CHANGELOG.md") + if cl and cl.strip(): + resolved["changelog"] = md_section(cl, "changelog", strip_cfg) + + return resolved + + +def read_config(path: str) -> dict: + if path and os.path.isfile(path): + import yaml + with open(path, encoding="utf-8") as fh: + return yaml.safe_load(fh) or {} + return {} + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("--repo-root", required=True) + ap.add_argument("--computed", required=True) + ap.add_argument("--out", required=True) + ap.add_argument("--config", default="", help="mu-release.yaml (for toggles)") + args = ap.parse_args() + + repo_root = args.repo_root + cfg = read_config(args.config) + sec_cfg = cfg.get("sections") or {} + autogen = bool(sec_cfg.get("autogen", True)) + strip_cfg = sec_cfg.get("strip_title") # None | bool | dict + + base = {} + base_path = os.path.join(repo_root, "manifest.json") + if os.path.isfile(base_path): + with open(base_path, encoding="utf-8") as fh: + base = json.load(fh) + if not isinstance(base, dict): + sys.exit("build-manifest: base manifest.json must be a JSON object") + + with open(args.computed, encoding="utf-8") as fh: + computed = json.load(fh) + + sections = build_sections(base, repo_root, autogen, strip_cfg) + + result = dict(computed) + for k, v in base.items(): + if k == "sections" or k in PROTECTED: + continue + result[k] = v + if sections: + result["sections"] = sections + + with open(args.out, "w", encoding="utf-8") as fh: + json.dump(result, fh, indent=2, ensure_ascii=False) + fh.write("\n") + sys.stderr.write( + f"build-manifest: wrote {args.out} " + f"(autogen={autogen}, strip_title={strip_cfg!r}; sections: {', '.join(sections) or 'none'})\n" + ) + + +if __name__ == "__main__": + main() diff --git a/.ci/release.sh b/.ci/release.sh new file mode 100644 index 0000000..d2873dc --- /dev/null +++ b/.ci/release.sh @@ -0,0 +1,189 @@ +#!/usr/bin/env bash +# Shared release pipeline for the ee-extensions plugin family. +# +# SOURCE OF TRUTH: this file lives in the Manifest Updater repo under .ci/ and is +# mirrored to foh-updates/.ci/ by MU's release workflow (the "seed" job). Each +# plugin's thin release.yml fetches it from the public channel and runs it. +# +# Run from a plugin checkout (CWD = the plugin repo at the tagged commit). +# Expects env: GITHUB_REPOSITORY, GITHUB_REF_NAME, GITHUB_SERVER_URL, GITHUB_TOKEN. +# Optional env: FOH_UPDATES_DEPLOY_KEY (+ FOH_UPDATES_SSH_PORT) to publish to the +# channel; WPORG_SVN_USERNAME/PASSWORD (+ WPORG_SLUG/WPORG_SVN_URL vars) for wp.org. +# +# Per-plugin knobs live in mu-release.yaml (all optional; convention defaults): +# channel: "ee-extensions/foh-updates" +# app_build: false # build app/ (Vite) + gate on assets/app staleness +# sections: { autogen: true } +set -euo pipefail + +SLUG="$(basename "${GITHUB_REPOSITORY}")" +VER="${GITHUB_REF_NAME#v}" +MAIN="${SLUG}.php" +HOST="${GITHUB_SERVER_URL}" +CFG="mu-release.yaml" + +# --- config helpers (yaml via python; pure-convention defaults if absent) ----- +cfg() { python3 - "$1" "$2" <<'PY' 2>/dev/null || true +import sys, os +key, default = sys.argv[1], sys.argv[2] +val = default +if os.path.isfile("mu-release.yaml"): + import yaml + data = yaml.safe_load(open("mu-release.yaml")) or {} + cur = data + for part in key.split("."): + cur = cur.get(part) if isinstance(cur, dict) else None + if cur is None: + cur = default; break + val = cur +print("true" if val is True else "false" if val is False else val) +PY +} +CHANNEL="$(cfg channel 'ee-extensions/foh-updates')" +APP_BUILD="$(cfg app_build 'false')" + +log() { printf '\n\033[1m▸ %s\033[0m\n' "$*"; } + +# --- 1. validate header/const/changelog/readme all match the tag -------------- +log "Validate ${MAIN} Version, CHANGELOG, readme.txt vs tag v${VER}" +[ -f "${MAIN}" ] || { echo "::error::main plugin file ${MAIN} not found"; exit 1; } +HVER="$(grep -iE '^[[:space:]]*\*?[[:space:]]*Version:' "${MAIN}" | head -1 | sed -E 's/.*[Vv]ersion:[[:space:]]*//' | tr -d '[:space:]\r')" +[ "${HVER}" = "${VER}" ] || { echo "::error::header Version (${HVER}) != tag (${VER})"; exit 1; } +if grep -qE "_VERSION'" "${MAIN}"; then + grep -qF "_VERSION', '${VER}'" "${MAIN}" || { echo "::error::*_VERSION const != ${VER}"; exit 1; } +fi +XY="$(printf '%s' "${VER}" | cut -d. -f1,2)" +if [ -f CHANGELOG.md ]; then + grep -qF "[${VER}]" CHANGELOG.md || { echo "::error::CHANGELOG.md has no [${VER}] entry"; exit 1; } +else + echo "::warning::no CHANGELOG.md — changelog gate skipped" +fi +[ -f readme.txt ] || { echo "::error::no readme.txt — required for packaging"; exit 1; } +RSTABLE="$(grep -iE '^Stable tag:' readme.txt | head -1 | sed -E 's/.*:[[:space:]]*//' | tr -d '[:space:]\r')" +[ "${RSTABLE}" = "${VER}" ] || { echo "::error::readme.txt Stable tag (${RSTABLE}) != tag (${VER})"; exit 1; } +grep -qiE '^License:' readme.txt || { echo "::error::readme.txt missing 'License:' header"; exit 1; } +grep -qE '^== Changelog ==' readme.txt || { echo "::error::readme.txt missing '== Changelog ==' section"; exit 1; } +echo "OK: version/changelog/readme all agree on ${VER}." + +# --- 2. optional Vite app build + staleness gate ------------------------------ +if [ "${APP_BUILD}" = "true" ] && [ -f app/package.json ]; then + log "Build app/ (Vite) + gate on assets/app staleness" + ( cd app && npm ci && npm run build ) + if ! git diff --quiet -- assets/app; then + echo "::error::assets/app is stale — rebuild, commit, re-tag." + git --no-pager diff --stat -- assets/app; exit 1 + fi +else + echo "No app build (app_build=${APP_BUILD})." +fi + +# --- 3. package-hygiene scan: nothing stray may ship -------------------------- +log "Package hygiene scan" +rm -rf _scan && mkdir _scan +git archive "${GITHUB_REF_NAME}" | tar -x -C _scan +BAD="$(cd _scan && find . \( -iname '*claude*' -o -name '.git' -o -name '.gitea' \ + -o -name '.github' -o -name 'node_modules' -o -name '.DS_Store' -o -name 'Thumbs.db' \ + -o -name '.build-notes-*' -o -name '*.scratch' -o -name 'dev' -o -name 'docs' \) -print)" +[ -z "${BAD}" ] || { echo "::error::disallowed files in package:"; echo "${BAD}"; exit 1; } +if grep -rIilE 'claude|sticky[-_]dvr' _scan; then + echo "::error::forbidden token in a shipped file"; exit 1 +fi +rm -rf _scan +echo "Package hygiene OK." + +# --- 4. build the zip (git archive honours .gitattributes export-ignore) ------ +log "Package ${SLUG}-${VER}.zip" +ZIP="${SLUG}-${VER}.zip" +git archive --format=zip --prefix="${SLUG}/" -o "${ZIP}" "${GITHUB_REF_NAME}" +ls -l "${ZIP}" + +# --- 5. cut the Gitea release + attach the zip -------------------------------- +log "Publish Gitea release" +API="${HOST}/api/v1/repos/${GITHUB_REPOSITORY}" +AUTH="Authorization: token ${GITHUB_TOKEN}" +# Idempotent: reuse the release if this tag was already released (re-run/re-tag). +REL="$(curl -fsS -H "${AUTH}" "${API}/releases/tags/${GITHUB_REF_NAME}" 2>/dev/null || true)" +REL_ID="$(printf '%s' "${REL}" | grep -o '"id":[0-9]*' | head -1 | grep -o '[0-9]*')" +if [ -z "${REL_ID}" ]; then + BODY="$(python3 -c 'import json,sys; print(json.dumps({"tag_name":sys.argv[1],"name":sys.argv[2],"body":sys.argv[3]}))' \ + "${GITHUB_REF_NAME}" "${SLUG} ${VER}" "Automated build of ${GITHUB_REF_NAME}.")" + REL="$(curl -fsS -X POST "${API}/releases" -H "${AUTH}" -H "Content-Type: application/json" --data-raw "${BODY}")" + REL_ID="$(printf '%s' "${REL}" | grep -o '"id":[0-9]*' | head -1 | grep -o '[0-9]*')" +fi +[ -n "${REL_ID}" ] || { echo "::error::could not create/get the release"; echo "${REL}"; exit 1; } +# Replace a same-named asset on a re-run rather than 409-ing. +OLD="$(curl -fsS -H "${AUTH}" "${API}/releases/${REL_ID}/assets" 2>/dev/null \ + | python3 -c "import json,sys;[print(a['id']) for a in (json.load(sys.stdin) or []) if a.get('name')==sys.argv[1]]" "${ZIP}" 2>/dev/null || true)" +[ -z "${OLD}" ] || curl -fsS -X DELETE -H "${AUTH}" "${API}/releases/${REL_ID}/assets/${OLD}" >/dev/null 2>&1 || true +curl -fsS -X POST "${API}/releases/${REL_ID}/assets?name=${ZIP}" -H "${AUTH}" \ + -F "attachment=@${ZIP};type=application/zip" >/dev/null +echo "Released ${GITHUB_REF_NAME}." + +# --- 6. publish to the self-hosted update channel (skipped without deploy key) - +if [ -z "${FOH_UPDATES_DEPLOY_KEY:-}" ]; then + echo "FOH_UPDATES_DEPLOY_KEY not set — skipping channel publish."; exit 0 +fi +log "Publish to channel ${CHANNEL}" +SSH_HOST="${HOST#https://}"; SSH_HOST="${SSH_HOST#http://}" +SSH_PORT="${FOH_UPDATES_SSH_PORT:-30009}" +DL="${HOST}/${CHANNEL}/raw/branch/main/${SLUG}/${ZIP}" +RAW="${HOST}/${CHANNEL}/raw/branch/main/${SLUG}" + +install -d -m 700 "${HOME}/.ssh" +KEY="${HOME}/.ssh/foh_updates_deploy"; KH="${HOME}/.ssh/known_hosts" +printf '%s\n' "${FOH_UPDATES_DEPLOY_KEY}" > "${KEY}"; chmod 600 "${KEY}" +ssh-keyscan -p "${SSH_PORT}" "${SSH_HOST}" >> "${KH}" 2>/dev/null +export GIT_SSH_COMMAND="ssh -i ${KEY} -o IdentitiesOnly=yes -o UserKnownHostsFile=${KH}" + +git clone --depth 1 "ssh://git@${SSH_HOST}:${SSH_PORT}/${CHANNEL}.git" _upd + +# computed (release-derived) fields → build-manifest merges them with manifest.json + sections. +NAME="$(grep -iE '^[[:space:]]*\*?[[:space:]]*Plugin Name:' "${MAIN}" | head -1 | sed -E 's/.*Plugin Name:[[:space:]]*//' | tr -d '\r')" +REQ="$(grep -iE '^Requires at least:' readme.txt | head -1 | sed -E 's/.*:[[:space:]]*//' | tr -d '[:space:]\r' || true)" +TESTED="$(grep -iE '^Tested up to:' readme.txt | head -1 | sed -E 's/.*:[[:space:]]*//' | tr -d '[:space:]\r' || true)" +RPHP="$(grep -iE '^Requires PHP:' readme.txt | head -1 | sed -E 's/.*:[[:space:]]*//' | tr -d '[:space:]\r' || true)" +NOW="$(date -u +%Y-%m-%d' '%H:%M:%S)" + +ICONS='{}' +if [ -f icons/icon-256x256.png ]; then + ICONS="$(printf '{"1x":"%s/icon-128x128.png","2x":"%s/icon-256x256.png"}' "${RAW}" "${RAW}")" +fi +python3 - "$NAME" "$SLUG" "$VER" "$DL" "$HOST/${CHANNEL%/*}/$SLUG" "$REQ" "$TESTED" "$RPHP" "$NOW" "$ICONS" > computed.json <<'PY' +import json, sys +k = ["name","slug","version","download_url","homepage","requires","tested","requires_php","last_updated"] +d = dict(zip(k, sys.argv[1:10])) +d["icons"] = json.loads(sys.argv[10]) +json.dump(d, sys.stdout) +PY + +# ensure python3 + pip + the builder's deps (markdown, pyyaml) are present. +command -v python3 >/dev/null 2>&1 || { sudo apt-get update -qq && sudo apt-get install -y -qq python3; } +python3 -m pip --version >/dev/null 2>&1 || { sudo apt-get update -qq && sudo apt-get install -y -qq python3-pip; } +python3 -m pip install --quiet markdown pyyaml 2>/dev/null \ + || python3 -m pip install --quiet --break-system-packages markdown pyyaml +# Prefer the tag-locked local builder (MU ships .ci/); consumer plugins fall back +# to the channel copy seeded by MU's release. (Keep FOH_UPDATES_DEPLOY_KEY tightly held.) +BUILDER="_upd/.ci/build-manifest.py" +[ -f .ci/build-manifest.py ] && BUILDER=".ci/build-manifest.py" +python3 "${BUILDER}" --repo-root . --computed computed.json --config "${CFG}" --out "_upd/${SLUG}.json" + +mkdir -p "_upd/${SLUG}" +rm -f "_upd/${SLUG}"/*.zip +cp "${ZIP}" "_upd/${SLUG}/" +[ -d icons ] && cp icons/icon-128x128.png icons/icon-256x256.png "_upd/${SLUG}/" 2>/dev/null || true + +( cd _upd + git config user.email "ci@${SSH_HOST}" + git config user.name "ee-extensions release bot" + git add -A + if git diff --cached --quiet; then echo "channel already current — nothing to publish."; exit 0; fi + git commit -m "release ${SLUG} ${VER}" + pushed=0 + for i in 1 2 3 4 5; do + if git push; then pushed=1; break; fi + echo "channel push raced; backing off ${i}/5" + sleep $(( (RANDOM % 20) + 1 )) # large random jitter to de-sync concurrent releases + git pull --rebase --no-edit + done + [ "${pushed}" = 1 ] || { echo "::error::channel push failed after 5 retries"; exit 1; } + echo "Published ${SLUG} ${VER} to ${CHANNEL}: ${DL}" )