Seed/update .ci pipeline from manifest-updater v1.1.0

This commit is contained in:
manifest-updater pipeline seed
2026-06-23 18:04:58 +00:00
parent 145f96cacf
commit 330ce5193e
2 changed files with 407 additions and 0 deletions
+218
View File
@@ -0,0 +1,218 @@
#!/usr/bin/env python3
"""Build a published update manifest from a plugin repo's base manifest + sections.
Foundation model: a plugin repo curates a base `manifest.json` (rich, static
fields) + section content; the release pipeline computes the moving parts
(version, download_url, requires/tested from readme, icons, last_updated) and
hands them in as `--computed`. This merges the two and writes the channel manifest.
SECTIONS (the "View details" modal body) resolve PER SECTION, gap-filling down a
cascade — Manifest-defined > ./sections/<name>.md > auto-generate:
1. base `manifest.json` `sections` — used verbatim, EXCEPT a section whose body
is a SINGLE line `@path.(md|html|txt)` existing under the repo root is
replaced by that file's converted contents (.md->HTML, .html->as-is,
.txt->escaped text).
2. `<repo-root>/sections/<name>.md` fills any section not set by (1).
3. auto-generate (toggle via mu-release.yaml `sections.autogen`, default true):
description <- README.md (fallback: readme.txt `== Description ==`)
changelog <- CHANGELOG.md
...for description/changelog only, if still unset.
`sections.strip_title` (mu-release.yaml) removes a leading H1 title from PARSED
markdown only (never inline HTML / .html / .txt):
null | false -> off (default)
true -> strip from every parsed .md section
{ "<name>": true|false } -> per-section
Computed fields always win for release-critical/readme-derived keys; the base
manifest wins for everything else it provides.
Usage:
build-manifest.py --repo-root . --computed computed.json --out out.json [--config mu-release.yaml]
"""
import argparse
import html
import json
import os
import re
import sys
try:
import markdown as _md
except ImportError:
sys.exit("build-manifest: python 'markdown' is required (pip install markdown)")
# Always taken from --computed; never overridable by the curated base manifest.
PROTECTED = {
"version", "slug", "download_url", "last_updated",
"icons", "requires", "tested", "requires_php",
}
SECTIONS_DIR = "sections"
INCLUDE_RE = re.compile(r"^@(?P<path>.+\.(?P<ext>md|html|txt))$")
def md_to_html(text: str) -> str:
return _md.markdown(text, extensions=["extra", "sane_lists", "nl2br"])
def strip_leading_title(md: str) -> str:
"""Drop a leading H1 (atx `# Title` or setext `Title\\n===`) from markdown."""
lines = md.splitlines()
i = 0
while i < len(lines) and lines[i].strip() == "":
i += 1
if i < len(lines):
if re.match(r"^#\s+\S", lines[i]): # atx H1
i += 1
elif i + 1 < len(lines) and lines[i].strip() and re.match(r"^=+\s*$", lines[i + 1]):
i += 2 # setext H1
return "\n".join(lines[i:]).lstrip("\n")
def should_strip(name: str, strip_cfg) -> bool:
if strip_cfg is True:
return True
if isinstance(strip_cfg, dict):
return bool(strip_cfg.get(name, False))
return False # None / False / anything else
def md_section(text: str, name: str, strip_cfg) -> str:
if should_strip(name, strip_cfg):
text = strip_leading_title(text)
return md_to_html(text)
def convert(text: str, ext: str, name: str, strip_cfg) -> str:
if ext == "md":
return md_section(text, name, strip_cfg)
if ext == "html":
return text # already HTML — not parsed, never title-stripped.
paras = [p.strip() for p in re.split(r"\n\s*\n", text) if p.strip()]
return "".join("<p>" + html.escape(p).replace("\n", "<br>") + "</p>" for p in paras)
def resolve_in_repo(repo_root: str, rel: str):
root = os.path.realpath(repo_root)
full = os.path.realpath(os.path.join(root, rel))
if full != root and not full.startswith(root + os.sep):
return None # path traversal — reject.
return full if os.path.isfile(full) else None
def read_text(repo_root: str, name: str):
p = os.path.join(repo_root, name)
return open(p, encoding="utf-8").read() if os.path.isfile(p) else None
def readme_txt_description(repo_root: str):
"""Fallback description source: the `== Description ==` block of readme.txt."""
text = read_text(repo_root, "readme.txt")
if not text:
return None
m = re.search(r"(?ms)^==\s*Description\s*==\s*\n(.*?)(?=^==\s|\Z)", text)
return (m.group(1).strip() or None) if m else None
def build_sections(base: dict, repo_root: str, autogen: bool, strip_cfg) -> dict:
resolved = {}
# 1. manifest-defined (with @include resolution).
if isinstance(base.get("sections"), dict):
for name, body in base["sections"].items():
name = str(name)
if isinstance(body, str):
one = body.strip()
m = INCLUDE_RE.match(one) if "\n" not in one else None
if m:
full = resolve_in_repo(repo_root, m.group("path"))
if full:
with open(full, encoding="utf-8") as fh:
resolved[name] = convert(fh.read(), m.group("ext"), name, strip_cfg)
continue
sys.stderr.write(
f"build-manifest: section '{name}' include "
f"'{m.group('path')}' not found — using literal\n"
)
resolved[name] = body
else:
resolved[name] = str(body)
# 2. ./sections/<name>.md fills any section not already set.
sdir = os.path.join(repo_root, SECTIONS_DIR)
if os.path.isdir(sdir):
for fn in sorted(os.listdir(sdir)):
if fn.endswith(".md") and fn[:-3] not in resolved:
with open(os.path.join(sdir, fn), encoding="utf-8") as fh:
resolved[fn[:-3]] = md_section(fh.read(), fn[:-3], strip_cfg)
# 3. auto-generate description (README.md, then readme.txt) + changelog (CHANGELOG.md).
if autogen:
if "description" not in resolved:
raw = read_text(repo_root, "README.md")
if not (raw and raw.strip()):
raw = readme_txt_description(repo_root)
if raw and raw.strip():
resolved["description"] = md_section(raw, "description", strip_cfg)
if "changelog" not in resolved:
cl = read_text(repo_root, "CHANGELOG.md")
if cl and cl.strip():
resolved["changelog"] = md_section(cl, "changelog", strip_cfg)
return resolved
def read_config(path: str) -> dict:
if path and os.path.isfile(path):
import yaml
with open(path, encoding="utf-8") as fh:
return yaml.safe_load(fh) or {}
return {}
def main() -> None:
ap = argparse.ArgumentParser()
ap.add_argument("--repo-root", required=True)
ap.add_argument("--computed", required=True)
ap.add_argument("--out", required=True)
ap.add_argument("--config", default="", help="mu-release.yaml (for toggles)")
args = ap.parse_args()
repo_root = args.repo_root
cfg = read_config(args.config)
sec_cfg = cfg.get("sections") or {}
autogen = bool(sec_cfg.get("autogen", True))
strip_cfg = sec_cfg.get("strip_title") # None | bool | dict
base = {}
base_path = os.path.join(repo_root, "manifest.json")
if os.path.isfile(base_path):
with open(base_path, encoding="utf-8") as fh:
base = json.load(fh)
if not isinstance(base, dict):
sys.exit("build-manifest: base manifest.json must be a JSON object")
with open(args.computed, encoding="utf-8") as fh:
computed = json.load(fh)
sections = build_sections(base, repo_root, autogen, strip_cfg)
result = dict(computed)
for k, v in base.items():
if k == "sections" or k in PROTECTED:
continue
result[k] = v
if sections:
result["sections"] = sections
with open(args.out, "w", encoding="utf-8") as fh:
json.dump(result, fh, indent=2, ensure_ascii=False)
fh.write("\n")
sys.stderr.write(
f"build-manifest: wrote {args.out} "
f"(autogen={autogen}, strip_title={strip_cfg!r}; sections: {', '.join(sections) or 'none'})\n"
)
if __name__ == "__main__":
main()
+189
View File
@@ -0,0 +1,189 @@
#!/usr/bin/env bash
# Shared release pipeline for the ee-extensions plugin family.
#
# SOURCE OF TRUTH: this file lives in the Manifest Updater repo under .ci/ and is
# mirrored to foh-updates/.ci/ by MU's release workflow (the "seed" job). Each
# plugin's thin release.yml fetches it from the public channel and runs it.
#
# Run from a plugin checkout (CWD = the plugin repo at the tagged commit).
# Expects env: GITHUB_REPOSITORY, GITHUB_REF_NAME, GITHUB_SERVER_URL, GITHUB_TOKEN.
# Optional env: FOH_UPDATES_DEPLOY_KEY (+ FOH_UPDATES_SSH_PORT) to publish to the
# channel; WPORG_SVN_USERNAME/PASSWORD (+ WPORG_SLUG/WPORG_SVN_URL vars) for wp.org.
#
# Per-plugin knobs live in mu-release.yaml (all optional; convention defaults):
# channel: "ee-extensions/foh-updates"
# app_build: false # build app/ (Vite) + gate on assets/app staleness
# sections: { autogen: true }
set -euo pipefail
SLUG="$(basename "${GITHUB_REPOSITORY}")"
VER="${GITHUB_REF_NAME#v}"
MAIN="${SLUG}.php"
HOST="${GITHUB_SERVER_URL}"
CFG="mu-release.yaml"
# --- config helpers (yaml via python; pure-convention defaults if absent) -----
cfg() { python3 - "$1" "$2" <<'PY' 2>/dev/null || true
import sys, os
key, default = sys.argv[1], sys.argv[2]
val = default
if os.path.isfile("mu-release.yaml"):
import yaml
data = yaml.safe_load(open("mu-release.yaml")) or {}
cur = data
for part in key.split("."):
cur = cur.get(part) if isinstance(cur, dict) else None
if cur is None:
cur = default; break
val = cur
print("true" if val is True else "false" if val is False else val)
PY
}
CHANNEL="$(cfg channel 'ee-extensions/foh-updates')"
APP_BUILD="$(cfg app_build 'false')"
log() { printf '\n\033[1m▸ %s\033[0m\n' "$*"; }
# --- 1. validate header/const/changelog/readme all match the tag --------------
log "Validate ${MAIN} Version, CHANGELOG, readme.txt vs tag v${VER}"
[ -f "${MAIN}" ] || { echo "::error::main plugin file ${MAIN} not found"; exit 1; }
HVER="$(grep -iE '^[[:space:]]*\*?[[:space:]]*Version:' "${MAIN}" | head -1 | sed -E 's/.*[Vv]ersion:[[:space:]]*//' | tr -d '[:space:]\r')"
[ "${HVER}" = "${VER}" ] || { echo "::error::header Version (${HVER}) != tag (${VER})"; exit 1; }
if grep -qE "_VERSION'" "${MAIN}"; then
grep -qF "_VERSION', '${VER}'" "${MAIN}" || { echo "::error::*_VERSION const != ${VER}"; exit 1; }
fi
XY="$(printf '%s' "${VER}" | cut -d. -f1,2)"
if [ -f CHANGELOG.md ]; then
grep -qF "[${VER}]" CHANGELOG.md || { echo "::error::CHANGELOG.md has no [${VER}] entry"; exit 1; }
else
echo "::warning::no CHANGELOG.md — changelog gate skipped"
fi
[ -f readme.txt ] || { echo "::error::no readme.txt — required for packaging"; exit 1; }
RSTABLE="$(grep -iE '^Stable tag:' readme.txt | head -1 | sed -E 's/.*:[[:space:]]*//' | tr -d '[:space:]\r')"
[ "${RSTABLE}" = "${VER}" ] || { echo "::error::readme.txt Stable tag (${RSTABLE}) != tag (${VER})"; exit 1; }
grep -qiE '^License:' readme.txt || { echo "::error::readme.txt missing 'License:' header"; exit 1; }
grep -qE '^== Changelog ==' readme.txt || { echo "::error::readme.txt missing '== Changelog ==' section"; exit 1; }
echo "OK: version/changelog/readme all agree on ${VER}."
# --- 2. optional Vite app build + staleness gate ------------------------------
if [ "${APP_BUILD}" = "true" ] && [ -f app/package.json ]; then
log "Build app/ (Vite) + gate on assets/app staleness"
( cd app && npm ci && npm run build )
if ! git diff --quiet -- assets/app; then
echo "::error::assets/app is stale — rebuild, commit, re-tag."
git --no-pager diff --stat -- assets/app; exit 1
fi
else
echo "No app build (app_build=${APP_BUILD})."
fi
# --- 3. package-hygiene scan: nothing stray may ship --------------------------
log "Package hygiene scan"
rm -rf _scan && mkdir _scan
git archive "${GITHUB_REF_NAME}" | tar -x -C _scan
BAD="$(cd _scan && find . \( -iname '*claude*' -o -name '.git' -o -name '.gitea' \
-o -name '.github' -o -name 'node_modules' -o -name '.DS_Store' -o -name 'Thumbs.db' \
-o -name '.build-notes-*' -o -name '*.scratch' -o -name 'dev' -o -name 'docs' \) -print)"
[ -z "${BAD}" ] || { echo "::error::disallowed files in package:"; echo "${BAD}"; exit 1; }
if grep -rIilE 'claude|sticky[-_]dvr' _scan; then
echo "::error::forbidden token in a shipped file"; exit 1
fi
rm -rf _scan
echo "Package hygiene OK."
# --- 4. build the zip (git archive honours .gitattributes export-ignore) ------
log "Package ${SLUG}-${VER}.zip"
ZIP="${SLUG}-${VER}.zip"
git archive --format=zip --prefix="${SLUG}/" -o "${ZIP}" "${GITHUB_REF_NAME}"
ls -l "${ZIP}"
# --- 5. cut the Gitea release + attach the zip --------------------------------
log "Publish Gitea release"
API="${HOST}/api/v1/repos/${GITHUB_REPOSITORY}"
AUTH="Authorization: token ${GITHUB_TOKEN}"
# Idempotent: reuse the release if this tag was already released (re-run/re-tag).
REL="$(curl -fsS -H "${AUTH}" "${API}/releases/tags/${GITHUB_REF_NAME}" 2>/dev/null || true)"
REL_ID="$(printf '%s' "${REL}" | grep -o '"id":[0-9]*' | head -1 | grep -o '[0-9]*')"
if [ -z "${REL_ID}" ]; then
BODY="$(python3 -c 'import json,sys; print(json.dumps({"tag_name":sys.argv[1],"name":sys.argv[2],"body":sys.argv[3]}))' \
"${GITHUB_REF_NAME}" "${SLUG} ${VER}" "Automated build of ${GITHUB_REF_NAME}.")"
REL="$(curl -fsS -X POST "${API}/releases" -H "${AUTH}" -H "Content-Type: application/json" --data-raw "${BODY}")"
REL_ID="$(printf '%s' "${REL}" | grep -o '"id":[0-9]*' | head -1 | grep -o '[0-9]*')"
fi
[ -n "${REL_ID}" ] || { echo "::error::could not create/get the release"; echo "${REL}"; exit 1; }
# Replace a same-named asset on a re-run rather than 409-ing.
OLD="$(curl -fsS -H "${AUTH}" "${API}/releases/${REL_ID}/assets" 2>/dev/null \
| python3 -c "import json,sys;[print(a['id']) for a in (json.load(sys.stdin) or []) if a.get('name')==sys.argv[1]]" "${ZIP}" 2>/dev/null || true)"
[ -z "${OLD}" ] || curl -fsS -X DELETE -H "${AUTH}" "${API}/releases/${REL_ID}/assets/${OLD}" >/dev/null 2>&1 || true
curl -fsS -X POST "${API}/releases/${REL_ID}/assets?name=${ZIP}" -H "${AUTH}" \
-F "attachment=@${ZIP};type=application/zip" >/dev/null
echo "Released ${GITHUB_REF_NAME}."
# --- 6. publish to the self-hosted update channel (skipped without deploy key) -
if [ -z "${FOH_UPDATES_DEPLOY_KEY:-}" ]; then
echo "FOH_UPDATES_DEPLOY_KEY not set — skipping channel publish."; exit 0
fi
log "Publish to channel ${CHANNEL}"
SSH_HOST="${HOST#https://}"; SSH_HOST="${SSH_HOST#http://}"
SSH_PORT="${FOH_UPDATES_SSH_PORT:-30009}"
DL="${HOST}/${CHANNEL}/raw/branch/main/${SLUG}/${ZIP}"
RAW="${HOST}/${CHANNEL}/raw/branch/main/${SLUG}"
install -d -m 700 "${HOME}/.ssh"
KEY="${HOME}/.ssh/foh_updates_deploy"; KH="${HOME}/.ssh/known_hosts"
printf '%s\n' "${FOH_UPDATES_DEPLOY_KEY}" > "${KEY}"; chmod 600 "${KEY}"
ssh-keyscan -p "${SSH_PORT}" "${SSH_HOST}" >> "${KH}" 2>/dev/null
export GIT_SSH_COMMAND="ssh -i ${KEY} -o IdentitiesOnly=yes -o UserKnownHostsFile=${KH}"
git clone --depth 1 "ssh://git@${SSH_HOST}:${SSH_PORT}/${CHANNEL}.git" _upd
# computed (release-derived) fields → build-manifest merges them with manifest.json + sections.
NAME="$(grep -iE '^[[:space:]]*\*?[[:space:]]*Plugin Name:' "${MAIN}" | head -1 | sed -E 's/.*Plugin Name:[[:space:]]*//' | tr -d '\r')"
REQ="$(grep -iE '^Requires at least:' readme.txt | head -1 | sed -E 's/.*:[[:space:]]*//' | tr -d '[:space:]\r' || true)"
TESTED="$(grep -iE '^Tested up to:' readme.txt | head -1 | sed -E 's/.*:[[:space:]]*//' | tr -d '[:space:]\r' || true)"
RPHP="$(grep -iE '^Requires PHP:' readme.txt | head -1 | sed -E 's/.*:[[:space:]]*//' | tr -d '[:space:]\r' || true)"
NOW="$(date -u +%Y-%m-%d' '%H:%M:%S)"
ICONS='{}'
if [ -f icons/icon-256x256.png ]; then
ICONS="$(printf '{"1x":"%s/icon-128x128.png","2x":"%s/icon-256x256.png"}' "${RAW}" "${RAW}")"
fi
python3 - "$NAME" "$SLUG" "$VER" "$DL" "$HOST/${CHANNEL%/*}/$SLUG" "$REQ" "$TESTED" "$RPHP" "$NOW" "$ICONS" > computed.json <<'PY'
import json, sys
k = ["name","slug","version","download_url","homepage","requires","tested","requires_php","last_updated"]
d = dict(zip(k, sys.argv[1:10]))
d["icons"] = json.loads(sys.argv[10])
json.dump(d, sys.stdout)
PY
# ensure python3 + pip + the builder's deps (markdown, pyyaml) are present.
command -v python3 >/dev/null 2>&1 || { sudo apt-get update -qq && sudo apt-get install -y -qq python3; }
python3 -m pip --version >/dev/null 2>&1 || { sudo apt-get update -qq && sudo apt-get install -y -qq python3-pip; }
python3 -m pip install --quiet markdown pyyaml 2>/dev/null \
|| python3 -m pip install --quiet --break-system-packages markdown pyyaml
# Prefer the tag-locked local builder (MU ships .ci/); consumer plugins fall back
# to the channel copy seeded by MU's release. (Keep FOH_UPDATES_DEPLOY_KEY tightly held.)
BUILDER="_upd/.ci/build-manifest.py"
[ -f .ci/build-manifest.py ] && BUILDER=".ci/build-manifest.py"
python3 "${BUILDER}" --repo-root . --computed computed.json --config "${CFG}" --out "_upd/${SLUG}.json"
mkdir -p "_upd/${SLUG}"
rm -f "_upd/${SLUG}"/*.zip
cp "${ZIP}" "_upd/${SLUG}/"
[ -d icons ] && cp icons/icon-128x128.png icons/icon-256x256.png "_upd/${SLUG}/" 2>/dev/null || true
( cd _upd
git config user.email "ci@${SSH_HOST}"
git config user.name "ee-extensions release bot"
git add -A
if git diff --cached --quiet; then echo "channel already current — nothing to publish."; exit 0; fi
git commit -m "release ${SLUG} ${VER}"
pushed=0
for i in 1 2 3 4 5; do
if git push; then pushed=1; break; fi
echo "channel push raced; backing off ${i}/5"
sleep $(( (RANDOM % 20) + 1 )) # large random jitter to de-sync concurrent releases
git pull --rebase --no-edit
done
[ "${pushed}" = 1 ] || { echo "::error::channel push failed after 5 retries"; exit 1; }
echo "Published ${SLUG} ${VER} to ${CHANNEL}: ${DL}" )