diff --git a/README.md b/README.md index db487c2..9388acb 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,12 @@ Reads the `README.md` of every repo in the GitLab challenges group (REST API, no clone) and shows which categories have challenges in which state, at which -difficulty — in the terminal and as a HedgeDoc note. +difficulty — in the terminal and as a HedgeDoc note. A status of Testing Ready +or beyond is only believed if the repo backs it up (see +[Checking the repo](#checking-the-repo)). + +Needs `requests` and `PyYAML` (`python-requests python-yaml` on Arch, +`python3-requests python3-yaml` on Debian). ## Setup @@ -23,7 +28,7 @@ $EDITOR ctf_status.env # GitLab group URL + token, HedgeDoc note The token is a fine-grained personal access token on the challenges group with **Project: Read** (to list the repos) and **Repository: Read** (to download the -READMEs). A plain `.env` with `GL_PAT=` / `GITLAB_URL=` is read as well. +READMEs and `description.yml`, and to list the repo's files). A plain `.env` with `GL_PAT=` / `GITLAB_URL=` is read as well. ## Keeping a HedgeDoc note up to date @@ -44,7 +49,7 @@ anonymous. HedgeDoc 1.x has no REST API for editing a note, so the script talks editor's own engine.io/socket.io + OT protocol over long-polling — it reads the current text, sends one "delete everything, insert this" operation, waits for the server's `ack`, and reads the note back to confirm. Anyone with the note open sees -the change live. Still `requests`-only, no new dependencies. +the change live. Plain `requests`, no websocket library. `--hedgedoc ` (POST a *new* note) is unrelated and only works where anonymous note creation is enabled — on `hedgedoc.cit.tum.de` it is not. @@ -52,6 +57,7 @@ anonymous note creation is enabled — on `hedgedoc.cit.tum.de` it is not. ## challs.txt for the deployment repo `--challs` commits the clone URL of every challenge at **Testing Ready or beyond** +that is not blocked (see [Checking the repo](#checking-the-repo)) to `CHALLS_PATH` (default `2026/challs.txt`) in the deployment repo `CHALLS_REPO_URL`, one per line, the same format as last year's: @@ -122,14 +128,16 @@ one-line error and a non-zero exit. 1. **✅ Ready for infra testing** — first thing in the note: every challenge at Testing Ready, Tested or Ready, linked to its repo, so infra knows what it - can deploy and test against right now. -2. **Status by category** — how many challenges each category has per stage + can deploy and test against right now. Blocked ones are not in it. +2. **⛔ Claims Testing Ready+, but the repo disagrees** — only shown when there + are any: what the README claims, and why the repo says otherwise. +3. **Status by category** — how many challenges each category has per stage (⚪ Planning → 🟠 PoC Draft → 🟡 Testing Ready → 🔵 Tested → 🟢 Ready). -3. **Category × difficulty** — one dot per challenge, coloured by its status, so - gaps and stragglers show up at a glance. -4. **Challenges** — one table per category, linked to the repo, with event, +4. **Category × difficulty** — one dot per challenge, coloured by its status (⛔ + when blocked), so gaps and stragglers show up at a glance. +5. **Challenges** — one table per category, linked to the repo, with event, author, tester and the repo's last activity. -5. **Needs attention** — see below. +6. **Needs attention** — see below. ## Reading the README @@ -156,6 +164,28 @@ People fill it in loosely, so values are normalised: is shown next to the challenge. Categories outside the usual list (e.g. `ai`) get their own row. +## Checking the repo + +Anyone can write `Status: Testing Ready`, so for every challenge whose README +claims Testing Ready, Tested or Ready the repo itself is looked at too. Each of +these makes it **blocked** (⛔): it keeps its claimed status in the counts, but +it is left out of *Ready for infra testing* and `challs.txt`, and the reason is +shown in the note. + +- no `description.yml` at the top level, or one that does not parse +- `description.yml` still has anything from the template it was copied from + (`netcat-check-multi-container`): its id, name, author `jh`, its flag, or the + `cont1`/`cont2` / `container-1`/`container-2` containers +- its `category` is not one of the README's categories +- the flag is not `h4tum{...}` +- `dynamic: true`, but no container has a `flag:` block to receive it +- a container's `deploy_dir` does not exist or has no `Dockerfile` +- no `solve.py` anywhere in the repo + +These are cheap signs that a challenge was never finished, not proof that it +works: a repo can pass every check and still not build or solve. That is what +actually testing it is for. + ## What it flags - challenge at Testing Ready or beyond with no tester (`[Tester Name]`, `??`, diff --git a/ctf_status.py b/ctf_status.py index a0ca53b..cc4051c 100755 --- a/ctf_status.py +++ b/ctf_status.py @@ -6,6 +6,11 @@ Lists every repo in the GitLab challenges group, reads each repo's README.md * a terminal summary: status per category, category x difficulty, open issues * a HedgeDoc-ready markdown overview +A README status of Testing Ready or beyond is only believed if the repo backs it +up: description.yml is filled in (not the template), its flag and containers make +sense, and the deploy dirs and solve.py exist. Otherwise the challenge is shown +as blocked and kept out of challs.txt. + Config: see ctf_status.env.example -> ctf_status.env (chmod 600) """ from __future__ import annotations @@ -28,6 +33,7 @@ from pathlib import Path from urllib.parse import quote, urlsplit import requests +import yaml # ---------------------------------------------------------------- config ---- @@ -74,6 +80,11 @@ class Repo: ssh_url: str last_activity: str # YYYY-MM-DD readme: str | None # None: the repo has no README at all + id: int = 0 + branch: str | None = None + # only fetched for challenges whose README claims Testing Ready or beyond + files: set[str] | None = None # every file and dir in the repo + desc: str | None = None # description.yml, None if there is none class GitLab: @@ -121,8 +132,10 @@ class GitLab: branch, url = p.get("default_branch"), p.get("readme_url") if not branch or not url or f"/-/blob/{branch}/" not in url: return None - path = url.split(f"/-/blob/{branch}/", 1)[1] - r = self._get(f"{self.api}/projects/{p['id']}/repository/files/" + return self._raw(p["id"], url.split(f"/-/blob/{branch}/", 1)[1], branch) + + def _raw(self, project_id: int, path: str, branch: str) -> str: + r = self._get(f"{self.api}/projects/{project_id}/repository/files/" f"{quote(path, safe='')}/raw", ref=branch) return r.content.decode("utf-8", "replace") @@ -133,11 +146,25 @@ class GitLab: return Repo(path=p["path"], web_url=p["web_url"], ssh_url=p["ssh_url_to_repo"], last_activity=(p.get("last_activity_at") or "")[:10], - readme=self.readme(p)) + readme=self.readme(p), id=p["id"], branch=p.get("default_branch")) with ThreadPoolExecutor(max_workers=8) as pool: return list(pool.map(fetch, projects)) + def load_files(self, repo: Repo) -> None: + """Fill in repo.files and repo.desc, for check_files().""" + repo.files = set() + if not repo.branch: + return + page = "1" + while page: + r = self._get(f"{self.api}/projects/{repo.id}/repository/tree", ref=repo.branch, + recursive="true", per_page="100", page=page) + repo.files |= {t["path"] for t in r.json()} + page = r.headers.get("X-Next-Page", "") + if "description.yml" in repo.files: + repo.desc = self._raw(repo.id, "description.yml", repo.branch) + def write_file(self, repo_url: str, path: str, content: str, branch: str | None, message) -> bool: """Commit `content` to `path` in the repo at `repo_url` (a web URL), creating @@ -175,6 +202,17 @@ CATEGORIES = ["crypto", "pwn", "web", "rev", "misc", "osint", "forensic", "hardw DIFFICULTIES = ["intro", "easy", "medium", "hard"] DIFF_RE = re.compile(r"intro|easy|medium|hard", re.I) DIFF_RANGE_RE = re.compile(r"(intro|easy|medium|hard)\s*(?:-|–|/|\bto\b)\s*(intro|easy|medium|hard)", re.I) +CATEGORY_SPLIT_RE = re.compile(r"[,/&+]| and ") + +BLOCKED_ICON = "⛔" +FLAG_RE = re.compile(r"^h4tum\{[^{}]+\}$") +# what every repo starts with: the challenge-format example "netcat-check-multi-container" +TEMPLATE_ID = "netcat-check-multi-container" +TEMPLATE_NAME = "Netcat Check Multi Container" +TEMPLATE_AUTHOR = "jh" +TEMPLATE_FLAG = "h4tum{n3tc4t_1s_c0ol}" +TEMPLATE_CONTAINERS = {"cont1", "cont2"} +TEMPLATE_DIRS = {"container-1", "container-2"} def clean(v: str) -> str: @@ -230,6 +268,7 @@ class Challenge: repo: Repo name: str = "" fields: dict = field(default_factory=dict) + blockers: list = field(default_factory=list) # from check_files() def get(self, key: str, dash: str = "—") -> str: v = clean(self.fields.get(key, "")) @@ -265,9 +304,18 @@ class Challenge: def category(self) -> str | None: """The first category named; "rev, misc" counts as rev.""" raw = self.get("Category", "") - first = re.split(r"[,/&+]| and ", raw)[0].strip().lower() + first = CATEGORY_SPLIT_RE.split(raw)[0].strip().lower() return first or None + @property + def claims_testable(self) -> bool: + """The README says Testing Ready or beyond; is_testable() also asks the repo.""" + return bool(self.stage) and STATUS_ORDER.index(self.stage) >= STATUS_ORDER.index("Testing Ready") + + @property + def icon(self) -> str: + return BLOCKED_ICON if self.blockers else STATUS_ICON[self.stage] + @property def has_tester(self) -> bool: t = self.get("Tester", "") @@ -287,7 +335,7 @@ class Challenge: if not self.difficulty: p.append(f"difficulty unclear ({self.get('Difficulty')!r})" if self.get("Difficulty") != "—" else "no difficulty") - if is_testable(self) and not self.has_tester: + if self.claims_testable and not self.has_tester: p.append("no tester") if not self.named: p.append("no challenge name in README") @@ -295,7 +343,57 @@ class Challenge: def is_testable(c: Challenge) -> bool: - return bool(c.stage) and STATUS_ORDER.index(c.stage) >= STATUS_ORDER.index("Testing Ready") + return c.claims_testable and not c.blockers + + +def check_files(c: Challenge) -> list[str]: + """Reasons not to believe a Testing Ready+ status: cheap signs in the repo that + the challenge was never finished. Passing is no proof that it builds or solves.""" + repo = c.repo + if repo.desc is None: + return ["no description.yml"] + try: + doc = yaml.safe_load(repo.desc) + except yaml.YAMLError as e: + return [f"description.yml does not parse ({str(e).splitlines()[0]})"] + if not isinstance(doc, dict) or len(doc) != 1 or not isinstance(next(iter(doc.values())), dict): + return ["description.yml is not a single `: {...}` block"] + cid, d = next(iter(doc.items())) + flag = d.get("flag") if isinstance(d.get("flag"), dict) else {} + containers = [x for x in d.get("containers") or [] if isinstance(x, dict)] + dirs = [os.path.normpath(str(x.get("deploy_dir") or ".")) for x in containers] + b = [] + + template = [what for what, hit in ( + (f"id {cid!r}", cid == TEMPLATE_ID), + (f"name {d.get('name')!r}", d.get("name") == TEMPLATE_NAME), + (f"author {d.get('author')!r}", d.get("author") == TEMPLATE_AUTHOR), + ("flag", flag.get("content") == TEMPLATE_FLAG), + ("containers", any(x.get("name") in TEMPLATE_CONTAINERS for x in containers) + or bool(TEMPLATE_DIRS & set(dirs))), + ) if hit] + if template: + b.append("description.yml still has the template's " + ", ".join(template)) + + cat = str(d.get("category") or "").strip().lower() + readme_cats = {x.strip().lower() for x in CATEGORY_SPLIT_RE.split(c.get("Category", ""))} + if c.category and cat not in readme_cats: + b.append(f"category {cat or 'missing'} in description.yml, README says {c.get('Category')}") + + content = str(flag.get("content") or "") + if not FLAG_RE.match(content): + b.append(f"flag {content} is not h4tum{{...}}" if content else "no flag in description.yml") + if flag.get("dynamic") and not any(isinstance(x.get("flag"), dict) for x in containers): + b.append("dynamic flag, but no container is handed one") + + for dd in dirs: + if dd != "." and dd not in repo.files: + b.append(f"deploy_dir {dd} does not exist") + elif ("Dockerfile" if dd == "." else f"{dd}/Dockerfile") not in repo.files: + b.append(f"no Dockerfile in {dd}") + if not any(p == "solve.py" or p.endswith("/solve.py") for p in repo.files): + b.append("no solve.py") + return b def parse_challenge(repo: Repo) -> Challenge: @@ -341,9 +439,17 @@ class Overview: @property def testable(self) -> list: """Everything infra can already deploy and test: Testing Ready or beyond.""" - return sorted((c for c in self.challenges if is_testable(c)), - key=lambda c: (self.categories.index(c.category or "?"), - difficulty_rank(c.difficulty), c.title.lower())) + return sorted((c for c in self.challenges if is_testable(c)), key=self._ready_order) + + @property + def blocked(self) -> list: + """Testing Ready or beyond according to the README, but not the repo.""" + return sorted((c for c in self.challenges if c.claims_testable and c.blockers), + key=self._ready_order) + + def _ready_order(self, c: Challenge) -> tuple: + return (self.categories.index(c.category or "?"), + difficulty_rank(c.difficulty), c.title.lower()) def by_category(self) -> dict[str, list]: out = defaultdict(list) @@ -360,11 +466,18 @@ def collect(gl: GitLab) -> Overview: for repo in gl.repos(): c = parse_challenge(repo) (chs if c.touched else blank).append(c) + + def check(c: Challenge) -> None: + gl.load_files(c.repo) + c.blockers = check_files(c) + + with ThreadPoolExecutor(max_workers=8) as pool: + list(pool.map(check, [c for c in chs if c.claims_testable])) return Overview(chs, sorted(blank, key=lambda c: c.repo.path.lower())) def icons(chs: list) -> str: - return "".join(STATUS_ICON[c.stage] for c in chs) + return "".join(c.icon for c in chs) def print_terminal(ov: Overview, c: bool) -> None: @@ -382,6 +495,13 @@ def print_terminal(ov: Overview, c: bool) -> None: print(f" {STATUS_ICON[ch.stage]} {ch.title[:34]:<35}{ch.category or '?':<10}" f"{ch.difficulty or '?':<13}{col(DIM)}{ch.repo.web_url}{col(RST)}") + if ov.blocked: + print(f"\n{col(BOLD)}{col(RED)}=== {BLOCKED_ICON} claims {STATUS_ORDER[2]}+, " + f"but the repo disagrees ({len(ov.blocked)}) ==={col(RST)}") + for ch in ov.blocked: + print(f" {STATUS_ICON[ch.stage]} {ch.title[:34]:<35}{col(DIM)}" + + "; ".join(ch.blockers) + col(RST)) + print(f"\n{col(BOLD)}=== status by category ==={col(RST)}") print(f" {'':<10}" + "".join(f"{s or '?':>14}" for s in stages) + f"{'total':>7}") for cat in ov.categories: @@ -392,7 +512,7 @@ def print_terminal(ov: Overview, c: bool) -> None: + f"{len(chs):>7}{col(RST)}") print(f"\n{col(BOLD)}=== category × difficulty ==={col(RST)} " - + " ".join(f"{STATUS_ICON[s]} {s}" for s in STATUS_ORDER)) + + " ".join(f"{STATUS_ICON[s]} {s}" for s in STATUS_ORDER) + f" {BLOCKED_ICON} blocked") diffs = ov.difficulties print(f" {'':<10}" + "".join(f"{d:<13}" for d in diffs)) for cat in ov.categories: @@ -415,7 +535,7 @@ def print_terminal(ov: Overview, c: bool) -> None: def render_challs(ov: Overview) -> str: """The deployment repo's challs.txt: one clone URL per line, for everything - that is Testing Ready or beyond.""" + that is Testing Ready or beyond and not blocked.""" return "".join(f"{c.repo.ssh_url}\n" for c in ov.testable) @@ -448,8 +568,8 @@ def render_hedgedoc(ov: Overview, title: str, tags: str = "") -> str: ready = ov.testable L += [f"## ✅ Ready for infra testing ({len(ready)} of {len(ov.challenges)})", "", - "*every challenge at Testing Ready or beyond — these can be deployed and " - "tested against the infrastructure now*", ""] + "*every challenge at Testing Ready or beyond whose repo backs that up — " + "these can be deployed and tested against the infrastructure now*", ""] if ready: L += ["| | Challenge | Category | Difficulty | Status | Event | Author | Tester |", "| --- | --- | --- | --- | --- | --- | --- | --- |"] @@ -462,6 +582,16 @@ def render_hedgedoc(ov: Overview, title: str, tags: str = "") -> str: L.append("*nothing yet*") L.append("") + if ov.blocked: + L += [f"## {BLOCKED_ICON} Claims Testing Ready+, but the repo disagrees " + f"({len(ov.blocked)})", "", + "*the README says so, but description.yml or the repo shows it is not " + "finished — kept out of challs.txt until this is fixed*", ""] + for ch in ov.blocked: + L.append(f"- {STATUS_ICON[ch.stage]} [{md_cell(ch.title)}]({ch.repo.web_url}) " + f"({ch.stage}): " + "; ".join(ch.blockers)) + L.append("") + L += ["## Status by category", "", "| Category | " + " | ".join(f"{STATUS_ICON[s]} {s or '?'}" for s in stages) + " | total |", @@ -477,7 +607,8 @@ def render_hedgedoc(ov: Overview, title: str, tags: str = "") -> str: diffs = ov.difficulties L += ["", "## Category × difficulty", "", "*one dot per challenge, coloured by status: " - + " ".join(f"{STATUS_ICON[s]} {s}" for s in STATUS_ORDER) + "*", "", + + " ".join(f"{STATUS_ICON[s]} {s}" for s in STATUS_ORDER) + + f" {BLOCKED_ICON} blocked*", "", "| Category | " + " | ".join(diffs) + " | total |", "| --- |" + " --- |" * len(diffs) + " ---: |"] for cat in ov.categories: @@ -505,7 +636,7 @@ def render_hedgedoc(ov: Overview, title: str, tags: str = "") -> str: if cat_raw.lower() != cat: name += f" *({cat_raw})*" L.append("| " + " | ".join(md_cell(x) for x in ( - STATUS_ICON[ch.stage], name, ch.difficulty or ch.get("Difficulty"), status, + ch.icon, name, ch.difficulty or ch.get("Difficulty"), status, ch.get("Event"), ch.get("Author"), ch.get("Tester") if ch.has_tester else "—", ch.repo.last_activity)) + " |") @@ -713,7 +844,7 @@ def main() -> None: help="overwrite an existing HedgeDoc note in place; defaults to " "HEDGEDOC_NOTE from the config. This is the one to cron.") ap.add_argument("--challs", action="store_true", - help="commit the clone URLs of every Testing Ready+ challenge to " + help="commit the clone URLs of every unblocked Testing Ready+ challenge to " f"CHALLS_PATH (default {DEFAULT_CHALLS_PATH!r}) in the " "deployment repo CHALLS_REPO_URL, if they changed") ap.add_argument("--challs-out", metavar="FILE",