From e8a04fe415bff004038d47a90b98913e6fe6cc44 Mon Sep 17 00:00:00 2001 From: cato447 Date: Tue, 20 Oct 2026 16:12:35 +0200 Subject: [PATCH] Source challenge status from GitLab instead of Nextcloud Read the README.md of every repo in the GitLab challenges group via the REST API and rework the HedgeDoc note for project tracking: challenges ready for infra testing first, then status by category and a category x difficulty grid. Free-text status/difficulty/category values are normalised; untouched templates and repos scheduled for deletion are left out of the counts. Co-Authored-By: Claude Opus 5.5 --- .gitignore | 1 + README.md | 83 ++++-- ctf-status.service | 2 +- ctf-status.timer | 2 +- ctf_status.env.example | 15 +- ctf_status.py | 611 ++++++++++++++++++++++++----------------- 6 files changed, 423 insertions(+), 291 deletions(-) diff --git a/.gitignore b/.gitignore index 276e14a..3d47cce 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ __pycache__/ ctf_status.env *.md.out +.env diff --git a/README.md b/README.md index 5f4cd79..830f09b 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,14 @@ # ctf_status.py -Reads the per-author challenge folders straight off the Nextcloud share (WebDAV, -no sync, no mount) and tells you who to scream at. +Reads the `README.md` of every repo in the GitLab challenges group (REST API, no +clone) and shows which categories have challenges in which state, at which +difficulty — in the terminal and as a HedgeDoc note. ## Setup ```sh cp ctf_status.env.example ctf_status.env && chmod 600 ctf_status.env -$EDITOR ctf_status.env # app password from Nextcloud → Settings → Security +$EDITOR ctf_status.env # GitLab group URL + token, HedgeDoc note ``` ## Use @@ -20,6 +21,10 @@ $EDITOR ctf_status.env # app password from Nextcloud → Settings → Secur ./ctf_status.py --update # ... or an explicit one ``` +The token is a fine-grained personal access token on the challenges group with +**Project: Read** (to list the repos) and **Repository: Read** (to download the +READMEs). A plain `.env` with `GL_PAT=` / `GITLAB_URL=` is read as well. + ## Keeping a HedgeDoc note up to date `--update` overwrites an **existing** note in place, so the URL you share stays the @@ -78,33 +83,51 @@ changed or something broke. Overlapping runs are prevented by a lock file, and t HedgeDoc write is retried (`--retries`, default 3) before the run gives up with a one-line error and a non-zero exit. +## What the note shows + +1. **✅ Ready for infra testing** — first thing in the note: every challenge at + Testing Ready, Tested or Ready, linked to its repo, so infra knows what it + can deploy and test against right now. +2. **Status by category** — how many challenges each category has per stage + (⚪ Planning → 🟠 PoC Draft → 🟡 Testing Ready → 🔵 Tested → 🟢 Ready). +3. **Category × difficulty** — one dot per challenge, coloured by its status, so + gaps and stragglers show up at a glance. +4. **Challenges** — one table per category, linked to the repo, with event, + author, tester and the repo's last activity. +5. **Needs attention** — see below. + +## Reading the README + +Only the template's header block is parsed: + +``` +### Challenge Name +- Category: crypto +- Difficulty: medium +- Event: TUM student CTF +- Author: ... +- Tester: ... +- Status: Testing Ready +``` + +People fill it in loosely, so values are normalised: + +- **Status** is matched against the five stages ("Tested with …" → Tested, + "PoC" → PoC Draft). If it names several ("Planning to PoC Draft", a half-deleted + option list) the *least* advanced one counts and it is flagged as unclear. +- **Difficulty** ranges written as `medium-hard`, `medium/hard` or + `easy to medium` become their own column; otherwise the first level named wins. +- **Category**: the first one named counts ("rev, misc" → rev); the full value + is shown next to the challenge. Categories outside the usual list (e.g. `ai`) + get their own row. + ## What it flags -| verdict | meaning | -| --- | --- | -| 🔴 SCREAM | nothing submitted, **or** they filled out `template.md` in place | -| 🟡 NAG | submitted, but placeholders / empty sections remain | -| 🟢 OK | every challenge fully filled out | +- challenge at Testing Ready or beyond with no tester (`[Tester Name]`, `??`, + `None`, `not found yet` all count as none) +- status / difficulty / category missing or unclear +- no challenge name in the README +- repos whose README is missing or still the untouched template — these are left + out of the counts -A `template.md` that is still pristine is ignored. One that has real content in it -is counted as a challenge *and* flagged — so nobody's work is lost, they just get -yelled at. - -Also flagged per challenge: unnamed challenge, `[placeholder]` values left in any -of Category/Difficulty/Event/Author/Tester/Status, empty prose sections, and -category/difficulty values outside the allowed set. - -## Layout it expects - -``` -/ - alice/ - README.md # ignored - template.md # ignored while pristine - baby-rsa.md # ← a challenge, any filename - bob/ -``` - -Any `.md` that isn't `README.md` is parsed as a challenge, so the file naming -convention doesn't have to be enforced by the script. If you want it enforced, -add the check in `Challenge.problems`. +Repos GitLab has scheduled for deletion, and archived repos, are skipped. diff --git a/ctf-status.service b/ctf-status.service index 93d5eb4..ba1c07d 100644 --- a/ctf-status.service +++ b/ctf-status.service @@ -8,5 +8,5 @@ Wants=network-online.target Type=oneshot WorkingDirectory=__DIR__ ExecStart=__DIR__/ctf_status.py --update --quiet -# Nextcloud walk + HedgeDoc write; never let one run squat on the next timer slot +# GitLab walk + HedgeDoc write; never let one run squat on the next timer slot TimeoutStartSec=5min diff --git a/ctf-status.timer b/ctf-status.timer index a05bf50..3746fa5 100644 --- a/ctf-status.timer +++ b/ctf-status.timer @@ -3,7 +3,7 @@ Description=Refresh the CTF challenge status every 30 minutes [Timer] OnCalendar=*:0/30 -# don't hammer Nextcloud and HedgeDoc on the exact half hour +# don't hammer GitLab and HedgeDoc on the exact half hour RandomizedDelaySec=60 AccuracySec=30s # catch up on the last run if the machine was down when it was due diff --git a/ctf_status.env.example b/ctf_status.env.example index e9b92ba..6155ea7 100644 --- a/ctf_status.env.example +++ b/ctf_status.env.example @@ -1,10 +1,11 @@ -# cp to ctf_status.env and chmod 600 -NC_URL=https://cloud.h4tum.de -NC_USER=your-nextcloud-login -# Settings -> Security -> "Create new app password" (NOT your real password) -NC_PASS=xxxxx-xxxxx-xxxxx-xxxxx-xxxxx -# path of the folder holding the per-author folders, relative to your Nextcloud root -NC_ROOT=CTF/challenges +# cp to ctf_status.env and chmod 600 (a plain .env next to the script works too) + +# --- GitLab: the group holding one repo per challenge ----------------------- +GITLAB_GROUP_URL=https://gitlab.lrz.de/h4tum-ctf/h4tum-ctf-2026/challenges +# fine-grained personal access token, scoped to that group, with +# Project: Read (list the repos) +# Repository: Read (download each README.md) +GITLAB_TOKEN=glpat-xxxxxxxxxxxxxxxxxxxx # --- HedgeDoc: the existing note that gets overwritten every run ------------ # full URL of the note (its permission must be "freely" so an anonymous diff --git a/ctf_status.py b/ctf_status.py index a001cec..7387bed 100755 --- a/ctf_status.py +++ b/ctf_status.py @@ -1,9 +1,9 @@ #!/usr/bin/env python3 """CTF challenge tracker. -Reads the per-author challenge folders from a Nextcloud share over WebDAV, -parses the filled-out challenge templates and reports: - * a terminal "who do I have to scream at" list +Lists every repo in the GitLab challenges group, reads each repo's README.md +(the filled-out challenge template) and reports: + * a terminal summary: status per category, category x difficulty, open issues * a HedgeDoc-ready markdown overview Config: see ctf_status.env.example -> ctf_status.env (chmod 600) @@ -20,125 +20,136 @@ import sys import tempfile import time from collections import Counter, defaultdict +from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass, field from datetime import datetime from pathlib import Path -from urllib.parse import quote, unquote, urljoin, urlsplit -from xml.etree import ElementTree as ET +from urllib.parse import quote, urlsplit import requests # ---------------------------------------------------------------- config ---- -CFG_KEYS = ("NC_URL", "NC_USER", "NC_PASS", "NC_ROOT") +CFG_KEYS = ("GITLAB_GROUP_URL", "GITLAB_TOKEN") OPT_KEYS = ("HEDGEDOC_NOTE", "HEDGEDOC_TAGS", "TITLE") +# the names a plain `.env` tends to use for the same thing +ALIASES = {"GL_PAT": "GITLAB_TOKEN", "GITLAB_URL": "GITLAB_GROUP_URL"} DEFAULT_TAGS = "Logistics, h4tum CTF 2026" DEFAULT_TITLE = "CTF challenge tracking" def load_env() -> dict: cfg = {} - for candidate in ( - Path(__file__).with_name("ctf_status.env"), - Path.home() / ".config" / "ctf-status" / "env", - ): - if candidate.exists(): - for line in candidate.read_text().splitlines(): - line = line.strip() - if not line or line.startswith("#") or "=" not in line: - continue - k, v = line.split("=", 1) - cfg[k.strip()] = v.strip().strip("'\"") - break - for k in CFG_KEYS + OPT_KEYS: + here = Path(__file__).parent + # every file that exists contributes; the first one to set a key wins + for candidate in (here / "ctf_status.env", here / ".env", + Path.home() / ".config" / "ctf-status" / "env"): + if not candidate.exists(): + continue + for line in candidate.read_text().splitlines(): + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + k, v = line.split("=", 1) + k = k.strip().upper() + cfg.setdefault(ALIASES.get(k, k), v.strip().strip("'\"")) + for k in CFG_KEYS + OPT_KEYS + tuple(ALIASES): if os.environ.get(k): - cfg[k] = os.environ[k] + cfg[ALIASES.get(k, k)] = os.environ[k] missing = [k for k in CFG_KEYS if not cfg.get(k)] if missing: sys.exit(f"missing config: {', '.join(missing)} (see ctf_status.env.example)") return cfg -# --------------------------------------------------------------- webdav ----- +# --------------------------------------------------------------- gitlab ----- -DAV_NS = "{DAV:}" +@dataclass +class Repo: + path: str + web_url: str + last_activity: str # YYYY-MM-DD + readme: str | None # None: the repo has no README at all -class Dav: - def __init__(self, base_url: str, user: str, password: str): - self.base = base_url.rstrip("/") + f"/remote.php/dav/files/{quote(user)}/" +class GitLab: + """Read-only access to one group. The token needs Project: Read (to list the + group) and Repository: Read (to download files).""" + + def __init__(self, group_url: str, token: str, timeout: int = 20): + parts = urlsplit(group_url) + self.api = f"{parts.scheme}://{parts.netloc}/api/v4" + self.group = parts.path.strip("/") + self.timeout = timeout self.s = requests.Session() - self.s.auth = (user, password) + self.s.headers["PRIVATE-TOKEN"] = token - def _url(self, path: str) -> str: - return urljoin(self.base, quote(path.strip("/")) + "/") - - def walk(self, root: str) -> list[str]: - """Return all file paths (relative to root) under root.""" - body = ( - '' - "" - ) - out: list[str] = [] - for depth in ("infinity", "1"): + def _get(self, url: str, **params) -> requests.Response: + r = self.s.get(url, params=params, timeout=self.timeout) + if r.status_code == 403: + # fine-grained tokens say exactly which permission is missing try: - out = self._propfind(root, body, depth) - if out: - return out - except requests.HTTPError as e: - if depth == "infinity" and e.response is not None and e.response.status_code in (403, 405, 507): - continue # server forbids Depth: infinity -> recurse manually - raise - return out - - def _propfind(self, root: str, body: str, depth: str) -> list[str]: - r = self.s.request( - "PROPFIND", self._url(root), - headers={"Depth": depth, "Content-Type": "application/xml"}, data=body, - ) + why = r.json().get("error_description") or r.text + except ValueError: + why = r.text + raise RuntimeError(f"gitlab refused {r.url}: {why}") r.raise_for_status() - tree = ET.fromstring(r.content) - base_path = unquote(self._url(root).split(self.base.split("/remote.php")[0], 1)[-1]) - files, dirs = [], [] - for resp in tree.findall(f"{DAV_NS}response"): - href = unquote(resp.findtext(f"{DAV_NS}href") or "") - if not href or href.rstrip("/") == base_path.rstrip("/"): - continue - rel = href.split(base_path, 1)[-1].strip("/") if base_path in href else href - is_dir = resp.find(f".//{DAV_NS}collection") is not None - (dirs if is_dir else files).append(rel) - if depth == "1": - for d in dirs: - files += [f"{d}/{f}" for f in self._propfind(f"{root}/{d}", body, "1")] - return files + return r - def get(self, path: str) -> str: - r = self.s.get(self._url(path).rstrip("/")) - r.raise_for_status() + def projects(self) -> list[dict]: + out, page = [], "1" + while page: + r = self._get(f"{self.api}/groups/{quote(self.group, safe='')}/projects", + include_subgroups="true", per_page="100", page=page, + order_by="path", sort="asc") + out += r.json() + page = r.headers.get("X-Next-Page", "") + # repos GitLab is about to delete are renamed "...-deletion_scheduled-" + return [p for p in out if not p.get("archived") and not p.get("marked_for_deletion_on")] + + def readme(self, p: dict) -> str | None: + branch, url = p.get("default_branch"), p.get("readme_url") + if not branch or not url or f"/-/blob/{branch}/" not in url: + return None + path = url.split(f"/-/blob/{branch}/", 1)[1] + r = self._get(f"{self.api}/projects/{p['id']}/repository/files/" + f"{quote(path, safe='')}/raw", ref=branch) return r.content.decode("utf-8", "replace") + def repos(self) -> list[Repo]: + projects = self.projects() + + def fetch(p: dict) -> Repo: + return Repo(path=p["path"], web_url=p["web_url"], + last_activity=(p.get("last_activity_at") or "")[:10], + readme=self.readme(p)) + + with ThreadPoolExecutor(max_workers=8) as pool: + return list(pool.map(fetch, projects)) + # --------------------------------------------------------------- parsing ---- FIELDS = ("Category", "Difficulty", "Event", "Author", "Tester", "Status") -HEADING_RE = re.compile(r"^\s{0,3}#{2,4}\s+(.+?)\s*$", re.M) +HEADING_RE = re.compile(r"^\s{0,3}#{1,4}\s+(.+?)\s*$", re.M) FIELD_RE = re.compile(r"^\s*[-*]\s*\**(?P[A-Za-z ]+?)\**\s*:\s*(?P.*?)\s*$", re.M) -SECTION_RE = re.compile(r"^\s{0,3}#{3,5}\s*(?P.+?)\s*$\n(?P.*?)(?=^\s{0,3}#{2,5}\s|\Z)", - re.M | re.S) # a value the author never touched: "[crypto, pwn, ...]" or "[Author Name]" -PLACEHOLDER_RE = re.compile(r"^\\?\[.*\\?\]$") -PROSE_PROMPTS = ( - "what should the player take home", "please answer in 1-2", - "how do you plan on implementing", "needed for infrastructure", - "how will the players exploit", -) +PLACEHOLDER_RE = re.compile(r"^\[.*\]$") +NO_TESTER_RE = re.compile(r"^(\?+|none|tester name|not found\b.*|tbd)(\s.*)?$", re.I) + STATUS_ORDER = ["Planning", "PoC Draft", "Testing Ready", "Tested", "Ready"] +STATUS_RE = re.compile(r"testing\s*ready|tested|ready|poc(?:\s*draft)?|planning", re.I) +STATUS_ICON = {"Planning": "⚪", "PoC Draft": "🟠", "Testing Ready": "🟡", + "Tested": "🔵", "Ready": "🟢", None: "❔"} CATEGORIES = ["crypto", "pwn", "web", "rev", "misc", "osint", "forensic", "hardware"] DIFFICULTIES = ["intro", "easy", "medium", "hard"] +DIFF_RE = re.compile(r"intro|easy|medium|hard", re.I) +DIFF_RANGE_RE = re.compile(r"(intro|easy|medium|hard)\s*(?:-|–|/|\bto\b)\s*(intro|easy|medium|hard)", re.I) def clean(v: str) -> str: - return v.replace("\\[", "[").replace("\\]", "]").strip() + v = re.sub(r"\\+([\[\]_*])", r"\1", v) # markdown escapes, sometimes doubled + return v.replace("**", "").strip() def is_placeholder(v: str) -> bool: @@ -146,68 +157,120 @@ def is_placeholder(v: str) -> bool: return not v or bool(PLACEHOLDER_RE.match(v)) +def stage_of(raw: str) -> tuple[str | None, bool]: + """Map a free-text status onto STATUS_ORDER -> (stage, clear). + + "Tested with h4tum-chall-format" is clearly Tested. "Planning to PoC Draft" + or a half-deleted option list is not; we take the least advanced stage it + names, so nothing looks further along than it is, and call it unclear.""" + if is_placeholder(raw): + return None, False + found = set() + for m in STATUS_RE.finditer(raw): + w = m.group(0).lower() + found.add("Testing Ready" if w.startswith("testing") else + "PoC Draft" if w.startswith("poc") else w.title()) + if not found: + return None, False + return min(found, key=STATUS_ORDER.index), len(found) == 1 + + +def difficulty_of(raw: str) -> str | None: + """"medium/hard", "medium-hard", "easy to medium" -> a range "medium-hard".""" + if is_placeholder(raw): + return None + m = DIFF_RANGE_RE.search(raw) + if m: + ends = sorted({m.group(1).lower(), m.group(2).lower()}, key=DIFFICULTIES.index) + return "-".join(ends) if len(ends) > 1 else ends[0] + # prose like "currently medium, maybe also an easy and a hard version": the first one + m = DIFF_RE.search(raw) + return m.group(0).lower() if m else None + + +def difficulty_rank(d: str | None) -> float: + if not d: + return 99 + ends = d.split("-") + return sum(DIFFICULTIES.index(e) for e in ends) / len(ends) + + @dataclass class Challenge: - author_dir: str - path: str + repo: Repo name: str = "" fields: dict = field(default_factory=dict) - sections: dict = field(default_factory=dict) - in_template: bool = False - - # -- derived ----------------------------------------------------------- - @property - def missing_fields(self) -> list[str]: - return [f for f in FIELDS if is_placeholder(self.fields.get(f, ""))] - - @property - def missing_sections(self) -> list[str]: - return [k for k, v in self.sections.items() if not v] - - @property - def named(self) -> bool: - return bool(self.name) and not is_placeholder(self.name) - - @property - def touched(self) -> bool: - """Did anyone put *any* real content in here?""" - return self.named or len(self.missing_fields) < len(FIELDS) or bool( - [k for k, v in self.sections.items() if v]) - - @property - def complete(self) -> bool: - return self.named and not self.missing_fields and not self.missing_sections def get(self, key: str, dash: str = "—") -> str: v = clean(self.fields.get(key, "")) return dash if is_placeholder(v) else v + # -- derived ----------------------------------------------------------- @property - def status(self) -> str: - return self.get("Status", "—") + def named(self) -> bool: + return bool(self.name) and not is_placeholder(self.name) + + @property + def title(self) -> str: + return self.name if self.named else self.repo.path + + @property + def touched(self) -> bool: + """Did anyone put *any* real content in the README template?""" + return self.named or any(not is_placeholder(self.fields.get(f, "")) for f in FIELDS) + + @property + def status_raw(self) -> str: + return self.get("Status") + + @property + def stage(self) -> str | None: + return stage_of(self.fields.get("Status", ""))[0] + + @property + def difficulty(self) -> str | None: + return difficulty_of(self.fields.get("Difficulty", "")) + + @property + def category(self) -> str | None: + """The first category named; "rev, misc" counts as rev.""" + raw = self.get("Category", "") + first = re.split(r"[,/&+]| and ", raw)[0].strip().lower() + return first or None + + @property + def has_tester(self) -> bool: + t = self.get("Tester", "") + return bool(t) and not NO_TESTER_RE.match(t) @property def problems(self) -> list[str]: p = [] - if self.in_template and self.touched: - p.append("filled out template.md in place (rename it!)") + stage, clear = stage_of(self.fields.get("Status", "")) + if stage is None: + p.append(f"status unknown ({self.status_raw!r})" if self.status_raw != "—" + else "no status") + elif not clear: + p.append(f"status unclear ({self.status_raw!r}, counted as {stage})") + if not self.category: + p.append("no category") + if not self.difficulty: + p.append(f"difficulty unclear ({self.get('Difficulty')!r})" + if self.get("Difficulty") != "—" else "no difficulty") + if is_testable(self) and not self.has_tester: + p.append("no tester") if not self.named: - p.append("no challenge name") - if self.missing_fields: - p.append("unfilled: " + ", ".join(self.missing_fields)) - if self.missing_sections: - p.append("empty section(s): " + ", ".join(self.missing_sections)) - cat = self.get("Category", "").lower() - if cat and cat != "—" and cat not in CATEGORIES and "?" not in cat: - p.append(f"odd category {cat!r}") - diff = self.get("Difficulty", "").lower() - if diff and diff != "—" and diff not in DIFFICULTIES: - p.append(f"odd difficulty {diff!r}") + p.append("no challenge name in README") return p -def parse_challenge(author: str, path: str, text: str, in_template: bool) -> Challenge: - c = Challenge(author_dir=author, path=path, in_template=in_template) +def is_testable(c: Challenge) -> bool: + return bool(c.stage) and STATUS_ORDER.index(c.stage) >= STATUS_ORDER.index("Testing Ready") + + +def parse_challenge(repo: Repo) -> Challenge: + c = Challenge(repo=repo) + text = repo.readme or "" m = HEADING_RE.search(text) if m: c.name = clean(m.group(1)) @@ -215,16 +278,6 @@ def parse_challenge(author: str, path: str, text: str, in_template: bool) -> Cha key = fm.group("key").strip().title() if key in FIELDS and key not in c.fields: c.fields[key] = fm.group("val") - for sm in SECTION_RE.finditer(text): - name = clean(sm.group("name")) - if name == c.name or name in FIELDS: - continue - body = sm.group("body") - # strip the template's own prompt lines, keep what the author wrote - real = [ln.strip() for ln in body.splitlines() - if ln.strip() and not ln.strip().startswith(("-", "*", "#")) - and not any(p in ln.lower() for p in PROSE_PROMPTS)] - c.sections[name] = " ".join(real) return c @@ -235,142 +288,193 @@ BOLD, DIM, RED, YEL, GRN, CYA, RST = ( @dataclass -class Author: - name: str - challenges: list = field(default_factory=list) - files_seen: list = field(default_factory=list) +class Overview: + challenges: list # repos with a README somebody filled in + blank: list # repos with no README, or an untouched template @property - def real(self) -> list: - return [c for c in self.challenges if c.touched] + def categories(self) -> list[str]: + extra = sorted({c.category or "?" for c in self.challenges} - set(CATEGORIES)) + return CATEGORIES + extra @property - def verdict(self) -> tuple[str, str]: - if not self.real: - return "SCREAM", "nothing submitted at all" - if any(c.in_template for c in self.real): - return "SCREAM", "worked inside template.md instead of copying it" - bad = [c for c in self.real if not c.complete] - if bad: - return "NAG", f"{len(bad)}/{len(self.real)} challenge(s) incomplete" - return "OK", f"{len(self.real)} challenge(s) done" + def difficulties(self) -> list[str]: + seen = {c.difficulty for c in self.challenges if c.difficulty} + unknown = ["?"] if any(not c.difficulty for c in self.challenges) else [] + return sorted(set(DIFFICULTIES) | seen, key=difficulty_rank) + unknown + + @property + def stages(self) -> list: + """The five stages, plus None ("?") only if some status is unreadable.""" + return STATUS_ORDER + ([None] if any(not c.stage for c in self.challenges) else []) + + @property + def testable(self) -> list: + """Everything infra can already deploy and test: Testing Ready or beyond.""" + return sorted((c for c in self.challenges if is_testable(c)), + key=lambda c: (self.categories.index(c.category or "?"), + difficulty_rank(c.difficulty), c.title.lower())) + + def by_category(self) -> dict[str, list]: + out = defaultdict(list) + for c in self.challenges: + out[c.category or "?"].append(c) + for chs in out.values(): + chs.sort(key=lambda c: (-STATUS_ORDER.index(c.stage) if c.stage else 1, + difficulty_rank(c.difficulty), c.title.lower())) + return out -def collect(dav: Dav, root: str, no_color: bool) -> dict: - paths = [p for p in dav.walk(root) if p.lower().endswith(".md")] - authors: dict[str, Author] = {} - for p in sorted(paths): - parts = p.split("/") - if len(parts) < 2: - continue # loose file in the root, not an author's - adir, fname = parts[0], parts[-1] - a = authors.setdefault(adir, Author(adir)) - a.files_seen.append(fname) - if fname.lower() == "readme.md": - continue - text = dav.get(f"{root}/{p}") - a.challenges.append( - parse_challenge(adir, p, text, in_template=fname.lower() == "template.md")) - # authors with a folder but zero md files still deserve screaming at - for p in dav.walk(root): - top = p.split("/")[0] - if "/" in p: - authors.setdefault(top, Author(top)) - return authors +def collect(gl: GitLab) -> Overview: + chs, blank = [], [] + for repo in gl.repos(): + c = parse_challenge(repo) + (chs if c.touched else blank).append(c) + return Overview(chs, sorted(blank, key=lambda c: c.repo.path.lower())) -def print_terminal(authors: dict, c: bool) -> None: +def icons(chs: list) -> str: + return "".join(STATUS_ICON[c.stage] for c in chs) + + +def print_terminal(ov: Overview, c: bool) -> None: def col(x): return x if c else "" - order = {"SCREAM": 0, "NAG": 1, "OK": 2} - rows = sorted(authors.values(), key=lambda a: (order[a.verdict[0]], a.name.lower())) - print(f"\n{col(BOLD)}=== authors ({len(rows)}) ==={col(RST)}") - for a in rows: - v, why = a.verdict - tint = {"SCREAM": RED, "NAG": YEL, "OK": GRN}[v] - print(f" {col(tint)}{v:<7}{col(RST)} {a.name:<24} {col(DIM)}{why}{col(RST)}") - for ch in a.real: - for prob in ch.problems: - print(f" {col(DIM)}↳ {Path(ch.path).name}: {prob}{col(RST)}") + stages = ov.stages + groups = ov.by_category() - chs = [ch for a in authors.values() for ch in a.real] - done = sum(1 for ch in chs if ch.complete) - print(f"\n{col(BOLD)}=== {len(chs)} challenges, {done} fully filled out ==={col(RST)}") - by_cat = Counter(ch.get("Category", "?").lower() for ch in chs) - by_diff = Counter(ch.get("Difficulty", "?").lower() for ch in chs) - by_stat = Counter(ch.status for ch in chs) - for label, ctr, ref in (("category", by_cat, CATEGORIES), - ("difficulty", by_diff, DIFFICULTIES), - ("status", by_stat, STATUS_ORDER)): - keys = [k for k in ref if ctr.get(k.lower() if label != "status" else k)] - keys += [k for k in ctr if k not in keys and k not in [r.lower() for r in ref]] - parts = [f"{k}:{ctr[k]}" for k in keys if ctr.get(k)] - print(f" {col(CYA)}{label:<11}{col(RST)} " + " ".join(parts)) - missing = [k for k in CATEGORIES if not by_cat.get(k)] - if missing: - print(f" {col(YEL)}no coverage {col(RST)}" + ", ".join(missing)) + total = Counter(ch.stage for ch in ov.challenges) + print(f"\n{col(BOLD)}=== {len(ov.challenges)} challenges ==={col(RST)} " + + " ".join(f"{STATUS_ICON[s]} {s or '?'}:{total[s]}" for s in stages if total[s])) + + ready = ov.testable + print(f"\n{col(BOLD)}{col(GRN)}=== ready for infra testing ({len(ready)}) ==={col(RST)}") + for ch in ready: + print(f" {STATUS_ICON[ch.stage]} {ch.title[:34]:<35}{ch.category or '?':<10}" + f"{ch.difficulty or '?':<13}{col(DIM)}{ch.repo.web_url}{col(RST)}") + + print(f"\n{col(BOLD)}=== status by category ==={col(RST)}") + print(f" {'':<10}" + "".join(f"{s or '?':>14}" for s in stages) + f"{'total':>7}") + for cat in ov.categories: + chs = groups.get(cat, []) + cnt = Counter(ch.stage for ch in chs) + tint = "" if chs else DIM + print(f" {col(tint)}{cat:<10}" + "".join(f"{cnt[s] or '·':>14}" for s in stages) + + f"{len(chs):>7}{col(RST)}") + + print(f"\n{col(BOLD)}=== category × difficulty ==={col(RST)} " + + " ".join(f"{STATUS_ICON[s]} {s}" for s in STATUS_ORDER)) + diffs = ov.difficulties + print(f" {'':<10}" + "".join(f"{d:<13}" for d in diffs)) + for cat in ov.categories: + chs = groups.get(cat, []) + cells = [icons([ch for ch in chs if (ch.difficulty or "?") == d]) for d in diffs] + # emoji are two columns wide in a terminal + print(f" {cat:<10}" + "".join(f"{x or '·'}{' ' * (13 - (2 * len(x) if x else 1))}" + for x in cells)) + + print(f"\n{col(BOLD)}=== needs attention ==={col(RST)}") + for ch in sorted(ov.challenges, key=lambda c: c.repo.path.lower()): + if ch.problems: + print(f" {col(YEL)}{ch.repo.path:<32}{col(RST)} {col(DIM)}" + + "; ".join(ch.problems) + col(RST)) + for ch in ov.blank: + why = "no README" if ch.repo.readme is None else "README is still the blank template" + print(f" {col(RED)}{ch.repo.path:<32}{col(RST)} {col(DIM)}{why}{col(RST)}") print() -def render_hedgedoc(authors: dict, title: str, tags: str = "") -> str: - chs = [ch for a in authors.values() for ch in a.real] +def md_cell(v: str) -> str: + return v.replace("|", "\\|") + + +def render_hedgedoc(ov: Overview, title: str, tags: str = "") -> str: stamp = datetime.now().astimezone().strftime("%Y-%m-%d %H:%M:%S %Z") + stages = ov.stages + groups = ov.by_category() + total = Counter(ch.stage for ch in ov.challenges) L = [] if tags: # HedgeDoc reads this YAML block off the top of the note L += ["---", f"tags: {tags}", "---", ""] L += [f"# {title}", "", f"**Last updated:** {stamp}", "", - f"*generated by `ctf_status.py` — {len(chs)} challenges from " - f"{len({ch.author_dir for ch in chs})} authors*", "", - "## Challenges", "", - "| Challenge | Category | Difficulty | Event | Author | Tester | Status |", - "| --- | --- | --- | --- | --- | --- | --- |"] - for ch in sorted(chs, key=lambda c: (c.get("Category", "zz").lower(), - DIFFICULTIES.index(c.get("Difficulty", "").lower()) - if c.get("Difficulty", "").lower() in DIFFICULTIES else 9, - c.name.lower())): - name = ch.name if ch.named else f"*(unnamed — {Path(ch.path).name})*" - flag = "" if ch.complete else " ⚠️" - L.append(f"| {name}{flag} | {ch.get('Category')} | {ch.get('Difficulty')} | " - f"{ch.get('Event')} | {ch.get('Author') if not is_placeholder(ch.fields.get('Author','')) else ch.author_dir} | " - f"{ch.get('Tester')} | {ch.status} |") + f"*generated by `ctf_status.py` from the README.md of every repo in the " + f"GitLab challenges group — {len(ov.challenges)} challenges*", "", + " · ".join(f"{STATUS_ICON[s]} **{s or 'unknown'}** {total[s]}" + for s in stages if total[s] or s), ""] - cols = DIFFICULTIES + ["?"] - L += ["", "## Coverage (category × difficulty)", "", - "| Category | " + " | ".join(cols) + " | total |", - "| --- |" + " --- |" * (len(cols) + 1)] - grid = defaultdict(Counter) - for ch in chs: - grid[ch.get("Category", "?").lower()][ch.get("Difficulty", "?").lower()] += 1 - for cat in CATEGORIES + [k for k in grid if k not in CATEGORIES]: - row = grid.get(cat, Counter()) - cells = [str(row.get(d, 0) or "·") for d in cols] - L.append(f"| {cat} | " + " | ".join(cells) + f" | **{sum(row.values())}** |") - tot = Counter() - for row in grid.values(): - tot.update(row) - L.append("| **total** | " + " | ".join(f"**{tot.get(d, 0)}**" for d in cols) - + f" | **{sum(tot.values())}** |") + ready = ov.testable + L += [f"## ✅ Ready for infra testing ({len(ready)} of {len(ov.challenges)})", "", + "*every challenge at Testing Ready or beyond — these can be deployed and " + "tested against the infrastructure now*", ""] + if ready: + L += ["| | Challenge | Category | Difficulty | Status | Event | Author | Tester |", + "| --- | --- | --- | --- | --- | --- | --- | --- |"] + for ch in ready: + L.append("| " + " | ".join(md_cell(x) for x in ( + STATUS_ICON[ch.stage], f"[{ch.title}]({ch.repo.web_url})", + ch.category or "?", ch.difficulty or "?", ch.stage, ch.get("Event"), + ch.get("Author"), ch.get("Tester") if ch.has_tester else "—")) + " |") + else: + L.append("*nothing yet*") + L.append("") - L += ["", "## Author status", "", "| Author | State | Note |", "| --- | --- | --- |"] - icon = {"SCREAM": "🔴", "NAG": "🟡", "OK": "🟢"} - for a in sorted(authors.values(), key=lambda a: a.name.lower()): - v, why = a.verdict - L.append(f"| {a.name} | {icon[v]} | {why} |") - L += ["", "## Open issues", ""] - any_issue = False - for a in sorted(authors.values(), key=lambda a: a.name.lower()): - items = [(ch, p) for ch in a.real for p in ch.problems] - if not items and a.real: + L += ["## Status by category", "", + "| Category | " + " | ".join(f"{STATUS_ICON[s]} {s or '?'}" for s in stages) + + " | total |", + "| --- |" + " ---: |" * (len(stages) + 1)] + for cat in ov.categories: + chs = groups.get(cat, []) + cnt = Counter(ch.stage for ch in chs) + L.append(f"| {cat} | " + " | ".join(str(cnt[s] or "·") for s in stages) + + f" | **{len(chs)}** |") + L.append("| **total** | " + " | ".join(f"**{total[s]}**" for s in stages) + + f" | **{len(ov.challenges)}** |") + + diffs = ov.difficulties + L += ["", "## Category × difficulty", "", + "*one dot per challenge, coloured by status: " + + " ".join(f"{STATUS_ICON[s]} {s}" for s in STATUS_ORDER) + "*", "", + "| Category | " + " | ".join(diffs) + " | total |", + "| --- |" + " --- |" * len(diffs) + " ---: |"] + for cat in ov.categories: + chs = groups.get(cat, []) + cells = [icons([ch for ch in chs if (ch.difficulty or "?") == d]) or "·" for d in diffs] + L.append(f"| {cat} | " + " | ".join(cells) + f" | **{len(chs)}** |") + dtot = Counter(ch.difficulty or "?" for ch in ov.challenges) + L.append("| **total** | " + " | ".join(f"**{dtot[d]}**" for d in diffs) + + f" | **{len(ov.challenges)}** |") + + L += ["", "## Challenges"] + for cat in ov.categories: + chs = groups.get(cat) + if not chs: continue - any_issue = True - L.append(f"- **{a.name}**") - if not a.real: - L.append(" - nothing submitted at all") - for ch, p in items: - L.append(f" - `{Path(ch.path).name}`: {p}") - if not any_issue: - L.append("*none — everyone is done. suspicious.*") + L += ["", f"### {cat} ({len(chs)})", "", + "| | Challenge | Difficulty | Status | Event | Author | Tester | Last activity |", + "| --- | --- | --- | --- | --- | --- | --- | --- |"] + for ch in chs: + status = ch.stage or "?" + if ch.status_raw.lower() != status.lower(): + status += f" *({ch.status_raw})*" + cat_raw = ch.get("Category", "") + name = f"[{md_cell(ch.title)}]({ch.repo.web_url})" + if cat_raw.lower() != cat: + name += f" *({cat_raw})*" + L.append("| " + " | ".join(md_cell(x) for x in ( + STATUS_ICON[ch.stage], name, ch.difficulty or ch.get("Difficulty"), status, + ch.get("Event"), ch.get("Author"), + ch.get("Tester") if ch.has_tester else "—", ch.repo.last_activity)) + " |") + + L += ["", "## Needs attention", ""] + issues = [ch for ch in sorted(ov.challenges, key=lambda c: c.repo.path.lower()) if ch.problems] + for ch in issues: + L.append(f"- [{md_cell(ch.title)}]({ch.repo.web_url}): " + "; ".join(ch.problems)) + for ch in ov.blank: + why = "no README" if ch.repo.readme is None else "README is still the blank template" + L.append(f"- [{ch.repo.path}]({ch.repo.web_url}): {why}") + if not issues and not ov.blank: + L.append("*nothing — every README is filled in. suspicious.*") return "\n".join(L) + "\n" @@ -584,23 +688,26 @@ def main() -> None: if not note_url: sys.exit("--update needs a note URL, or HEDGEDOC_NOTE in the config") try: - HedgeDocNote(note_url) # fail on a typo now, not after the Nextcloud walk + HedgeDocNote(note_url) # fail on a typo now, not after the GitLab walk except ValueError as e: sys.exit(str(e)) lock = single_run_lock() # noqa: F841 (held for the lifetime of the process) - dav = Dav(cfg["NC_URL"], cfg["NC_USER"], cfg["NC_PASS"]) - authors = collect(dav, cfg["NC_ROOT"].strip("/"), args.no_color) - if not authors: - sys.exit(f"no author folders found under {cfg['NC_ROOT']!r}") + gl = GitLab(cfg["GITLAB_GROUP_URL"], cfg["GITLAB_TOKEN"]) + try: + ov = collect(gl) + except (requests.RequestException, RuntimeError) as e: + sys.exit(f"reading {cfg['GITLAB_GROUP_URL']} failed: {e}") + if not ov.challenges and not ov.blank: + sys.exit(f"no repos found in {cfg['GITLAB_GROUP_URL']!r}") - md = render_hedgedoc(authors, title, tags) + md = render_hedgedoc(ov, title, tags) if args.out: Path(args.out).write_text(md) if args.stdout: print(md, end="") elif not args.quiet: - print_terminal(authors, c=not args.no_color and sys.stdout.isatty()) + print_terminal(ov, c=not args.no_color and sys.stdout.isatty()) if args.out: print(f"markdown → {args.out}") if args.hedgedoc: