diff --git a/.gitignore b/.gitignore index 276e14a..3d47cce 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ __pycache__/ ctf_status.env *.md.out +.env diff --git a/README.md b/README.md index 5f4cd79..830f09b 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,14 @@ # ctf_status.py -Reads the per-author challenge folders straight off the Nextcloud share (WebDAV, -no sync, no mount) and tells you who to scream at. +Reads the `README.md` of every repo in the GitLab challenges group (REST API, no +clone) and shows which categories have challenges in which state, at which +difficulty — in the terminal and as a HedgeDoc note. ## Setup ```sh cp ctf_status.env.example ctf_status.env && chmod 600 ctf_status.env -$EDITOR ctf_status.env # app password from Nextcloud → Settings → Security +$EDITOR ctf_status.env # GitLab group URL + token, HedgeDoc note ``` ## Use @@ -20,6 +21,10 @@ $EDITOR ctf_status.env # app password from Nextcloud → Settings → Secur ./ctf_status.py --update # ... or an explicit one ``` +The token is a fine-grained personal access token on the challenges group with +**Project: Read** (to list the repos) and **Repository: Read** (to download the +READMEs). A plain `.env` with `GL_PAT=` / `GITLAB_URL=` is read as well. + ## Keeping a HedgeDoc note up to date `--update` overwrites an **existing** note in place, so the URL you share stays the @@ -78,33 +83,51 @@ changed or something broke. Overlapping runs are prevented by a lock file, and t HedgeDoc write is retried (`--retries`, default 3) before the run gives up with a one-line error and a non-zero exit. +## What the note shows + +1. **✅ Ready for infra testing** — first thing in the note: every challenge at + Testing Ready, Tested or Ready, linked to its repo, so infra knows what it + can deploy and test against right now. +2. **Status by category** — how many challenges each category has per stage + (⚪ Planning → 🟠 PoC Draft → 🟡 Testing Ready → 🔵 Tested → 🟢 Ready). +3. **Category × difficulty** — one dot per challenge, coloured by its status, so + gaps and stragglers show up at a glance. +4. **Challenges** — one table per category, linked to the repo, with event, + author, tester and the repo's last activity. +5. **Needs attention** — see below. + +## Reading the README + +Only the template's header block is parsed: + +``` +### Challenge Name +- Category: crypto +- Difficulty: medium +- Event: TUM student CTF +- Author: ... +- Tester: ... +- Status: Testing Ready +``` + +People fill it in loosely, so values are normalised: + +- **Status** is matched against the five stages ("Tested with …" → Tested, + "PoC" → PoC Draft). If it names several ("Planning to PoC Draft", a half-deleted + option list) the *least* advanced one counts and it is flagged as unclear. +- **Difficulty** ranges written as `medium-hard`, `medium/hard` or + `easy to medium` become their own column; otherwise the first level named wins. +- **Category**: the first one named counts ("rev, misc" → rev); the full value + is shown next to the challenge. Categories outside the usual list (e.g. `ai`) + get their own row. + ## What it flags -| verdict | meaning | -| --- | --- | -| 🔴 SCREAM | nothing submitted, **or** they filled out `template.md` in place | -| 🟡 NAG | submitted, but placeholders / empty sections remain | -| 🟢 OK | every challenge fully filled out | +- challenge at Testing Ready or beyond with no tester (`[Tester Name]`, `??`, + `None`, `not found yet` all count as none) +- status / difficulty / category missing or unclear +- no challenge name in the README +- repos whose README is missing or still the untouched template — these are left + out of the counts -A `template.md` that is still pristine is ignored. One that has real content in it -is counted as a challenge *and* flagged — so nobody's work is lost, they just get -yelled at. - -Also flagged per challenge: unnamed challenge, `[placeholder]` values left in any -of Category/Difficulty/Event/Author/Tester/Status, empty prose sections, and -category/difficulty values outside the allowed set. - -## Layout it expects - -``` -/ - alice/ - README.md # ignored - template.md # ignored while pristine - baby-rsa.md # ← a challenge, any filename - bob/ -``` - -Any `.md` that isn't `README.md` is parsed as a challenge, so the file naming -convention doesn't have to be enforced by the script. If you want it enforced, -add the check in `Challenge.problems`. +Repos GitLab has scheduled for deletion, and archived repos, are skipped. diff --git a/ctf-status.service b/ctf-status.service index 93d5eb4..ba1c07d 100644 --- a/ctf-status.service +++ b/ctf-status.service @@ -8,5 +8,5 @@ Wants=network-online.target Type=oneshot WorkingDirectory=__DIR__ ExecStart=__DIR__/ctf_status.py --update --quiet -# Nextcloud walk + HedgeDoc write; never let one run squat on the next timer slot +# GitLab walk + HedgeDoc write; never let one run squat on the next timer slot TimeoutStartSec=5min diff --git a/ctf-status.timer b/ctf-status.timer index a05bf50..3746fa5 100644 --- a/ctf-status.timer +++ b/ctf-status.timer @@ -3,7 +3,7 @@ Description=Refresh the CTF challenge status every 30 minutes [Timer] OnCalendar=*:0/30 -# don't hammer Nextcloud and HedgeDoc on the exact half hour +# don't hammer GitLab and HedgeDoc on the exact half hour RandomizedDelaySec=60 AccuracySec=30s # catch up on the last run if the machine was down when it was due diff --git a/ctf_status.env.example b/ctf_status.env.example index e9b92ba..6155ea7 100644 --- a/ctf_status.env.example +++ b/ctf_status.env.example @@ -1,10 +1,11 @@ -# cp to ctf_status.env and chmod 600 -NC_URL=https://cloud.h4tum.de -NC_USER=your-nextcloud-login -# Settings -> Security -> "Create new app password" (NOT your real password) -NC_PASS=xxxxx-xxxxx-xxxxx-xxxxx-xxxxx -# path of the folder holding the per-author folders, relative to your Nextcloud root -NC_ROOT=CTF/challenges +# cp to ctf_status.env and chmod 600 (a plain .env next to the script works too) + +# --- GitLab: the group holding one repo per challenge ----------------------- +GITLAB_GROUP_URL=https://gitlab.lrz.de/h4tum-ctf/h4tum-ctf-2026/challenges +# fine-grained personal access token, scoped to that group, with +# Project: Read (list the repos) +# Repository: Read (download each README.md) +GITLAB_TOKEN=glpat-xxxxxxxxxxxxxxxxxxxx # --- HedgeDoc: the existing note that gets overwritten every run ------------ # full URL of the note (its permission must be "freely" so an anonymous diff --git a/ctf_status.py b/ctf_status.py index a001cec..7387bed 100755 --- a/ctf_status.py +++ b/ctf_status.py @@ -1,9 +1,9 @@ #!/usr/bin/env python3 """CTF challenge tracker. -Reads the per-author challenge folders from a Nextcloud share over WebDAV, -parses the filled-out challenge templates and reports: - * a terminal "who do I have to scream at" list +Lists every repo in the GitLab challenges group, reads each repo's README.md +(the filled-out challenge template) and reports: + * a terminal summary: status per category, category x difficulty, open issues * a HedgeDoc-ready markdown overview Config: see ctf_status.env.example -> ctf_status.env (chmod 600) @@ -20,125 +20,136 @@ import sys import tempfile import time from collections import Counter, defaultdict +from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass, field from datetime import datetime from pathlib import Path -from urllib.parse import quote, unquote, urljoin, urlsplit -from xml.etree import ElementTree as ET +from urllib.parse import quote, urlsplit import requests # ---------------------------------------------------------------- config ---- -CFG_KEYS = ("NC_URL", "NC_USER", "NC_PASS", "NC_ROOT") +CFG_KEYS = ("GITLAB_GROUP_URL", "GITLAB_TOKEN") OPT_KEYS = ("HEDGEDOC_NOTE", "HEDGEDOC_TAGS", "TITLE") +# the names a plain `.env` tends to use for the same thing +ALIASES = {"GL_PAT": "GITLAB_TOKEN", "GITLAB_URL": "GITLAB_GROUP_URL"} DEFAULT_TAGS = "Logistics, h4tum CTF 2026" DEFAULT_TITLE = "CTF challenge tracking" def load_env() -> dict: cfg = {} - for candidate in ( - Path(__file__).with_name("ctf_status.env"), - Path.home() / ".config" / "ctf-status" / "env", - ): - if candidate.exists(): - for line in candidate.read_text().splitlines(): - line = line.strip() - if not line or line.startswith("#") or "=" not in line: - continue - k, v = line.split("=", 1) - cfg[k.strip()] = v.strip().strip("'\"") - break - for k in CFG_KEYS + OPT_KEYS: + here = Path(__file__).parent + # every file that exists contributes; the first one to set a key wins + for candidate in (here / "ctf_status.env", here / ".env", + Path.home() / ".config" / "ctf-status" / "env"): + if not candidate.exists(): + continue + for line in candidate.read_text().splitlines(): + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + k, v = line.split("=", 1) + k = k.strip().upper() + cfg.setdefault(ALIASES.get(k, k), v.strip().strip("'\"")) + for k in CFG_KEYS + OPT_KEYS + tuple(ALIASES): if os.environ.get(k): - cfg[k] = os.environ[k] + cfg[ALIASES.get(k, k)] = os.environ[k] missing = [k for k in CFG_KEYS if not cfg.get(k)] if missing: sys.exit(f"missing config: {', '.join(missing)} (see ctf_status.env.example)") return cfg -# --------------------------------------------------------------- webdav ----- +# --------------------------------------------------------------- gitlab ----- -DAV_NS = "{DAV:}" +@dataclass +class Repo: + path: str + web_url: str + last_activity: str # YYYY-MM-DD + readme: str | None # None: the repo has no README at all -class Dav: - def __init__(self, base_url: str, user: str, password: str): - self.base = base_url.rstrip("/") + f"/remote.php/dav/files/{quote(user)}/" +class GitLab: + """Read-only access to one group. The token needs Project: Read (to list the + group) and Repository: Read (to download files).""" + + def __init__(self, group_url: str, token: str, timeout: int = 20): + parts = urlsplit(group_url) + self.api = f"{parts.scheme}://{parts.netloc}/api/v4" + self.group = parts.path.strip("/") + self.timeout = timeout self.s = requests.Session() - self.s.auth = (user, password) + self.s.headers["PRIVATE-TOKEN"] = token - def _url(self, path: str) -> str: - return urljoin(self.base, quote(path.strip("/")) + "/") - - def walk(self, root: str) -> list[str]: - """Return all file paths (relative to root) under root.""" - body = ( - '' - "" - ) - out: list[str] = [] - for depth in ("infinity", "1"): + def _get(self, url: str, **params) -> requests.Response: + r = self.s.get(url, params=params, timeout=self.timeout) + if r.status_code == 403: + # fine-grained tokens say exactly which permission is missing try: - out = self._propfind(root, body, depth) - if out: - return out - except requests.HTTPError as e: - if depth == "infinity" and e.response is not None and e.response.status_code in (403, 405, 507): - continue # server forbids Depth: infinity -> recurse manually - raise - return out - - def _propfind(self, root: str, body: str, depth: str) -> list[str]: - r = self.s.request( - "PROPFIND", self._url(root), - headers={"Depth": depth, "Content-Type": "application/xml"}, data=body, - ) + why = r.json().get("error_description") or r.text + except ValueError: + why = r.text + raise RuntimeError(f"gitlab refused {r.url}: {why}") r.raise_for_status() - tree = ET.fromstring(r.content) - base_path = unquote(self._url(root).split(self.base.split("/remote.php")[0], 1)[-1]) - files, dirs = [], [] - for resp in tree.findall(f"{DAV_NS}response"): - href = unquote(resp.findtext(f"{DAV_NS}href") or "") - if not href or href.rstrip("/") == base_path.rstrip("/"): - continue - rel = href.split(base_path, 1)[-1].strip("/") if base_path in href else href - is_dir = resp.find(f".//{DAV_NS}collection") is not None - (dirs if is_dir else files).append(rel) - if depth == "1": - for d in dirs: - files += [f"{d}/{f}" for f in self._propfind(f"{root}/{d}", body, "1")] - return files + return r - def get(self, path: str) -> str: - r = self.s.get(self._url(path).rstrip("/")) - r.raise_for_status() + def projects(self) -> list[dict]: + out, page = [], "1" + while page: + r = self._get(f"{self.api}/groups/{quote(self.group, safe='')}/projects", + include_subgroups="true", per_page="100", page=page, + order_by="path", sort="asc") + out += r.json() + page = r.headers.get("X-Next-Page", "") + # repos GitLab is about to delete are renamed "...-deletion_scheduled-" + return [p for p in out if not p.get("archived") and not p.get("marked_for_deletion_on")] + + def readme(self, p: dict) -> str | None: + branch, url = p.get("default_branch"), p.get("readme_url") + if not branch or not url or f"/-/blob/{branch}/" not in url: + return None + path = url.split(f"/-/blob/{branch}/", 1)[1] + r = self._get(f"{self.api}/projects/{p['id']}/repository/files/" + f"{quote(path, safe='')}/raw", ref=branch) return r.content.decode("utf-8", "replace") + def repos(self) -> list[Repo]: + projects = self.projects() + + def fetch(p: dict) -> Repo: + return Repo(path=p["path"], web_url=p["web_url"], + last_activity=(p.get("last_activity_at") or "")[:10], + readme=self.readme(p)) + + with ThreadPoolExecutor(max_workers=8) as pool: + return list(pool.map(fetch, projects)) + # --------------------------------------------------------------- parsing ---- FIELDS = ("Category", "Difficulty", "Event", "Author", "Tester", "Status") -HEADING_RE = re.compile(r"^\s{0,3}#{2,4}\s+(.+?)\s*$", re.M) +HEADING_RE = re.compile(r"^\s{0,3}#{1,4}\s+(.+?)\s*$", re.M) FIELD_RE = re.compile(r"^\s*[-*]\s*\**(?P[A-Za-z ]+?)\**\s*:\s*(?P.*?)\s*$", re.M) -SECTION_RE = re.compile(r"^\s{0,3}#{3,5}\s*(?P.+?)\s*$\n(?P.*?)(?=^\s{0,3}#{2,5}\s|\Z)", - re.M | re.S) # a value the author never touched: "[crypto, pwn, ...]" or "[Author Name]" -PLACEHOLDER_RE = re.compile(r"^\\?\[.*\\?\]$") -PROSE_PROMPTS = ( - "what should the player take home", "please answer in 1-2", - "how do you plan on implementing", "needed for infrastructure", - "how will the players exploit", -) +PLACEHOLDER_RE = re.compile(r"^\[.*\]$") +NO_TESTER_RE = re.compile(r"^(\?+|none|tester name|not found\b.*|tbd)(\s.*)?$", re.I) + STATUS_ORDER = ["Planning", "PoC Draft", "Testing Ready", "Tested", "Ready"] +STATUS_RE = re.compile(r"testing\s*ready|tested|ready|poc(?:\s*draft)?|planning", re.I) +STATUS_ICON = {"Planning": "⚪", "PoC Draft": "🟠", "Testing Ready": "🟡", + "Tested": "🔵", "Ready": "🟢", None: "❔"} CATEGORIES = ["crypto", "pwn", "web", "rev", "misc", "osint", "forensic", "hardware"] DIFFICULTIES = ["intro", "easy", "medium", "hard"] +DIFF_RE = re.compile(r"intro|easy|medium|hard", re.I) +DIFF_RANGE_RE = re.compile(r"(intro|easy|medium|hard)\s*(?:-|–|/|\bto\b)\s*(intro|easy|medium|hard)", re.I) def clean(v: str) -> str: - return v.replace("\\[", "[").replace("\\]", "]").strip() + v = re.sub(r"\\+([\[\]_*])", r"\1", v) # markdown escapes, sometimes doubled + return v.replace("**", "").strip() def is_placeholder(v: str) -> bool: @@ -146,68 +157,120 @@ def is_placeholder(v: str) -> bool: return not v or bool(PLACEHOLDER_RE.match(v)) +def stage_of(raw: str) -> tuple[str | None, bool]: + """Map a free-text status onto STATUS_ORDER -> (stage, clear). + + "Tested with h4tum-chall-format" is clearly Tested. "Planning to PoC Draft" + or a half-deleted option list is not; we take the least advanced stage it + names, so nothing looks further along than it is, and call it unclear.""" + if is_placeholder(raw): + return None, False + found = set() + for m in STATUS_RE.finditer(raw): + w = m.group(0).lower() + found.add("Testing Ready" if w.startswith("testing") else + "PoC Draft" if w.startswith("poc") else w.title()) + if not found: + return None, False + return min(found, key=STATUS_ORDER.index), len(found) == 1 + + +def difficulty_of(raw: str) -> str | None: + """"medium/hard", "medium-hard", "easy to medium" -> a range "medium-hard".""" + if is_placeholder(raw): + return None + m = DIFF_RANGE_RE.search(raw) + if m: + ends = sorted({m.group(1).lower(), m.group(2).lower()}, key=DIFFICULTIES.index) + return "-".join(ends) if len(ends) > 1 else ends[0] + # prose like "currently medium, maybe also an easy and a hard version": the first one + m = DIFF_RE.search(raw) + return m.group(0).lower() if m else None + + +def difficulty_rank(d: str | None) -> float: + if not d: + return 99 + ends = d.split("-") + return sum(DIFFICULTIES.index(e) for e in ends) / len(ends) + + @dataclass class Challenge: - author_dir: str - path: str + repo: Repo name: str = "" fields: dict = field(default_factory=dict) - sections: dict = field(default_factory=dict) - in_template: bool = False - - # -- derived ----------------------------------------------------------- - @property - def missing_fields(self) -> list[str]: - return [f for f in FIELDS if is_placeholder(self.fields.get(f, ""))] - - @property - def missing_sections(self) -> list[str]: - return [k for k, v in self.sections.items() if not v] - - @property - def named(self) -> bool: - return bool(self.name) and not is_placeholder(self.name) - - @property - def touched(self) -> bool: - """Did anyone put *any* real content in here?""" - return self.named or len(self.missing_fields) < len(FIELDS) or bool( - [k for k, v in self.sections.items() if v]) - - @property - def complete(self) -> bool: - return self.named and not self.missing_fields and not self.missing_sections def get(self, key: str, dash: str = "—") -> str: v = clean(self.fields.get(key, "")) return dash if is_placeholder(v) else v + # -- derived ----------------------------------------------------------- @property - def status(self) -> str: - return self.get("Status", "—") + def named(self) -> bool: + return bool(self.name) and not is_placeholder(self.name) + + @property + def title(self) -> str: + return self.name if self.named else self.repo.path + + @property + def touched(self) -> bool: + """Did anyone put *any* real content in the README template?""" + return self.named or any(not is_placeholder(self.fields.get(f, "")) for f in FIELDS) + + @property + def status_raw(self) -> str: + return self.get("Status") + + @property + def stage(self) -> str | None: + return stage_of(self.fields.get("Status", ""))[0] + + @property + def difficulty(self) -> str | None: + return difficulty_of(self.fields.get("Difficulty", "")) + + @property + def category(self) -> str | None: + """The first category named; "rev, misc" counts as rev.""" + raw = self.get("Category", "") + first = re.split(r"[,/&+]| and ", raw)[0].strip().lower() + return first or None + + @property + def has_tester(self) -> bool: + t = self.get("Tester", "") + return bool(t) and not NO_TESTER_RE.match(t) @property def problems(self) -> list[str]: p = [] - if self.in_template and self.touched: - p.append("filled out template.md in place (rename it!)") + stage, clear = stage_of(self.fields.get("Status", "")) + if stage is None: + p.append(f"status unknown ({self.status_raw!r})" if self.status_raw != "—" + else "no status") + elif not clear: + p.append(f"status unclear ({self.status_raw!r}, counted as {stage})") + if not self.category: + p.append("no category") + if not self.difficulty: + p.append(f"difficulty unclear ({self.get('Difficulty')!r})" + if self.get("Difficulty") != "—" else "no difficulty") + if is_testable(self) and not self.has_tester: + p.append("no tester") if not self.named: - p.append("no challenge name") - if self.missing_fields: - p.append("unfilled: " + ", ".join(self.missing_fields)) - if self.missing_sections: - p.append("empty section(s): " + ", ".join(self.missing_sections)) - cat = self.get("Category", "").lower() - if cat and cat != "—" and cat not in CATEGORIES and "?" not in cat: - p.append(f"odd category {cat!r}") - diff = self.get("Difficulty", "").lower() - if diff and diff != "—" and diff not in DIFFICULTIES: - p.append(f"odd difficulty {diff!r}") + p.append("no challenge name in README") return p -def parse_challenge(author: str, path: str, text: str, in_template: bool) -> Challenge: - c = Challenge(author_dir=author, path=path, in_template=in_template) +def is_testable(c: Challenge) -> bool: + return bool(c.stage) and STATUS_ORDER.index(c.stage) >= STATUS_ORDER.index("Testing Ready") + + +def parse_challenge(repo: Repo) -> Challenge: + c = Challenge(repo=repo) + text = repo.readme or "" m = HEADING_RE.search(text) if m: c.name = clean(m.group(1)) @@ -215,16 +278,6 @@ def parse_challenge(author: str, path: str, text: str, in_template: bool) -> Cha key = fm.group("key").strip().title() if key in FIELDS and key not in c.fields: c.fields[key] = fm.group("val") - for sm in SECTION_RE.finditer(text): - name = clean(sm.group("name")) - if name == c.name or name in FIELDS: - continue - body = sm.group("body") - # strip the template's own prompt lines, keep what the author wrote - real = [ln.strip() for ln in body.splitlines() - if ln.strip() and not ln.strip().startswith(("-", "*", "#")) - and not any(p in ln.lower() for p in PROSE_PROMPTS)] - c.sections[name] = " ".join(real) return c @@ -235,142 +288,193 @@ BOLD, DIM, RED, YEL, GRN, CYA, RST = ( @dataclass -class Author: - name: str - challenges: list = field(default_factory=list) - files_seen: list = field(default_factory=list) +class Overview: + challenges: list # repos with a README somebody filled in + blank: list # repos with no README, or an untouched template @property - def real(self) -> list: - return [c for c in self.challenges if c.touched] + def categories(self) -> list[str]: + extra = sorted({c.category or "?" for c in self.challenges} - set(CATEGORIES)) + return CATEGORIES + extra @property - def verdict(self) -> tuple[str, str]: - if not self.real: - return "SCREAM", "nothing submitted at all" - if any(c.in_template for c in self.real): - return "SCREAM", "worked inside template.md instead of copying it" - bad = [c for c in self.real if not c.complete] - if bad: - return "NAG", f"{len(bad)}/{len(self.real)} challenge(s) incomplete" - return "OK", f"{len(self.real)} challenge(s) done" + def difficulties(self) -> list[str]: + seen = {c.difficulty for c in self.challenges if c.difficulty} + unknown = ["?"] if any(not c.difficulty for c in self.challenges) else [] + return sorted(set(DIFFICULTIES) | seen, key=difficulty_rank) + unknown + + @property + def stages(self) -> list: + """The five stages, plus None ("?") only if some status is unreadable.""" + return STATUS_ORDER + ([None] if any(not c.stage for c in self.challenges) else []) + + @property + def testable(self) -> list: + """Everything infra can already deploy and test: Testing Ready or beyond.""" + return sorted((c for c in self.challenges if is_testable(c)), + key=lambda c: (self.categories.index(c.category or "?"), + difficulty_rank(c.difficulty), c.title.lower())) + + def by_category(self) -> dict[str, list]: + out = defaultdict(list) + for c in self.challenges: + out[c.category or "?"].append(c) + for chs in out.values(): + chs.sort(key=lambda c: (-STATUS_ORDER.index(c.stage) if c.stage else 1, + difficulty_rank(c.difficulty), c.title.lower())) + return out -def collect(dav: Dav, root: str, no_color: bool) -> dict: - paths = [p for p in dav.walk(root) if p.lower().endswith(".md")] - authors: dict[str, Author] = {} - for p in sorted(paths): - parts = p.split("/") - if len(parts) < 2: - continue # loose file in the root, not an author's - adir, fname = parts[0], parts[-1] - a = authors.setdefault(adir, Author(adir)) - a.files_seen.append(fname) - if fname.lower() == "readme.md": - continue - text = dav.get(f"{root}/{p}") - a.challenges.append( - parse_challenge(adir, p, text, in_template=fname.lower() == "template.md")) - # authors with a folder but zero md files still deserve screaming at - for p in dav.walk(root): - top = p.split("/")[0] - if "/" in p: - authors.setdefault(top, Author(top)) - return authors +def collect(gl: GitLab) -> Overview: + chs, blank = [], [] + for repo in gl.repos(): + c = parse_challenge(repo) + (chs if c.touched else blank).append(c) + return Overview(chs, sorted(blank, key=lambda c: c.repo.path.lower())) -def print_terminal(authors: dict, c: bool) -> None: +def icons(chs: list) -> str: + return "".join(STATUS_ICON[c.stage] for c in chs) + + +def print_terminal(ov: Overview, c: bool) -> None: def col(x): return x if c else "" - order = {"SCREAM": 0, "NAG": 1, "OK": 2} - rows = sorted(authors.values(), key=lambda a: (order[a.verdict[0]], a.name.lower())) - print(f"\n{col(BOLD)}=== authors ({len(rows)}) ==={col(RST)}") - for a in rows: - v, why = a.verdict - tint = {"SCREAM": RED, "NAG": YEL, "OK": GRN}[v] - print(f" {col(tint)}{v:<7}{col(RST)} {a.name:<24} {col(DIM)}{why}{col(RST)}") - for ch in a.real: - for prob in ch.problems: - print(f" {col(DIM)}↳ {Path(ch.path).name}: {prob}{col(RST)}") + stages = ov.stages + groups = ov.by_category() - chs = [ch for a in authors.values() for ch in a.real] - done = sum(1 for ch in chs if ch.complete) - print(f"\n{col(BOLD)}=== {len(chs)} challenges, {done} fully filled out ==={col(RST)}") - by_cat = Counter(ch.get("Category", "?").lower() for ch in chs) - by_diff = Counter(ch.get("Difficulty", "?").lower() for ch in chs) - by_stat = Counter(ch.status for ch in chs) - for label, ctr, ref in (("category", by_cat, CATEGORIES), - ("difficulty", by_diff, DIFFICULTIES), - ("status", by_stat, STATUS_ORDER)): - keys = [k for k in ref if ctr.get(k.lower() if label != "status" else k)] - keys += [k for k in ctr if k not in keys and k not in [r.lower() for r in ref]] - parts = [f"{k}:{ctr[k]}" for k in keys if ctr.get(k)] - print(f" {col(CYA)}{label:<11}{col(RST)} " + " ".join(parts)) - missing = [k for k in CATEGORIES if not by_cat.get(k)] - if missing: - print(f" {col(YEL)}no coverage {col(RST)}" + ", ".join(missing)) + total = Counter(ch.stage for ch in ov.challenges) + print(f"\n{col(BOLD)}=== {len(ov.challenges)} challenges ==={col(RST)} " + + " ".join(f"{STATUS_ICON[s]} {s or '?'}:{total[s]}" for s in stages if total[s])) + + ready = ov.testable + print(f"\n{col(BOLD)}{col(GRN)}=== ready for infra testing ({len(ready)}) ==={col(RST)}") + for ch in ready: + print(f" {STATUS_ICON[ch.stage]} {ch.title[:34]:<35}{ch.category or '?':<10}" + f"{ch.difficulty or '?':<13}{col(DIM)}{ch.repo.web_url}{col(RST)}") + + print(f"\n{col(BOLD)}=== status by category ==={col(RST)}") + print(f" {'':<10}" + "".join(f"{s or '?':>14}" for s in stages) + f"{'total':>7}") + for cat in ov.categories: + chs = groups.get(cat, []) + cnt = Counter(ch.stage for ch in chs) + tint = "" if chs else DIM + print(f" {col(tint)}{cat:<10}" + "".join(f"{cnt[s] or '·':>14}" for s in stages) + + f"{len(chs):>7}{col(RST)}") + + print(f"\n{col(BOLD)}=== category × difficulty ==={col(RST)} " + + " ".join(f"{STATUS_ICON[s]} {s}" for s in STATUS_ORDER)) + diffs = ov.difficulties + print(f" {'':<10}" + "".join(f"{d:<13}" for d in diffs)) + for cat in ov.categories: + chs = groups.get(cat, []) + cells = [icons([ch for ch in chs if (ch.difficulty or "?") == d]) for d in diffs] + # emoji are two columns wide in a terminal + print(f" {cat:<10}" + "".join(f"{x or '·'}{' ' * (13 - (2 * len(x) if x else 1))}" + for x in cells)) + + print(f"\n{col(BOLD)}=== needs attention ==={col(RST)}") + for ch in sorted(ov.challenges, key=lambda c: c.repo.path.lower()): + if ch.problems: + print(f" {col(YEL)}{ch.repo.path:<32}{col(RST)} {col(DIM)}" + + "; ".join(ch.problems) + col(RST)) + for ch in ov.blank: + why = "no README" if ch.repo.readme is None else "README is still the blank template" + print(f" {col(RED)}{ch.repo.path:<32}{col(RST)} {col(DIM)}{why}{col(RST)}") print() -def render_hedgedoc(authors: dict, title: str, tags: str = "") -> str: - chs = [ch for a in authors.values() for ch in a.real] +def md_cell(v: str) -> str: + return v.replace("|", "\\|") + + +def render_hedgedoc(ov: Overview, title: str, tags: str = "") -> str: stamp = datetime.now().astimezone().strftime("%Y-%m-%d %H:%M:%S %Z") + stages = ov.stages + groups = ov.by_category() + total = Counter(ch.stage for ch in ov.challenges) L = [] if tags: # HedgeDoc reads this YAML block off the top of the note L += ["---", f"tags: {tags}", "---", ""] L += [f"# {title}", "", f"**Last updated:** {stamp}", "", - f"*generated by `ctf_status.py` — {len(chs)} challenges from " - f"{len({ch.author_dir for ch in chs})} authors*", "", - "## Challenges", "", - "| Challenge | Category | Difficulty | Event | Author | Tester | Status |", - "| --- | --- | --- | --- | --- | --- | --- |"] - for ch in sorted(chs, key=lambda c: (c.get("Category", "zz").lower(), - DIFFICULTIES.index(c.get("Difficulty", "").lower()) - if c.get("Difficulty", "").lower() in DIFFICULTIES else 9, - c.name.lower())): - name = ch.name if ch.named else f"*(unnamed — {Path(ch.path).name})*" - flag = "" if ch.complete else " ⚠️" - L.append(f"| {name}{flag} | {ch.get('Category')} | {ch.get('Difficulty')} | " - f"{ch.get('Event')} | {ch.get('Author') if not is_placeholder(ch.fields.get('Author','')) else ch.author_dir} | " - f"{ch.get('Tester')} | {ch.status} |") + f"*generated by `ctf_status.py` from the README.md of every repo in the " + f"GitLab challenges group — {len(ov.challenges)} challenges*", "", + " · ".join(f"{STATUS_ICON[s]} **{s or 'unknown'}** {total[s]}" + for s in stages if total[s] or s), ""] - cols = DIFFICULTIES + ["?"] - L += ["", "## Coverage (category × difficulty)", "", - "| Category | " + " | ".join(cols) + " | total |", - "| --- |" + " --- |" * (len(cols) + 1)] - grid = defaultdict(Counter) - for ch in chs: - grid[ch.get("Category", "?").lower()][ch.get("Difficulty", "?").lower()] += 1 - for cat in CATEGORIES + [k for k in grid if k not in CATEGORIES]: - row = grid.get(cat, Counter()) - cells = [str(row.get(d, 0) or "·") for d in cols] - L.append(f"| {cat} | " + " | ".join(cells) + f" | **{sum(row.values())}** |") - tot = Counter() - for row in grid.values(): - tot.update(row) - L.append("| **total** | " + " | ".join(f"**{tot.get(d, 0)}**" for d in cols) - + f" | **{sum(tot.values())}** |") + ready = ov.testable + L += [f"## ✅ Ready for infra testing ({len(ready)} of {len(ov.challenges)})", "", + "*every challenge at Testing Ready or beyond — these can be deployed and " + "tested against the infrastructure now*", ""] + if ready: + L += ["| | Challenge | Category | Difficulty | Status | Event | Author | Tester |", + "| --- | --- | --- | --- | --- | --- | --- | --- |"] + for ch in ready: + L.append("| " + " | ".join(md_cell(x) for x in ( + STATUS_ICON[ch.stage], f"[{ch.title}]({ch.repo.web_url})", + ch.category or "?", ch.difficulty or "?", ch.stage, ch.get("Event"), + ch.get("Author"), ch.get("Tester") if ch.has_tester else "—")) + " |") + else: + L.append("*nothing yet*") + L.append("") - L += ["", "## Author status", "", "| Author | State | Note |", "| --- | --- | --- |"] - icon = {"SCREAM": "🔴", "NAG": "🟡", "OK": "🟢"} - for a in sorted(authors.values(), key=lambda a: a.name.lower()): - v, why = a.verdict - L.append(f"| {a.name} | {icon[v]} | {why} |") - L += ["", "## Open issues", ""] - any_issue = False - for a in sorted(authors.values(), key=lambda a: a.name.lower()): - items = [(ch, p) for ch in a.real for p in ch.problems] - if not items and a.real: + L += ["## Status by category", "", + "| Category | " + " | ".join(f"{STATUS_ICON[s]} {s or '?'}" for s in stages) + + " | total |", + "| --- |" + " ---: |" * (len(stages) + 1)] + for cat in ov.categories: + chs = groups.get(cat, []) + cnt = Counter(ch.stage for ch in chs) + L.append(f"| {cat} | " + " | ".join(str(cnt[s] or "·") for s in stages) + + f" | **{len(chs)}** |") + L.append("| **total** | " + " | ".join(f"**{total[s]}**" for s in stages) + + f" | **{len(ov.challenges)}** |") + + diffs = ov.difficulties + L += ["", "## Category × difficulty", "", + "*one dot per challenge, coloured by status: " + + " ".join(f"{STATUS_ICON[s]} {s}" for s in STATUS_ORDER) + "*", "", + "| Category | " + " | ".join(diffs) + " | total |", + "| --- |" + " --- |" * len(diffs) + " ---: |"] + for cat in ov.categories: + chs = groups.get(cat, []) + cells = [icons([ch for ch in chs if (ch.difficulty or "?") == d]) or "·" for d in diffs] + L.append(f"| {cat} | " + " | ".join(cells) + f" | **{len(chs)}** |") + dtot = Counter(ch.difficulty or "?" for ch in ov.challenges) + L.append("| **total** | " + " | ".join(f"**{dtot[d]}**" for d in diffs) + + f" | **{len(ov.challenges)}** |") + + L += ["", "## Challenges"] + for cat in ov.categories: + chs = groups.get(cat) + if not chs: continue - any_issue = True - L.append(f"- **{a.name}**") - if not a.real: - L.append(" - nothing submitted at all") - for ch, p in items: - L.append(f" - `{Path(ch.path).name}`: {p}") - if not any_issue: - L.append("*none — everyone is done. suspicious.*") + L += ["", f"### {cat} ({len(chs)})", "", + "| | Challenge | Difficulty | Status | Event | Author | Tester | Last activity |", + "| --- | --- | --- | --- | --- | --- | --- | --- |"] + for ch in chs: + status = ch.stage or "?" + if ch.status_raw.lower() != status.lower(): + status += f" *({ch.status_raw})*" + cat_raw = ch.get("Category", "") + name = f"[{md_cell(ch.title)}]({ch.repo.web_url})" + if cat_raw.lower() != cat: + name += f" *({cat_raw})*" + L.append("| " + " | ".join(md_cell(x) for x in ( + STATUS_ICON[ch.stage], name, ch.difficulty or ch.get("Difficulty"), status, + ch.get("Event"), ch.get("Author"), + ch.get("Tester") if ch.has_tester else "—", ch.repo.last_activity)) + " |") + + L += ["", "## Needs attention", ""] + issues = [ch for ch in sorted(ov.challenges, key=lambda c: c.repo.path.lower()) if ch.problems] + for ch in issues: + L.append(f"- [{md_cell(ch.title)}]({ch.repo.web_url}): " + "; ".join(ch.problems)) + for ch in ov.blank: + why = "no README" if ch.repo.readme is None else "README is still the blank template" + L.append(f"- [{ch.repo.path}]({ch.repo.web_url}): {why}") + if not issues and not ov.blank: + L.append("*nothing — every README is filled in. suspicious.*") return "\n".join(L) + "\n" @@ -584,23 +688,26 @@ def main() -> None: if not note_url: sys.exit("--update needs a note URL, or HEDGEDOC_NOTE in the config") try: - HedgeDocNote(note_url) # fail on a typo now, not after the Nextcloud walk + HedgeDocNote(note_url) # fail on a typo now, not after the GitLab walk except ValueError as e: sys.exit(str(e)) lock = single_run_lock() # noqa: F841 (held for the lifetime of the process) - dav = Dav(cfg["NC_URL"], cfg["NC_USER"], cfg["NC_PASS"]) - authors = collect(dav, cfg["NC_ROOT"].strip("/"), args.no_color) - if not authors: - sys.exit(f"no author folders found under {cfg['NC_ROOT']!r}") + gl = GitLab(cfg["GITLAB_GROUP_URL"], cfg["GITLAB_TOKEN"]) + try: + ov = collect(gl) + except (requests.RequestException, RuntimeError) as e: + sys.exit(f"reading {cfg['GITLAB_GROUP_URL']} failed: {e}") + if not ov.challenges and not ov.blank: + sys.exit(f"no repos found in {cfg['GITLAB_GROUP_URL']!r}") - md = render_hedgedoc(authors, title, tags) + md = render_hedgedoc(ov, title, tags) if args.out: Path(args.out).write_text(md) if args.stdout: print(md, end="") elif not args.quiet: - print_terminal(authors, c=not args.no_color and sys.stdout.isatty()) + print_terminal(ov, c=not args.no_color and sys.stdout.isatty()) if args.out: print(f"markdown → {args.out}") if args.hedgedoc: