diff --git a/README.md b/README.md index de3569a..eb1740b 100644 --- a/README.md +++ b/README.md @@ -5,10 +5,11 @@ GitLab challenges group, works out each challenge's status, category and difficulty from its `README.md`, and checks that challenges marked as ready really are. The result goes to: -- **a HedgeDoc note**: an overview of all challenges, refreshed every 30 minutes +- **a HedgeDoc note**: an overview of the challenges included in the CTF, + refreshed every 30 minutes - **`challs.txt` in the deployment repo**: the clone URLs of every challenge - ready for infra testing, updated daily -- **the terminal**: the same overview, for a quick look + included in the CTF and ready for infra testing, updated daily +- **the terminal**: the same overview, plus the repos left out of the CTF ## Requirements @@ -38,6 +39,34 @@ repo, and permission to push to the branch if it is protected. Environment variables override the values in `.env`. +## Choosing what is in the CTF + +[`challenges.yml`](challenges.yml) maps each repo path to its state: + +```yaml +challenges: + baby_rsa: included # part of this CTF + static_antivirus: abandoned # dropped for good + some_chall: postponed # left to be finished for another CTF + +skip_checks: + whoami: [has_solve_script] # OSINT: there is nothing to script +``` + +- Only **included** challenges appear in the HedgeDoc note and the counts, and + only they can end up in `challs.txt`. +- **Abandoned** and **postponed** challenges are only listed in the terminal. +- A repo that is **not listed** is also only listed in the terminal, in its own + section, and is not tracked or deployed until somebody decides. The timers run + quietly, so run `./ctf_status.py` by hand to see new repos. +- **`skip_checks`** exempts a repo from individual repo checks, named after their + functions in [`tracker/checks.py`](tracker/checks.py). + +Entries for repos that do not exist are flagged under *Needs attention*. If the +file is missing or invalid, the run stops rather than committing an empty +`challs.txt`. The server reads the file from its checkout, so changes must be +pulled there. + ## Usage ```sh @@ -101,6 +130,7 @@ Two kinds of finding are reported: actually finished: `description.yml` is missing or still the template, its category or flag is wrong, a `deploy_dir` or its `Dockerfile` is missing, or there is no `solve.py`. A blocked challenge (โ›”) is kept out of `challs.txt`. + `skip_checks` in `challenges.yml` can exempt a challenge from a check. Passing every check does not prove that a challenge builds or solves. That is what testing is for. @@ -111,10 +141,12 @@ All rules are in [`tracker/checks.py`](tracker/checks.py), one function each. ``` ctf_status.py command line entry point +challenges.yml what is part of this CTF, and check exceptions tracker/ checks.py the rules: README problems and repo blockers challenge.py reads a README header into a Challenge gitlab.py GitLab API access + selection.py reads challenges.yml overview.py runs parsing and checks over all repos terminal.py terminal output markdown.py HedgeDoc note diff --git a/challenges.yml b/challenges.yml new file mode 100644 index 0000000..fb29514 --- /dev/null +++ b/challenges.yml @@ -0,0 +1,62 @@ +# Which repos of the challenges group are part of this CTF. Keys are repo paths, +# as in the GitLab URL. Only "included" challenges can end up in challs.txt. +# +# included part of this CTF +# abandoned dropped for good +# postponed left to be finished for another CTF +# +# The HedgeDoc note and the counts only cover included challenges. The others are +# only listed in the terminal summary. A repo that is not listed here is treated +# the same until somebody decides. +challenges: + # crypto + baby_rsa: included + Diffie-Cult-Agreements: included + gram_schmidt: included + intro-lwe: included + Just_go: included + LFSRiously: included + Really-Diffie-Culty-Agreements: included + safe: included + Static-Diffie-Cult-Agreement: included + # pwn + byog: included + haldenueberlauf: included + luaphole: included + top_3_syscalls: included + # web + jwt: included + name-tbd: included + orbiting_around: included + php: included + SATisfaction: included + # rev + break-in: included + mine-turtle: included + moving_target: included + STm8-OF-THE-ART: included + stm8-of-the-art-2: included + # misc + agent-harness: included + you-should-trust-me: included + # osint + concert: included + satellite: included + whoami: included + + # not in this CTF + static_antivirus: abandoned + TMPDRAFT_TumAuth: abandoned # never started + mirageos: abandoned # too complicated + arcabe: postponed + dwarf: postponed + resurrection: postponed + the_ART_of_pwning: postponed # unstable + + # deliberately unlisted, waiting for the authors to confirm they will finish: + # router_challenge, classifier_rev, the_cure + +# Repo checks a challenge does not have to pass: function names from +# REPO_CHECKS in tracker/checks.py, per repo. +skip_checks: + whoami: [has_solve_script] # OSINT: there is nothing to script diff --git a/ctf_status.py b/ctf_status.py index 080ca39..ea732d6 100755 --- a/ctf_status.py +++ b/ctf_status.py @@ -11,6 +11,9 @@ up: description.yml is filled in (not the template), its flag and containers mak sense, and the deploy dirs and solve.py exist. Otherwise the challenge is shown as blocked and kept out of challs.txt. The rules are in tracker/checks.py. +challenges.yml says which repos are part of this CTF; only those are counted, put +in the note and deployed. The terminal summary also lists the others. + Config: see .env.example -> .env (chmod 600) """ from __future__ import annotations @@ -29,6 +32,7 @@ from tracker.gitlab import GitLab from tracker.hedgedoc import HedgeDocNote, push_hedgedoc, update_hedgedoc from tracker.markdown import render_hedgedoc from tracker.overview import collect +from tracker.selection import load_selection from tracker.terminal import print_terminal @@ -59,7 +63,7 @@ def main() -> None: help="overwrite an existing HedgeDoc note in place; defaults to " "HEDGEDOC_NOTE from the config. This is the one to cron.") ap.add_argument("--challs", action="store_true", - help="commit the clone URLs of every unblocked Testing Ready+ challenge to " + help="commit the clone URLs of every included, unblocked Testing Ready+ challenge to " f"CHALLS_PATH (default {DEFAULT_CHALLS_PATH!r}) in the " "deployment repo CHALLS_REPO_URL, if they changed") ap.add_argument("--challs-out", metavar="FILE", @@ -89,6 +93,10 @@ def main() -> None: sys.exit(str(e)) if args.challs and not cfg.get("CHALLS_REPO_URL"): sys.exit("--challs needs CHALLS_REPO_URL in the config") + try: + selection = load_selection() + except ValueError as e: + sys.exit(str(e)) if note_url: single_run_lock("hedgedoc") if args.challs: @@ -96,10 +104,10 @@ def main() -> None: gl = GitLab(cfg["GITLAB_GROUP_URL"], cfg["GITLAB_TOKEN"]) try: - ov = collect(gl) + ov = collect(gl, selection) except (requests.RequestException, RuntimeError) as e: sys.exit(f"reading {cfg['GITLAB_GROUP_URL']} failed: {e}") - if not ov.challenges and not ov.blank: + if not ov.challenges and not ov.blank and not ov.excluded and not ov.undecided: sys.exit(f"no repos found in {cfg['GITLAB_GROUP_URL']!r}") md = render_hedgedoc(ov, title, tags) diff --git a/tracker/__init__.py b/tracker/__init__.py index f50001c..2accfe0 100644 --- a/tracker/__init__.py +++ b/tracker/__init__.py @@ -5,6 +5,7 @@ challenge.py turn a README header into a Challenge (status, category, ...) checks.py decide what is wrong with a challenge, and whether to believe a Testing Ready+ status <- the rules live here + selection.py read challenges.yml: what is part of this CTF, check exceptions overview.py run all of the above over every repo, group and sort the result terminal.py the terminal summary markdown.py the HedgeDoc note diff --git a/tracker/challenge.py b/tracker/challenge.py index cc74eb3..ca36028 100644 --- a/tracker/challenge.py +++ b/tracker/challenge.py @@ -30,6 +30,8 @@ STATUS_RE = re.compile(r"testing\s*ready|tested|ready|poc(?:\s*draft)?|planning" STATUS_ICON = {"Planning": "โšช", "PoC Draft": "๐ŸŸ ", "Testing Ready": "๐ŸŸก", "Tested": "๐Ÿ”ต", "Ready": "๐ŸŸข", None: "โ”"} BLOCKED_ICON = "โ›”" +# what challenges.yml can say about a repo, and how the reports call it +SELECTIONS = {"included": "included", "abandoned": "abandoned", "postponed": "left for another CTF"} CATEGORIES = ["crypto", "pwn", "web", "rev", "misc", "osint", "forensic", "hardware"] DIFFICULTIES = ["intro", "easy", "medium", "hard"] DIFF_RE = re.compile(r"intro|easy|medium|hard", re.I) @@ -90,7 +92,9 @@ class Challenge: repo: Repo name: str = "" fields: dict = field(default_factory=dict) - # filled in by overview.collect(), see checks.py + # filled in by overview.collect(), see selection.py and checks.py + selection: str | None = None # None: not in challenges.yml + skip_checks: list = field(default_factory=list) problems: list = field(default_factory=list) # readme_problems() blockers: list = field(default_factory=list) # repo_blockers() @@ -131,6 +135,10 @@ class Challenge: first = CATEGORY_SPLIT_RE.split(raw)[0].strip().lower() return first or None + @property + def included(self) -> bool: + return self.selection == "included" + @property def claims_testable(self) -> bool: """The README says Testing Ready or beyond; checks.is_testable() also asks the repo.""" diff --git a/tracker/challs.py b/tracker/challs.py index c7cbe77..da34998 100644 --- a/tracker/challs.py +++ b/tracker/challs.py @@ -5,8 +5,8 @@ from .overview import Overview def render_challs(ov: Overview) -> str: - """One clone URL per line, for everything that is Testing Ready or beyond - and not blocked.""" + """One clone URL per line, for everything that is included in this CTF, + Testing Ready or beyond and not blocked.""" return "".join(f"{c.repo.ssh_url}\n" for c in ov.testable) diff --git a/tracker/checks.py b/tracker/checks.py index a1556e5..d203b26 100644 --- a/tracker/checks.py +++ b/tracker/checks.py @@ -8,7 +8,8 @@ There are two kinds of finding: * blockers (repo_blockers): the README claims Testing Ready or beyond, but the repo shows the challenge is not finished. A blocked challenge keeps its status in the counts, but is left out of "ready for infra testing" and out of - challs.txt, so it does not get deployed. + challs.txt, so it does not get deployed. challenges.yml can exempt a challenge + from individual repo checks. Every rule is one small function that returns what it found wrong, as a list of messages (empty: nothing wrong). README_CHECKS and REPO_CHECKS list the rules @@ -28,13 +29,13 @@ from .gitlab import Repo def is_testable(c: Challenge) -> bool: - """Can infra deploy and test it now? The README says Testing Ready or beyond, - and the repo has no blockers.""" - return c.claims_testable and not c.blockers + """Can infra deploy and test it now? It is included in this CTF, the README + says Testing Ready or beyond, and the repo has no blockers.""" + return c.included and c.claims_testable and not c.blockers # ---------------------------------------------------------------- README ---- -# Run on every challenge whose README somebody filled in. +# Run on every included challenge whose README somebody filled in. def status_is_clear(c: Challenge) -> list[str]: stage, clear = stage_of(c.fields.get("Status", "")) @@ -196,4 +197,5 @@ def repo_blockers(c: Challenge) -> list[str]: d = read_description(c.repo) except DescriptionError as e: return [str(e)] - return [msg for check in REPO_CHECKS for msg in check(c, d)] + return [msg for check in REPO_CHECKS if check.__name__ not in c.skip_checks + for msg in check(c, d)] diff --git a/tracker/markdown.py b/tracker/markdown.py index 0d1802f..dbf9c74 100644 --- a/tracker/markdown.py +++ b/tracker/markdown.py @@ -1,4 +1,4 @@ -"""The HedgeDoc note, as markdown.""" +"""The HedgeDoc note, as markdown. It only covers challenges included in this CTF.""" from __future__ import annotations from collections import Counter @@ -25,14 +25,14 @@ def render_hedgedoc(ov: Overview, title: str, tags: str = "") -> str: L += ["---", f"tags: {tags}", "---", ""] L += [f"# {title}", "", f"**Last updated:** {stamp}", "", - f"*generated by [challenge_status_tracker]({REPO_URL}) from the README.md of every repo in the " - f"GitLab challenges group โ€” {len(ov.challenges)} challenges*", "", + f"*generated by [challenge_status_tracker]({REPO_URL}) from the README.md of every challenge " + f"included in this CTF โ€” {len(ov.challenges)} challenges*", "", " ยท ".join(f"{STATUS_ICON[s]} **{s or 'unknown'}** {total[s]}" for s in stages if total[s] or s), ""] ready = ov.testable L += [f"## โœ… Ready for infra testing ({len(ready)} of {len(ov.challenges)})", "", - "*every challenge at Testing Ready or beyond whose repo backs that up โ€” " + "*every included challenge at Testing Ready or beyond whose repo backs that up โ€” " "these can be deployed and tested against the infrastructure now*", ""] if ready: L += ["| | Challenge | Category | Difficulty | Status | Event | Author | Tester |", @@ -111,6 +111,8 @@ def render_hedgedoc(ov: Overview, title: str, tags: str = "") -> str: for ch in ov.blank: why = "no README" if ch.repo.readme is None else "README is still the blank template" L.append(f"- [{ch.repo.path}]({ch.repo.web_url}): {why}") - if not issues and not ov.blank: + for path in ov.unknown: + L.append(f"- {path}: in challenges.yml, but no such repo") + if not issues and not ov.blank and not ov.unknown: L.append("*nothing โ€” every README is filled in. suspicious.*") return "\n".join(L) + "\n" diff --git a/tracker/overview.py b/tracker/overview.py index df0b39d..46154ec 100644 --- a/tracker/overview.py +++ b/tracker/overview.py @@ -3,17 +3,21 @@ from __future__ import annotations from collections import defaultdict from concurrent.futures import ThreadPoolExecutor -from dataclasses import dataclass +from dataclasses import dataclass, field from .challenge import CATEGORIES, DIFFICULTIES, STATUS_ORDER, Challenge, difficulty_rank, parse_challenge from .checks import is_testable, readme_problems, repo_blockers from .gitlab import GitLab +from .selection import Selection @dataclass class Overview: - challenges: list # repos with a README somebody filled in - blank: list # repos with no README, or an untouched template + challenges: list # included repos with a README somebody filled in + blank: list # included repos with no README, or an untouched template + excluded: list = field(default_factory=list) # abandoned or postponed in challenges.yml + undecided: list = field(default_factory=list) # repos not in challenges.yml + unknown: list = field(default_factory=list) # repo paths in challenges.yml that do not exist @property def categories(self) -> list[str]: @@ -56,11 +60,19 @@ class Overview: return out -def collect(gl: GitLab) -> Overview: - chs, blank = [], [] +def collect(gl: GitLab, sel: Selection) -> Overview: + chs, blank, excluded, undecided, seen = [], [], [], [], set() for repo in gl.repos(): + seen.add(repo.path) c = parse_challenge(repo) - (chs if c.touched else blank).append(c) + c.selection = sel.states.get(repo.path) + c.skip_checks = sel.skip_checks.get(repo.path, []) + if not c.selection: + undecided.append(c) + elif not c.included: + excluded.append(c) + else: + (chs if c.touched else blank).append(c) for c in chs: c.problems = readme_problems(c) @@ -71,7 +83,10 @@ def collect(gl: GitLab) -> Overview: with ThreadPoolExecutor(max_workers=8) as pool: list(pool.map(check, [c for c in chs if c.claims_testable])) - return Overview(chs, sorted(blank, key=lambda c: c.repo.path.lower())) + return Overview(chs, sorted(blank, key=lambda c: c.repo.path.lower()), + sorted(excluded, key=lambda c: c.repo.path.lower()), + sorted(undecided, key=lambda c: c.repo.path.lower()), + sorted((set(sel.states) | set(sel.skip_checks)) - seen)) def icons(chs: list) -> str: diff --git a/tracker/selection.py b/tracker/selection.py new file mode 100644 index 0000000..180d275 --- /dev/null +++ b/tracker/selection.py @@ -0,0 +1,43 @@ +"""challenges.yml: which repos are part of this CTF, and their check exceptions.""" +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path + +import yaml + +from .challenge import SELECTIONS +from .checks import REPO_CHECKS + +SELECTION_FILE = Path(__file__).parent.parent / "challenges.yml" # next to ctf_status.py + + +@dataclass +class Selection: + states: dict # repo path -> one of SELECTIONS + skip_checks: dict # repo path -> names of REPO_CHECKS it need not pass + + +def load_selection(path: Path = SELECTION_FILE) -> Selection: + """Raises ValueError on anything off. A broken file must stop the run: without + it nothing counts as included, and challs.txt would be committed empty.""" + try: + doc = yaml.safe_load(path.read_text()) + except (OSError, yaml.YAMLError) as e: + raise ValueError(f"cannot read {path.name}: {e}") from None + states = doc.get("challenges") if isinstance(doc, dict) else None + if not states or not isinstance(states, dict): + raise ValueError(f"{path.name} has no `challenges:` mapping") + bad = [f"{k}: {v}" for k, v in states.items() if v not in SELECTIONS] + if bad: + raise ValueError(f"{path.name}: expected one of {', '.join(SELECTIONS)}, got " + ", ".join(bad)) + + skip = doc.get("skip_checks") or {} + if not isinstance(skip, dict) or not all(isinstance(v, list) for v in skip.values()): + raise ValueError(f"{path.name}: `skip_checks:` must map repos to lists of checks") + known = {check.__name__ for check in REPO_CHECKS} + bad = sorted({str(n) for names in skip.values() for n in names} - known) + if bad: + raise ValueError(f"{path.name}: no such repo check: " + ", ".join(bad)) + return Selection({str(k): v for k, v in states.items()}, + {str(k): [str(n) for n in v] for k, v in skip.items()}) diff --git a/tracker/terminal.py b/tracker/terminal.py index ba4d9bb..b2fb58e 100644 --- a/tracker/terminal.py +++ b/tracker/terminal.py @@ -3,7 +3,7 @@ from __future__ import annotations from collections import Counter -from .challenge import BLOCKED_ICON, STATUS_ICON, STATUS_ORDER +from .challenge import BLOCKED_ICON, SELECTIONS, STATUS_ICON, STATUS_ORDER from .overview import Overview, icons BOLD, DIM, RED, YEL, GRN, CYA, RST = ( @@ -52,6 +52,17 @@ def print_terminal(ov: Overview, c: bool) -> None: print(f" {cat:<10}" + "".join(f"{x or 'ยท'}{' ' * (13 - (2 * len(x) if x else 1))}" for x in cells)) + if ov.excluded: + print(f"\n{col(BOLD)}=== not in this CTF ({len(ov.excluded)}) ==={col(RST)}") + for ch in ov.excluded: + print(f" {ch.title[:34]:<35}{col(DIM)}{SELECTIONS[ch.selection]}{col(RST)}") + + if ov.undecided: + print(f"\n{col(BOLD)}{col(YEL)}=== not in challenges.yml: not tracked or deployed " + f"({len(ov.undecided)}) ==={col(RST)}") + for ch in ov.undecided: + print(f" {ch.title[:34]:<35}{col(DIM)}{ch.repo.path}{col(RST)}") + print(f"\n{col(BOLD)}=== needs attention ==={col(RST)}") for ch in sorted(ov.challenges, key=lambda c: c.repo.path.lower()): if ch.problems: @@ -60,4 +71,6 @@ def print_terminal(ov: Overview, c: bool) -> None: for ch in ov.blank: why = "no README" if ch.repo.readme is None else "README is still the blank template" print(f" {col(RED)}{ch.repo.path:<32}{col(RST)} {col(DIM)}{why}{col(RST)}") + for path in ov.unknown: + print(f" {col(RED)}{path:<32}{col(RST)} {col(DIM)}in challenges.yml, but no such repo{col(RST)}") print()