Split ctf_status.py into a tracker package, config from .env only
ctf_status.py is now just the command line; the work lives in tracker/. The rules that decide whether a challenge is believed are in tracker/checks.py, one function per rule, listed in README_CHECKS and REPO_CHECKS. Output is unchanged. Read the config from .env only (ctf_status.env.example -> .env.example), let single_run_lock() hold its own lock files, and rewrite the README as a short overview of what the repo does and how to run it. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5.5
@@ -1,22 +1,21 @@
|
|||||||
# cp to ctf_status.env and chmod 600 (a plain .env next to the script works too)
|
# cp .env.example .env && chmod 600 .env
|
||||||
|
|
||||||
# --- GitLab: the group holding one repo per challenge -----------------------
|
# --- GitLab: the group holding one repo per challenge -----------------------
|
||||||
GITLAB_GROUP_URL=https://gitlab.lrz.de/h4tum-ctf/h4tum-ctf-2026/challenges
|
GITLAB_GROUP_URL=https://gitlab.lrz.de/h4tum-ctf/h4tum-ctf-2026/challenges
|
||||||
# fine-grained personal access token, scoped to that group, with
|
# fine-grained personal access token, scoped to that group, with
|
||||||
# Project: Read (list the repos)
|
# Project: Read (list the repos)
|
||||||
# Repository: Read (download each README.md)
|
# Repository: Read (download README.md and description.yml, list the files)
|
||||||
GITLAB_TOKEN=glpat-xxxxxxxxxxxxxxxxxxxx
|
GITLAB_TOKEN=glpat-xxxxxxxxxxxxxxxxxxxx
|
||||||
|
|
||||||
# --- HedgeDoc: the existing note that gets overwritten every run ------------
|
# --- HedgeDoc: the existing note that --update overwrites -------------------
|
||||||
# full URL of the note (its permission must be "freely" so an anonymous
|
# full URL of the note; its permission must be "freely" so an anonymous run
|
||||||
# run may write to it), e.g. https://hedgedoc.cit.tum.de/AbCdEf123
|
# may write to it
|
||||||
HEDGEDOC_NOTE=
|
HEDGEDOC_NOTE=
|
||||||
# YAML front-matter tags written at the top of the note
|
# YAML front-matter tags written at the top of the note
|
||||||
HEDGEDOC_TAGS=Logistics, h4tum CTF 2026
|
HEDGEDOC_TAGS=Logistics, h4tum CTF 2026
|
||||||
|
|
||||||
# --- deployment repo: --challs keeps <CHALLS_PATH> in it up to date --------
|
# --- deployment repo: --challs keeps <CHALLS_PATH> in it up to date --------
|
||||||
# one clone URL per line for every challenge at Testing Ready or beyond. The
|
# the token additionally needs, on this project: Repository: Read + Write
|
||||||
# token additionally needs, on this project: Repository: Read + Repository: Write
|
|
||||||
CHALLS_REPO_URL=https://gitlab.lrz.de/h4tum-ctf/h4tum-ctf-deployment/ctf-deployment
|
CHALLS_REPO_URL=https://gitlab.lrz.de/h4tum-ctf/h4tum-ctf-deployment/ctf-deployment
|
||||||
CHALLS_PATH=2026/challs.txt
|
CHALLS_PATH=2026/challs.txt
|
||||||
CHALLS_BRANCH=main
|
CHALLS_BRANCH=main
|
||||||
@@ -1,4 +1,3 @@
|
|||||||
__pycache__/
|
__pycache__/
|
||||||
ctf_status.env
|
|
||||||
*.md.out
|
*.md.out
|
||||||
.env
|
.env
|
||||||
|
|||||||
@@ -1,198 +1,126 @@
|
|||||||
# ctf_status.py
|
# CTF challenge status tracker
|
||||||
|
|
||||||
Reads the `README.md` of every repo in the GitLab challenges group (REST API, no
|
Tracks the progress of the h4tum CTF challenges. It reads every repo in the
|
||||||
clone) and shows which categories have challenges in which state, at which
|
GitLab challenges group, works out each challenge's status, category and
|
||||||
difficulty — in the terminal and as a HedgeDoc note. A status of Testing Ready
|
difficulty from its `README.md`, and checks that challenges marked as ready
|
||||||
or beyond is only believed if the repo backs it up (see
|
really are. The result goes to:
|
||||||
[Checking the repo](#checking-the-repo)).
|
|
||||||
|
|
||||||
Needs `requests` and `PyYAML` (`python-requests python-yaml` on Arch,
|
- **a HedgeDoc note**: an overview of all challenges, refreshed every 30 minutes
|
||||||
`python3-requests python3-yaml` on Debian).
|
- **`challs.txt` in the deployment repo**: the clone URLs of every challenge
|
||||||
|
ready for infra testing, updated daily
|
||||||
|
- **the terminal**: the same overview, for a quick look
|
||||||
|
|
||||||
## Setup
|
## Requirements
|
||||||
|
|
||||||
|
- Python 3.9+
|
||||||
|
- [`requests`](https://pypi.org/project/requests/) and [`PyYAML`](https://pypi.org/project/PyYAML/)
|
||||||
|
(`python-requests python-yaml` on Arch, `python3-requests python3-yaml` on Debian)
|
||||||
|
|
||||||
|
## Configuration
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
cp ctf_status.env.example ctf_status.env && chmod 600 ctf_status.env
|
cp .env.example .env && chmod 600 .env
|
||||||
$EDITOR ctf_status.env # GitLab group URL + token, HedgeDoc note
|
|
||||||
```
|
```
|
||||||
|
|
||||||
## Use
|
| Key | Required | Meaning |
|
||||||
|
|
||||||
```sh
|
|
||||||
./ctf_status.py # terminal summary
|
|
||||||
./ctf_status.py -o overview.md # + HedgeDoc markdown to a file
|
|
||||||
./ctf_status.py --stdout | wl-copy # straight to clipboard, paste into HedgeDoc
|
|
||||||
./ctf_status.py --update # overwrite the note in HEDGEDOC_NOTE
|
|
||||||
./ctf_status.py --update <note-url> # ... or an explicit one
|
|
||||||
```
|
|
||||||
|
|
||||||
The token is a fine-grained personal access token on the challenges group with
|
|
||||||
**Project: Read** (to list the repos) and **Repository: Read** (to download the
|
|
||||||
READMEs and `description.yml`, and to list the repo's files). A plain `.env` with `GL_PAT=` / `GITLAB_URL=` is read as well.
|
|
||||||
|
|
||||||
## Keeping a HedgeDoc note up to date
|
|
||||||
|
|
||||||
`--update` overwrites an **existing** note in place, so the URL you share stays the
|
|
||||||
same. Every run rewrites the whole body, starting with
|
|
||||||
|
|
||||||
```
|
|
||||||
---
|
|
||||||
tags: Logistics, h4tum CTF 2026
|
|
||||||
---
|
|
||||||
```
|
|
||||||
|
|
||||||
and a `**Last updated:** <date> <time> <tz>` line under the heading, so a stale note
|
|
||||||
is obvious at a glance. Tags come from `HEDGEDOC_TAGS` (or `--tags`).
|
|
||||||
|
|
||||||
The note's permission has to be **freely** (Menu → Permission), because the run is
|
|
||||||
anonymous. HedgeDoc 1.x has no REST API for editing a note, so the script talks the
|
|
||||||
editor's own engine.io/socket.io + OT protocol over long-polling — it reads the
|
|
||||||
current text, sends one "delete everything, insert this" operation, waits for the
|
|
||||||
server's `ack`, and reads the note back to confirm. Anyone with the note open sees
|
|
||||||
the change live. Plain `requests`, no websocket library.
|
|
||||||
|
|
||||||
`--hedgedoc <instance>` (POST a *new* note) is unrelated and only works where
|
|
||||||
anonymous note creation is enabled — on `hedgedoc.cit.tum.de` it is not.
|
|
||||||
|
|
||||||
## challs.txt for the deployment repo
|
|
||||||
|
|
||||||
`--challs` commits the clone URL of every challenge at **Testing Ready or beyond**
|
|
||||||
that is not blocked (see [Checking the repo](#checking-the-repo))
|
|
||||||
to `CHALLS_PATH` (default `2026/challs.txt`) in the deployment repo
|
|
||||||
`CHALLS_REPO_URL`, one per line, the same format as last year's:
|
|
||||||
|
|
||||||
```
|
|
||||||
git@gitlab.lrz.de:h4tum-ctf/h4tum-ctf-2026/challenges/baby_rsa.git
|
|
||||||
...
|
|
||||||
```
|
|
||||||
|
|
||||||
It only commits when the list actually changed, with a message naming what was
|
|
||||||
added and removed (`Update 2026/challs.txt from challenge status: +jwt -safe`).
|
|
||||||
A challenge that falls back below Testing Ready drops out again. Preview without
|
|
||||||
committing:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
./ctf_status.py -q --challs-out -
|
|
||||||
```
|
|
||||||
|
|
||||||
The token additionally needs **Repository: Read** and **Repository: Write** on the
|
|
||||||
deployment project. If the branch is protected, the token's user must be allowed
|
|
||||||
to push to it. Without `CHALLS_BRANCH`, the default branch is looked up, which
|
|
||||||
also needs **Project: Read** there.
|
|
||||||
|
|
||||||
## On a schedule, on a server
|
|
||||||
|
|
||||||
Two timers, independent of each other:
|
|
||||||
|
|
||||||
| timer | when | runs |
|
|
||||||
|---|---|---|
|
|---|---|---|
|
||||||
| `ctf-status.timer` | every 30 minutes | `--update` (HedgeDoc note) |
|
| `GITLAB_GROUP_URL` | yes | GitLab group that holds one repo per challenge |
|
||||||
| `ctf-challs.timer` | daily at 04:00 (server local time) | `--challs` (challs.txt commit) |
|
| `GITLAB_TOKEN` | yes | Personal access token, see below |
|
||||||
|
| `HEDGEDOC_NOTE` | for `--update` | URL of an existing note; its permission must be **freely** |
|
||||||
|
| `HEDGEDOC_TAGS` | no | Tags written into the note's front matter |
|
||||||
|
| `CHALLS_REPO_URL` | for `--challs` | Deployment repo that holds `challs.txt` |
|
||||||
|
| `CHALLS_PATH` | no | Path of the file in that repo (default `2026/challs.txt`) |
|
||||||
|
| `CHALLS_BRANCH` | no | Branch to commit to (default: the repo's default branch) |
|
||||||
|
|
||||||
Copy the repo to the server, create `ctf_status.env` or `.env` there (both are gitignored, so they
|
The token needs **Project: Read** and **Repository: Read** on the challenges
|
||||||
do not travel with the code), then:
|
group. For `--challs` it also needs **Repository: Read + Write** on the deployment
|
||||||
|
repo, and permission to push to the branch if it is protected.
|
||||||
|
|
||||||
|
Environment variables override the values in `.env`.
|
||||||
|
|
||||||
|
## Usage
|
||||||
|
|
||||||
|
```sh
|
||||||
|
./ctf_status.py # print the overview
|
||||||
|
./ctf_status.py -o overview.md # also write the note's markdown to a file
|
||||||
|
./ctf_status.py --update # overwrite the HedgeDoc note
|
||||||
|
./ctf_status.py --challs # commit challs.txt, if it changed
|
||||||
|
./ctf_status.py -q --challs-out - # preview challs.txt without committing
|
||||||
|
```
|
||||||
|
|
||||||
|
`./ctf_status.py --help` lists all options. With `--quiet`, a run prints nothing
|
||||||
|
unless something changed or failed.
|
||||||
|
|
||||||
|
## Deployment
|
||||||
|
|
||||||
|
On the server, with `.env` in place:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
./install-timer.sh
|
./install-timer.sh
|
||||||
```
|
```
|
||||||
|
|
||||||
It writes systemd **user** units into `~/.config/systemd/user/`, substituting
|
This installs and starts two systemd user timers:
|
||||||
whatever directory the repo actually sits in, enables both timers, turns on lingering
|
|
||||||
so they keep firing when you are not logged in, and does one run of each to prove they work.
|
| Timer | Runs | Does |
|
||||||
Re-run it any time; it just overwrites the units.
|
|---|---|---|
|
||||||
|
| `ctf-status.timer` | every 30 minutes | `--update`: refresh the HedgeDoc note |
|
||||||
|
| `ctf-challs.timer` | daily at 04:00 | `--challs`: commit `challs.txt` |
|
||||||
|
|
||||||
|
Re-running the script is safe. To check on the timers:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
systemctl --user list-timers 'ctf-*' # when they next fire
|
systemctl --user list-timers 'ctf-*'
|
||||||
journalctl --user -u ctf-status.service -n 50 # what the HedgeDoc run did
|
journalctl --user -u ctf-status.service -n 50
|
||||||
journalctl --user -u ctf-challs.service -n 50 # what the challs.txt run did
|
journalctl --user -u ctf-challs.service -n 50
|
||||||
systemctl --user start ctf-challs.service # commit challs.txt now
|
|
||||||
systemctl --user disable --now ctf-challs.timer # stop the daily commit
|
|
||||||
```
|
```
|
||||||
|
|
||||||
The HedgeDoc timer fires at `:00` and `:30` with up to a minute of jitter; both
|
## How challenges are evaluated
|
||||||
timers have `Persistent=true`, so a run missed while the machine was down is caught up.
|
|
||||||
Plain cron works too, if you would rather:
|
|
||||||
|
|
||||||
```
|
Each challenge repo starts from a template whose README begins with:
|
||||||
*/30 * * * * /path/to/planner/ctf_status.py --update --quiet
|
|
||||||
0 4 * * * /path/to/planner/ctf_status.py --challs --quiet
|
|
||||||
```
|
|
||||||
|
|
||||||
`--quiet` keeps a successful run silent, so you only hear about it when something
|
|
||||||
changed or broke. Overlapping runs of the same target are prevented by a lock file
|
|
||||||
per target (the 04:00 challs run and the HedgeDoc run can overlap), and the
|
|
||||||
HedgeDoc write is retried (`--retries`, default 3) before the run gives up with a
|
|
||||||
one-line error and a non-zero exit.
|
|
||||||
|
|
||||||
## What the note shows
|
|
||||||
|
|
||||||
1. **✅ Ready for infra testing** — first thing in the note: every challenge at
|
|
||||||
Testing Ready, Tested or Ready, linked to its repo, so infra knows what it
|
|
||||||
can deploy and test against right now. Blocked ones are not in it.
|
|
||||||
2. **⛔ Claims Testing Ready+, but the repo disagrees** — only shown when there
|
|
||||||
are any: what the README claims, and why the repo says otherwise.
|
|
||||||
3. **Status by category** — how many challenges each category has per stage
|
|
||||||
(⚪ Planning → 🟠 PoC Draft → 🟡 Testing Ready → 🔵 Tested → 🟢 Ready).
|
|
||||||
4. **Category × difficulty** — one dot per challenge, coloured by its status (⛔
|
|
||||||
when blocked), so gaps and stragglers show up at a glance.
|
|
||||||
5. **Challenges** — one table per category, linked to the repo, with event,
|
|
||||||
author, tester and the repo's last activity.
|
|
||||||
6. **Needs attention** — see below.
|
|
||||||
|
|
||||||
## Reading the README
|
|
||||||
|
|
||||||
Only the template's header block is parsed:
|
|
||||||
|
|
||||||
```
|
```
|
||||||
### Challenge Name
|
### Challenge Name
|
||||||
- Category: crypto
|
- Category: crypto
|
||||||
- Difficulty: medium
|
- Difficulty: medium
|
||||||
- Event: TUM student CTF
|
- Event: ...
|
||||||
- Author: ...
|
- Author: ...
|
||||||
- Tester: ...
|
- Tester: ...
|
||||||
- Status: Testing Ready
|
- Status: Testing Ready
|
||||||
```
|
```
|
||||||
|
|
||||||
People fill it in loosely, so values are normalised:
|
The status is one of **Planning → PoC Draft → Testing Ready → Tested → Ready**.
|
||||||
|
Loosely written values are normalised. If a status names several stages, the
|
||||||
|
least advanced one counts.
|
||||||
|
|
||||||
- **Status** is matched against the five stages ("Tested with …" → Tested,
|
Two kinds of finding are reported:
|
||||||
"PoC" → PoC Draft). If it names several ("Planning to PoC Draft", a half-deleted
|
|
||||||
option list) the *least* advanced one counts and it is flagged as unclear.
|
|
||||||
- **Difficulty** ranges written as `medium-hard`, `medium/hard` or
|
|
||||||
`easy to medium` become their own column; otherwise the first level named wins.
|
|
||||||
- **Category**: the first one named counts ("rev, misc" → rev); the full value
|
|
||||||
is shown next to the challenge. Categories outside the usual list (e.g. `ai`)
|
|
||||||
get their own row.
|
|
||||||
|
|
||||||
## Checking the repo
|
- **Problems** are missing or unclear README fields, such as no category, an
|
||||||
|
unreadable status, or a Testing Ready challenge with no tester. They are
|
||||||
|
listed under *Needs attention* and change nothing else.
|
||||||
|
- **Blockers** are signs that a challenge claiming Testing Ready or beyond is not
|
||||||
|
actually finished: `description.yml` is missing or still the template, its
|
||||||
|
category or flag is wrong, a `deploy_dir` or its `Dockerfile` is missing, or
|
||||||
|
there is no `solve.py`. A blocked challenge (⛔) is kept out of `challs.txt`.
|
||||||
|
|
||||||
Anyone can write `Status: Testing Ready`, so for every challenge whose README
|
Passing every check does not prove that a challenge builds or solves. That is
|
||||||
claims Testing Ready, Tested or Ready the repo itself is looked at too. Each of
|
what testing is for.
|
||||||
these makes it **blocked** (⛔): it keeps its claimed status in the counts, but
|
|
||||||
it is left out of *Ready for infra testing* and `challs.txt`, and the reason is
|
|
||||||
shown in the note.
|
|
||||||
|
|
||||||
- no `description.yml` at the top level, or one that does not parse
|
All rules are in [`tracker/checks.py`](tracker/checks.py), one function each.
|
||||||
- `description.yml` still has anything from the template it was copied from
|
|
||||||
(`netcat-check-multi-container`): its id, name, author `jh`, its flag, or the
|
|
||||||
`cont1`/`cont2` / `container-1`/`container-2` containers
|
|
||||||
- its `category` is not one of the README's categories
|
|
||||||
- the flag is not `h4tum{...}`
|
|
||||||
- `dynamic: true`, but no container has a `flag:` block to receive it
|
|
||||||
- a container's `deploy_dir` does not exist or has no `Dockerfile`
|
|
||||||
- no `solve.py` anywhere in the repo
|
|
||||||
|
|
||||||
These are cheap signs that a challenge was never finished, not proof that it
|
## Project layout
|
||||||
works: a repo can pass every check and still not build or solve. That is what
|
|
||||||
actually testing it is for.
|
|
||||||
|
|
||||||
## What it flags
|
```
|
||||||
|
ctf_status.py command line entry point
|
||||||
- challenge at Testing Ready or beyond with no tester (`[Tester Name]`, `??`,
|
tracker/
|
||||||
`None`, `not found yet` all count as none)
|
checks.py the rules: README problems and repo blockers
|
||||||
- status / difficulty / category missing or unclear
|
challenge.py reads a README header into a Challenge
|
||||||
- no challenge name in the README
|
gitlab.py GitLab API access
|
||||||
- repos whose README is missing or still the untouched template — these are left
|
overview.py runs parsing and checks over all repos
|
||||||
out of the counts
|
terminal.py terminal output
|
||||||
|
markdown.py HedgeDoc note
|
||||||
Repos GitLab has scheduled for deletion, and archived repos, are skipped.
|
challs.py challs.txt
|
||||||
|
hedgedoc.py writes the note to HedgeDoc
|
||||||
|
config.py reads the configuration
|
||||||
|
install-timer.sh installs the systemd units below
|
||||||
|
ctf-*.service/.timer systemd unit templates
|
||||||
|
```
|
||||||
|
|||||||
+17
-801
@@ -9,819 +9,34 @@ Lists every repo in the GitLab challenges group, reads each repo's README.md
|
|||||||
A README status of Testing Ready or beyond is only believed if the repo backs it
|
A README status of Testing Ready or beyond is only believed if the repo backs it
|
||||||
up: description.yml is filled in (not the template), its flag and containers make
|
up: description.yml is filled in (not the template), its flag and containers make
|
||||||
sense, and the deploy dirs and solve.py exist. Otherwise the challenge is shown
|
sense, and the deploy dirs and solve.py exist. Otherwise the challenge is shown
|
||||||
as blocked and kept out of challs.txt.
|
as blocked and kept out of challs.txt. The rules are in tracker/checks.py.
|
||||||
|
|
||||||
Config: see ctf_status.env.example -> ctf_status.env (chmod 600)
|
Config: see .env.example -> .env (chmod 600)
|
||||||
"""
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
import base64
|
|
||||||
import fcntl
|
import fcntl
|
||||||
import json
|
|
||||||
import os
|
|
||||||
import random
|
|
||||||
import re
|
|
||||||
import sys
|
import sys
|
||||||
import tempfile
|
import tempfile
|
||||||
import time
|
|
||||||
from collections import Counter, defaultdict
|
|
||||||
from concurrent.futures import ThreadPoolExecutor
|
|
||||||
from dataclasses import dataclass, field
|
|
||||||
from datetime import datetime
|
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from urllib.parse import quote, urlsplit
|
|
||||||
|
|
||||||
import requests
|
import requests
|
||||||
import yaml
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------- config ----
|
from tracker.challs import challs_message, render_challs
|
||||||
|
from tracker.config import DEFAULT_CHALLS_PATH, DEFAULT_TAGS, DEFAULT_TITLE, load_env
|
||||||
|
from tracker.gitlab import GitLab
|
||||||
|
from tracker.hedgedoc import HedgeDocNote, push_hedgedoc, update_hedgedoc
|
||||||
|
from tracker.markdown import render_hedgedoc
|
||||||
|
from tracker.overview import collect
|
||||||
|
from tracker.terminal import print_terminal
|
||||||
|
|
||||||
CFG_KEYS = ("GITLAB_GROUP_URL", "GITLAB_TOKEN")
|
|
||||||
OPT_KEYS = ("HEDGEDOC_NOTE", "HEDGEDOC_TAGS", "TITLE",
|
|
||||||
"CHALLS_REPO_URL", "CHALLS_PATH", "CHALLS_BRANCH")
|
|
||||||
# the names a plain `.env` tends to use for the same thing
|
|
||||||
ALIASES = {"GL_PAT": "GITLAB_TOKEN", "GITLAB_URL": "GITLAB_GROUP_URL"}
|
|
||||||
DEFAULT_TAGS = "Logistics, h4tum CTF 2026"
|
|
||||||
DEFAULT_TITLE = "CTF challenge tracking"
|
|
||||||
DEFAULT_CHALLS_PATH = "2026/challs.txt"
|
|
||||||
|
|
||||||
|
# the lock is released when its file is closed, so keep them open until we exit
|
||||||
|
_held_locks = []
|
||||||
|
|
||||||
def load_env() -> dict:
|
|
||||||
cfg = {}
|
|
||||||
here = Path(__file__).parent
|
|
||||||
# every file that exists contributes; the first one to set a key wins
|
|
||||||
for candidate in (here / "ctf_status.env", here / ".env",
|
|
||||||
Path.home() / ".config" / "ctf-status" / "env"):
|
|
||||||
if not candidate.exists():
|
|
||||||
continue
|
|
||||||
for line in candidate.read_text().splitlines():
|
|
||||||
line = line.strip()
|
|
||||||
if not line or line.startswith("#") or "=" not in line:
|
|
||||||
continue
|
|
||||||
k, v = line.split("=", 1)
|
|
||||||
k = k.strip().upper()
|
|
||||||
cfg.setdefault(ALIASES.get(k, k), v.strip().strip("'\""))
|
|
||||||
for k in CFG_KEYS + OPT_KEYS + tuple(ALIASES):
|
|
||||||
if os.environ.get(k):
|
|
||||||
cfg[ALIASES.get(k, k)] = os.environ[k]
|
|
||||||
missing = [k for k in CFG_KEYS if not cfg.get(k)]
|
|
||||||
if missing:
|
|
||||||
sys.exit(f"missing config: {', '.join(missing)} (see ctf_status.env.example)")
|
|
||||||
return cfg
|
|
||||||
|
|
||||||
|
def single_run_lock(target: str) -> None:
|
||||||
# --------------------------------------------------------------- gitlab -----
|
|
||||||
|
|
||||||
@dataclass
|
|
||||||
class Repo:
|
|
||||||
path: str
|
|
||||||
web_url: str
|
|
||||||
ssh_url: str
|
|
||||||
last_activity: str # YYYY-MM-DD
|
|
||||||
readme: str | None # None: the repo has no README at all
|
|
||||||
id: int = 0
|
|
||||||
branch: str | None = None
|
|
||||||
# only fetched for challenges whose README claims Testing Ready or beyond
|
|
||||||
files: set[str] | None = None # every file and dir in the repo
|
|
||||||
desc: str | None = None # description.yml, None if there is none
|
|
||||||
|
|
||||||
|
|
||||||
class GitLab:
|
|
||||||
"""Access to one group, plus the one file we write elsewhere. The token needs
|
|
||||||
Project: Read (to list the group) and Repository: Read (to download files);
|
|
||||||
--challs additionally needs read + write on the deployment repo."""
|
|
||||||
|
|
||||||
def __init__(self, group_url: str, token: str, timeout: int = 20):
|
|
||||||
parts = urlsplit(group_url)
|
|
||||||
self.api = f"{parts.scheme}://{parts.netloc}/api/v4"
|
|
||||||
self.group = parts.path.strip("/")
|
|
||||||
self.timeout = timeout
|
|
||||||
self.s = requests.Session()
|
|
||||||
self.s.headers["PRIVATE-TOKEN"] = token
|
|
||||||
|
|
||||||
def _get(self, url: str, **params) -> requests.Response:
|
|
||||||
return self._req("GET", url, params=params)
|
|
||||||
|
|
||||||
def _req(self, method: str, url: str, ok: tuple = (), **kw) -> requests.Response:
|
|
||||||
r = self.s.request(method, url, timeout=self.timeout, **kw)
|
|
||||||
if r.status_code in ok:
|
|
||||||
return r
|
|
||||||
if r.status_code == 403:
|
|
||||||
# fine-grained tokens say exactly which permission is missing
|
|
||||||
try:
|
|
||||||
why = r.json().get("error_description") or r.text
|
|
||||||
except ValueError:
|
|
||||||
why = r.text
|
|
||||||
raise RuntimeError(f"gitlab refused {r.url}: {why}")
|
|
||||||
r.raise_for_status()
|
|
||||||
return r
|
|
||||||
|
|
||||||
def projects(self) -> list[dict]:
|
|
||||||
out, page = [], "1"
|
|
||||||
while page:
|
|
||||||
r = self._get(f"{self.api}/groups/{quote(self.group, safe='')}/projects",
|
|
||||||
include_subgroups="true", per_page="100", page=page,
|
|
||||||
order_by="path", sort="asc")
|
|
||||||
out += r.json()
|
|
||||||
page = r.headers.get("X-Next-Page", "")
|
|
||||||
# repos GitLab is about to delete are renamed "...-deletion_scheduled-<id>"
|
|
||||||
return [p for p in out if not p.get("archived") and not p.get("marked_for_deletion_on")]
|
|
||||||
|
|
||||||
def readme(self, p: dict) -> str | None:
|
|
||||||
branch, url = p.get("default_branch"), p.get("readme_url")
|
|
||||||
if not branch or not url or f"/-/blob/{branch}/" not in url:
|
|
||||||
return None
|
|
||||||
return self._raw(p["id"], url.split(f"/-/blob/{branch}/", 1)[1], branch)
|
|
||||||
|
|
||||||
def _raw(self, project_id: int, path: str, branch: str) -> str:
|
|
||||||
r = self._get(f"{self.api}/projects/{project_id}/repository/files/"
|
|
||||||
f"{quote(path, safe='')}/raw", ref=branch)
|
|
||||||
return r.content.decode("utf-8", "replace")
|
|
||||||
|
|
||||||
def repos(self) -> list[Repo]:
|
|
||||||
projects = self.projects()
|
|
||||||
|
|
||||||
def fetch(p: dict) -> Repo:
|
|
||||||
return Repo(path=p["path"], web_url=p["web_url"],
|
|
||||||
ssh_url=p["ssh_url_to_repo"],
|
|
||||||
last_activity=(p.get("last_activity_at") or "")[:10],
|
|
||||||
readme=self.readme(p), id=p["id"], branch=p.get("default_branch"))
|
|
||||||
|
|
||||||
with ThreadPoolExecutor(max_workers=8) as pool:
|
|
||||||
return list(pool.map(fetch, projects))
|
|
||||||
|
|
||||||
def load_files(self, repo: Repo) -> None:
|
|
||||||
"""Fill in repo.files and repo.desc, for check_files()."""
|
|
||||||
repo.files = set()
|
|
||||||
if not repo.branch:
|
|
||||||
return
|
|
||||||
page = "1"
|
|
||||||
while page:
|
|
||||||
r = self._get(f"{self.api}/projects/{repo.id}/repository/tree", ref=repo.branch,
|
|
||||||
recursive="true", per_page="100", page=page)
|
|
||||||
repo.files |= {t["path"] for t in r.json()}
|
|
||||||
page = r.headers.get("X-Next-Page", "")
|
|
||||||
if "description.yml" in repo.files:
|
|
||||||
repo.desc = self._raw(repo.id, "description.yml", repo.branch)
|
|
||||||
|
|
||||||
def write_file(self, repo_url: str, path: str, content: str, branch: str | None,
|
|
||||||
message) -> bool:
|
|
||||||
"""Commit `content` to `path` in the repo at `repo_url` (a web URL), creating
|
|
||||||
the file if needed; `message(old_content)` makes the commit message.
|
|
||||||
False if the file already said exactly this."""
|
|
||||||
proj = quote(urlsplit(repo_url).path.strip("/").split("/-/", 1)[0], safe="")
|
|
||||||
base = f"{self.api}/projects/{proj}"
|
|
||||||
if not branch:
|
|
||||||
branch = self._get(base).json()["default_branch"]
|
|
||||||
url = f"{base}/repository/files/{quote(path, safe='')}"
|
|
||||||
r = self._req("GET", url, ok=(404,), params={"ref": branch})
|
|
||||||
exists = r.status_code != 404
|
|
||||||
old = base64.b64decode(r.json()["content"]).decode("utf-8", "replace") if exists else ""
|
|
||||||
if exists and old == content:
|
|
||||||
return False
|
|
||||||
self._req("PUT" if exists else "POST", url,
|
|
||||||
json={"branch": branch, "content": content, "commit_message": message(old)})
|
|
||||||
return True
|
|
||||||
|
|
||||||
|
|
||||||
# --------------------------------------------------------------- parsing ----
|
|
||||||
|
|
||||||
FIELDS = ("Category", "Difficulty", "Event", "Author", "Tester", "Status")
|
|
||||||
HEADING_RE = re.compile(r"^\s{0,3}#{1,4}\s+(.+?)\s*$", re.M)
|
|
||||||
FIELD_RE = re.compile(r"^\s*[-*]\s*\**(?P<key>[A-Za-z ]+?)\**\s*:\s*(?P<val>.*?)\s*$", re.M)
|
|
||||||
# a value the author never touched: "[crypto, pwn, ...]" or "[Author Name]"
|
|
||||||
PLACEHOLDER_RE = re.compile(r"^\[.*\]$")
|
|
||||||
NO_TESTER_RE = re.compile(r"^(\?+|none|tester name|not found\b.*|tbd)(\s.*)?$", re.I)
|
|
||||||
|
|
||||||
STATUS_ORDER = ["Planning", "PoC Draft", "Testing Ready", "Tested", "Ready"]
|
|
||||||
STATUS_RE = re.compile(r"testing\s*ready|tested|ready|poc(?:\s*draft)?|planning", re.I)
|
|
||||||
STATUS_ICON = {"Planning": "⚪", "PoC Draft": "🟠", "Testing Ready": "🟡",
|
|
||||||
"Tested": "🔵", "Ready": "🟢", None: "❔"}
|
|
||||||
CATEGORIES = ["crypto", "pwn", "web", "rev", "misc", "osint", "forensic", "hardware"]
|
|
||||||
DIFFICULTIES = ["intro", "easy", "medium", "hard"]
|
|
||||||
DIFF_RE = re.compile(r"intro|easy|medium|hard", re.I)
|
|
||||||
DIFF_RANGE_RE = re.compile(r"(intro|easy|medium|hard)\s*(?:-|–|/|\bto\b)\s*(intro|easy|medium|hard)", re.I)
|
|
||||||
CATEGORY_SPLIT_RE = re.compile(r"[,/&+]| and ")
|
|
||||||
|
|
||||||
BLOCKED_ICON = "⛔"
|
|
||||||
FLAG_RE = re.compile(r"^h4tum\{[^{}]+\}$")
|
|
||||||
# what every repo starts with: the challenge-format example "netcat-check-multi-container"
|
|
||||||
TEMPLATE_ID = "netcat-check-multi-container"
|
|
||||||
TEMPLATE_NAME = "Netcat Check Multi Container"
|
|
||||||
TEMPLATE_AUTHOR = "jh"
|
|
||||||
TEMPLATE_FLAG = "h4tum{n3tc4t_1s_c0ol}"
|
|
||||||
TEMPLATE_CONTAINERS = {"cont1", "cont2"}
|
|
||||||
TEMPLATE_DIRS = {"container-1", "container-2"}
|
|
||||||
|
|
||||||
|
|
||||||
def clean(v: str) -> str:
|
|
||||||
v = re.sub(r"\\+([\[\]_*])", r"\1", v) # markdown escapes, sometimes doubled
|
|
||||||
return v.replace("**", "").strip()
|
|
||||||
|
|
||||||
|
|
||||||
def is_placeholder(v: str) -> bool:
|
|
||||||
v = clean(v)
|
|
||||||
return not v or bool(PLACEHOLDER_RE.match(v))
|
|
||||||
|
|
||||||
|
|
||||||
def stage_of(raw: str) -> tuple[str | None, bool]:
|
|
||||||
"""Map a free-text status onto STATUS_ORDER -> (stage, clear).
|
|
||||||
|
|
||||||
"Tested with h4tum-chall-format" is clearly Tested. "Planning to PoC Draft"
|
|
||||||
or a half-deleted option list is not; we take the least advanced stage it
|
|
||||||
names, so nothing looks further along than it is, and call it unclear."""
|
|
||||||
if is_placeholder(raw):
|
|
||||||
return None, False
|
|
||||||
found = set()
|
|
||||||
for m in STATUS_RE.finditer(raw):
|
|
||||||
w = m.group(0).lower()
|
|
||||||
found.add("Testing Ready" if w.startswith("testing") else
|
|
||||||
"PoC Draft" if w.startswith("poc") else w.title())
|
|
||||||
if not found:
|
|
||||||
return None, False
|
|
||||||
return min(found, key=STATUS_ORDER.index), len(found) == 1
|
|
||||||
|
|
||||||
|
|
||||||
def difficulty_of(raw: str) -> str | None:
|
|
||||||
""""medium/hard", "medium-hard", "easy to medium" -> a range "medium-hard"."""
|
|
||||||
if is_placeholder(raw):
|
|
||||||
return None
|
|
||||||
m = DIFF_RANGE_RE.search(raw)
|
|
||||||
if m:
|
|
||||||
ends = sorted({m.group(1).lower(), m.group(2).lower()}, key=DIFFICULTIES.index)
|
|
||||||
return "-".join(ends) if len(ends) > 1 else ends[0]
|
|
||||||
# prose like "currently medium, maybe also an easy and a hard version": the first one
|
|
||||||
m = DIFF_RE.search(raw)
|
|
||||||
return m.group(0).lower() if m else None
|
|
||||||
|
|
||||||
|
|
||||||
def difficulty_rank(d: str | None) -> float:
|
|
||||||
if not d:
|
|
||||||
return 99
|
|
||||||
ends = d.split("-")
|
|
||||||
return sum(DIFFICULTIES.index(e) for e in ends) / len(ends)
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
|
||||||
class Challenge:
|
|
||||||
repo: Repo
|
|
||||||
name: str = ""
|
|
||||||
fields: dict = field(default_factory=dict)
|
|
||||||
blockers: list = field(default_factory=list) # from check_files()
|
|
||||||
|
|
||||||
def get(self, key: str, dash: str = "—") -> str:
|
|
||||||
v = clean(self.fields.get(key, ""))
|
|
||||||
return dash if is_placeholder(v) else v
|
|
||||||
|
|
||||||
# -- derived -----------------------------------------------------------
|
|
||||||
@property
|
|
||||||
def named(self) -> bool:
|
|
||||||
return bool(self.name) and not is_placeholder(self.name)
|
|
||||||
|
|
||||||
@property
|
|
||||||
def title(self) -> str:
|
|
||||||
return self.name if self.named else self.repo.path
|
|
||||||
|
|
||||||
@property
|
|
||||||
def touched(self) -> bool:
|
|
||||||
"""Did anyone put *any* real content in the README template?"""
|
|
||||||
return self.named or any(not is_placeholder(self.fields.get(f, "")) for f in FIELDS)
|
|
||||||
|
|
||||||
@property
|
|
||||||
def status_raw(self) -> str:
|
|
||||||
return self.get("Status")
|
|
||||||
|
|
||||||
@property
|
|
||||||
def stage(self) -> str | None:
|
|
||||||
return stage_of(self.fields.get("Status", ""))[0]
|
|
||||||
|
|
||||||
@property
|
|
||||||
def difficulty(self) -> str | None:
|
|
||||||
return difficulty_of(self.fields.get("Difficulty", ""))
|
|
||||||
|
|
||||||
@property
|
|
||||||
def category(self) -> str | None:
|
|
||||||
"""The first category named; "rev, misc" counts as rev."""
|
|
||||||
raw = self.get("Category", "")
|
|
||||||
first = CATEGORY_SPLIT_RE.split(raw)[0].strip().lower()
|
|
||||||
return first or None
|
|
||||||
|
|
||||||
@property
|
|
||||||
def claims_testable(self) -> bool:
|
|
||||||
"""The README says Testing Ready or beyond; is_testable() also asks the repo."""
|
|
||||||
return bool(self.stage) and STATUS_ORDER.index(self.stage) >= STATUS_ORDER.index("Testing Ready")
|
|
||||||
|
|
||||||
@property
|
|
||||||
def icon(self) -> str:
|
|
||||||
return BLOCKED_ICON if self.blockers else STATUS_ICON[self.stage]
|
|
||||||
|
|
||||||
@property
|
|
||||||
def has_tester(self) -> bool:
|
|
||||||
t = self.get("Tester", "")
|
|
||||||
return bool(t) and not NO_TESTER_RE.match(t)
|
|
||||||
|
|
||||||
@property
|
|
||||||
def problems(self) -> list[str]:
|
|
||||||
p = []
|
|
||||||
stage, clear = stage_of(self.fields.get("Status", ""))
|
|
||||||
if stage is None:
|
|
||||||
p.append(f"status unknown ({self.status_raw!r})" if self.status_raw != "—"
|
|
||||||
else "no status")
|
|
||||||
elif not clear:
|
|
||||||
p.append(f"status unclear ({self.status_raw!r}, counted as {stage})")
|
|
||||||
if not self.category:
|
|
||||||
p.append("no category")
|
|
||||||
if not self.difficulty:
|
|
||||||
p.append(f"difficulty unclear ({self.get('Difficulty')!r})"
|
|
||||||
if self.get("Difficulty") != "—" else "no difficulty")
|
|
||||||
if self.claims_testable and not self.has_tester:
|
|
||||||
p.append("no tester")
|
|
||||||
if not self.named:
|
|
||||||
p.append("no challenge name in README")
|
|
||||||
return p
|
|
||||||
|
|
||||||
|
|
||||||
def is_testable(c: Challenge) -> bool:
|
|
||||||
return c.claims_testable and not c.blockers
|
|
||||||
|
|
||||||
|
|
||||||
def check_files(c: Challenge) -> list[str]:
|
|
||||||
"""Reasons not to believe a Testing Ready+ status: cheap signs in the repo that
|
|
||||||
the challenge was never finished. Passing is no proof that it builds or solves."""
|
|
||||||
repo = c.repo
|
|
||||||
if repo.desc is None:
|
|
||||||
return ["no description.yml"]
|
|
||||||
try:
|
|
||||||
doc = yaml.safe_load(repo.desc)
|
|
||||||
except yaml.YAMLError as e:
|
|
||||||
return [f"description.yml does not parse ({str(e).splitlines()[0]})"]
|
|
||||||
if not isinstance(doc, dict) or len(doc) != 1 or not isinstance(next(iter(doc.values())), dict):
|
|
||||||
return ["description.yml is not a single `<id>: {...}` block"]
|
|
||||||
cid, d = next(iter(doc.items()))
|
|
||||||
flag = d.get("flag") if isinstance(d.get("flag"), dict) else {}
|
|
||||||
containers = [x for x in d.get("containers") or [] if isinstance(x, dict)]
|
|
||||||
dirs = [os.path.normpath(str(x.get("deploy_dir") or ".")) for x in containers]
|
|
||||||
b = []
|
|
||||||
|
|
||||||
template = [what for what, hit in (
|
|
||||||
(f"id {cid!r}", cid == TEMPLATE_ID),
|
|
||||||
(f"name {d.get('name')!r}", d.get("name") == TEMPLATE_NAME),
|
|
||||||
(f"author {d.get('author')!r}", d.get("author") == TEMPLATE_AUTHOR),
|
|
||||||
("flag", flag.get("content") == TEMPLATE_FLAG),
|
|
||||||
("containers", any(x.get("name") in TEMPLATE_CONTAINERS for x in containers)
|
|
||||||
or bool(TEMPLATE_DIRS & set(dirs))),
|
|
||||||
) if hit]
|
|
||||||
if template:
|
|
||||||
b.append("description.yml still has the template's " + ", ".join(template))
|
|
||||||
|
|
||||||
cat = str(d.get("category") or "").strip().lower()
|
|
||||||
readme_cats = {x.strip().lower() for x in CATEGORY_SPLIT_RE.split(c.get("Category", ""))}
|
|
||||||
if c.category and cat not in readme_cats:
|
|
||||||
b.append(f"category {cat or 'missing'} in description.yml, README says {c.get('Category')}")
|
|
||||||
|
|
||||||
content = str(flag.get("content") or "")
|
|
||||||
if not FLAG_RE.match(content):
|
|
||||||
b.append(f"flag {content} is not h4tum{{...}}" if content else "no flag in description.yml")
|
|
||||||
if flag.get("dynamic") and not any(isinstance(x.get("flag"), dict) for x in containers):
|
|
||||||
b.append("dynamic flag, but no container is handed one")
|
|
||||||
|
|
||||||
for dd in dirs:
|
|
||||||
if dd != "." and dd not in repo.files:
|
|
||||||
b.append(f"deploy_dir {dd} does not exist")
|
|
||||||
elif ("Dockerfile" if dd == "." else f"{dd}/Dockerfile") not in repo.files:
|
|
||||||
b.append(f"no Dockerfile in {dd}")
|
|
||||||
if not any(p == "solve.py" or p.endswith("/solve.py") for p in repo.files):
|
|
||||||
b.append("no solve.py")
|
|
||||||
return b
|
|
||||||
|
|
||||||
|
|
||||||
def parse_challenge(repo: Repo) -> Challenge:
|
|
||||||
c = Challenge(repo=repo)
|
|
||||||
text = repo.readme or ""
|
|
||||||
m = HEADING_RE.search(text)
|
|
||||||
if m:
|
|
||||||
c.name = clean(m.group(1))
|
|
||||||
for fm in FIELD_RE.finditer(text):
|
|
||||||
key = fm.group("key").strip().title()
|
|
||||||
if key in FIELDS and key not in c.fields:
|
|
||||||
c.fields[key] = fm.group("val")
|
|
||||||
return c
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------- report ----
|
|
||||||
|
|
||||||
BOLD, DIM, RED, YEL, GRN, CYA, RST = (
|
|
||||||
"\033[1m", "\033[2m", "\033[31m", "\033[33m", "\033[32m", "\033[36m", "\033[0m")
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
|
||||||
class Overview:
|
|
||||||
challenges: list # repos with a README somebody filled in
|
|
||||||
blank: list # repos with no README, or an untouched template
|
|
||||||
|
|
||||||
@property
|
|
||||||
def categories(self) -> list[str]:
|
|
||||||
extra = sorted({c.category or "?" for c in self.challenges} - set(CATEGORIES))
|
|
||||||
return CATEGORIES + extra
|
|
||||||
|
|
||||||
@property
|
|
||||||
def difficulties(self) -> list[str]:
|
|
||||||
seen = {c.difficulty for c in self.challenges if c.difficulty}
|
|
||||||
unknown = ["?"] if any(not c.difficulty for c in self.challenges) else []
|
|
||||||
return sorted(set(DIFFICULTIES) | seen, key=difficulty_rank) + unknown
|
|
||||||
|
|
||||||
@property
|
|
||||||
def stages(self) -> list:
|
|
||||||
"""The five stages, plus None ("?") only if some status is unreadable."""
|
|
||||||
return STATUS_ORDER + ([None] if any(not c.stage for c in self.challenges) else [])
|
|
||||||
|
|
||||||
@property
|
|
||||||
def testable(self) -> list:
|
|
||||||
"""Everything infra can already deploy and test: Testing Ready or beyond."""
|
|
||||||
return sorted((c for c in self.challenges if is_testable(c)), key=self._ready_order)
|
|
||||||
|
|
||||||
@property
|
|
||||||
def blocked(self) -> list:
|
|
||||||
"""Testing Ready or beyond according to the README, but not the repo."""
|
|
||||||
return sorted((c for c in self.challenges if c.claims_testable and c.blockers),
|
|
||||||
key=self._ready_order)
|
|
||||||
|
|
||||||
def _ready_order(self, c: Challenge) -> tuple:
|
|
||||||
return (self.categories.index(c.category or "?"),
|
|
||||||
difficulty_rank(c.difficulty), c.title.lower())
|
|
||||||
|
|
||||||
def by_category(self) -> dict[str, list]:
|
|
||||||
out = defaultdict(list)
|
|
||||||
for c in self.challenges:
|
|
||||||
out[c.category or "?"].append(c)
|
|
||||||
for chs in out.values():
|
|
||||||
chs.sort(key=lambda c: (-STATUS_ORDER.index(c.stage) if c.stage else 1,
|
|
||||||
difficulty_rank(c.difficulty), c.title.lower()))
|
|
||||||
return out
|
|
||||||
|
|
||||||
|
|
||||||
def collect(gl: GitLab) -> Overview:
|
|
||||||
chs, blank = [], []
|
|
||||||
for repo in gl.repos():
|
|
||||||
c = parse_challenge(repo)
|
|
||||||
(chs if c.touched else blank).append(c)
|
|
||||||
|
|
||||||
def check(c: Challenge) -> None:
|
|
||||||
gl.load_files(c.repo)
|
|
||||||
c.blockers = check_files(c)
|
|
||||||
|
|
||||||
with ThreadPoolExecutor(max_workers=8) as pool:
|
|
||||||
list(pool.map(check, [c for c in chs if c.claims_testable]))
|
|
||||||
return Overview(chs, sorted(blank, key=lambda c: c.repo.path.lower()))
|
|
||||||
|
|
||||||
|
|
||||||
def icons(chs: list) -> str:
|
|
||||||
return "".join(c.icon for c in chs)
|
|
||||||
|
|
||||||
|
|
||||||
def print_terminal(ov: Overview, c: bool) -> None:
|
|
||||||
def col(x): return x if c else ""
|
|
||||||
stages = ov.stages
|
|
||||||
groups = ov.by_category()
|
|
||||||
|
|
||||||
total = Counter(ch.stage for ch in ov.challenges)
|
|
||||||
print(f"\n{col(BOLD)}=== {len(ov.challenges)} challenges ==={col(RST)} "
|
|
||||||
+ " ".join(f"{STATUS_ICON[s]} {s or '?'}:{total[s]}" for s in stages if total[s]))
|
|
||||||
|
|
||||||
ready = ov.testable
|
|
||||||
print(f"\n{col(BOLD)}{col(GRN)}=== ready for infra testing ({len(ready)}) ==={col(RST)}")
|
|
||||||
for ch in ready:
|
|
||||||
print(f" {STATUS_ICON[ch.stage]} {ch.title[:34]:<35}{ch.category or '?':<10}"
|
|
||||||
f"{ch.difficulty or '?':<13}{col(DIM)}{ch.repo.web_url}{col(RST)}")
|
|
||||||
|
|
||||||
if ov.blocked:
|
|
||||||
print(f"\n{col(BOLD)}{col(RED)}=== {BLOCKED_ICON} claims {STATUS_ORDER[2]}+, "
|
|
||||||
f"but the repo disagrees ({len(ov.blocked)}) ==={col(RST)}")
|
|
||||||
for ch in ov.blocked:
|
|
||||||
print(f" {STATUS_ICON[ch.stage]} {ch.title[:34]:<35}{col(DIM)}"
|
|
||||||
+ "; ".join(ch.blockers) + col(RST))
|
|
||||||
|
|
||||||
print(f"\n{col(BOLD)}=== status by category ==={col(RST)}")
|
|
||||||
print(f" {'':<10}" + "".join(f"{s or '?':>14}" for s in stages) + f"{'total':>7}")
|
|
||||||
for cat in ov.categories:
|
|
||||||
chs = groups.get(cat, [])
|
|
||||||
cnt = Counter(ch.stage for ch in chs)
|
|
||||||
tint = "" if chs else DIM
|
|
||||||
print(f" {col(tint)}{cat:<10}" + "".join(f"{cnt[s] or '·':>14}" for s in stages)
|
|
||||||
+ f"{len(chs):>7}{col(RST)}")
|
|
||||||
|
|
||||||
print(f"\n{col(BOLD)}=== category × difficulty ==={col(RST)} "
|
|
||||||
+ " ".join(f"{STATUS_ICON[s]} {s}" for s in STATUS_ORDER) + f" {BLOCKED_ICON} blocked")
|
|
||||||
diffs = ov.difficulties
|
|
||||||
print(f" {'':<10}" + "".join(f"{d:<13}" for d in diffs))
|
|
||||||
for cat in ov.categories:
|
|
||||||
chs = groups.get(cat, [])
|
|
||||||
cells = [icons([ch for ch in chs if (ch.difficulty or "?") == d]) for d in diffs]
|
|
||||||
# emoji are two columns wide in a terminal
|
|
||||||
print(f" {cat:<10}" + "".join(f"{x or '·'}{' ' * (13 - (2 * len(x) if x else 1))}"
|
|
||||||
for x in cells))
|
|
||||||
|
|
||||||
print(f"\n{col(BOLD)}=== needs attention ==={col(RST)}")
|
|
||||||
for ch in sorted(ov.challenges, key=lambda c: c.repo.path.lower()):
|
|
||||||
if ch.problems:
|
|
||||||
print(f" {col(YEL)}{ch.repo.path:<32}{col(RST)} {col(DIM)}"
|
|
||||||
+ "; ".join(ch.problems) + col(RST))
|
|
||||||
for ch in ov.blank:
|
|
||||||
why = "no README" if ch.repo.readme is None else "README is still the blank template"
|
|
||||||
print(f" {col(RED)}{ch.repo.path:<32}{col(RST)} {col(DIM)}{why}{col(RST)}")
|
|
||||||
print()
|
|
||||||
|
|
||||||
|
|
||||||
def render_challs(ov: Overview) -> str:
|
|
||||||
"""The deployment repo's challs.txt: one clone URL per line, for everything
|
|
||||||
that is Testing Ready or beyond and not blocked."""
|
|
||||||
return "".join(f"{c.repo.ssh_url}\n" for c in ov.testable)
|
|
||||||
|
|
||||||
|
|
||||||
def challs_message(old: str, new: str, path: str) -> str:
|
|
||||||
name = lambda u: u.rsplit("/", 1)[-1].removesuffix(".git")
|
|
||||||
was, now = set(old.split()), set(new.split())
|
|
||||||
parts = [f"+{name(u)}" for u in sorted(now - was)] + [f"-{name(u)}" for u in sorted(was - now)]
|
|
||||||
return f"Update {path} from challenge status" + (f": {' '.join(parts)}" if parts else "")
|
|
||||||
|
|
||||||
|
|
||||||
def md_cell(v: str) -> str:
|
|
||||||
return v.replace("|", "\\|")
|
|
||||||
|
|
||||||
|
|
||||||
def render_hedgedoc(ov: Overview, title: str, tags: str = "") -> str:
|
|
||||||
stamp = datetime.now().astimezone().strftime("%Y-%m-%d %H:%M:%S %Z")
|
|
||||||
stages = ov.stages
|
|
||||||
groups = ov.by_category()
|
|
||||||
total = Counter(ch.stage for ch in ov.challenges)
|
|
||||||
L = []
|
|
||||||
if tags:
|
|
||||||
# HedgeDoc reads this YAML block off the top of the note
|
|
||||||
L += ["---", f"tags: {tags}", "---", ""]
|
|
||||||
L += [f"# {title}", "",
|
|
||||||
f"**Last updated:** {stamp}", "",
|
|
||||||
f"*generated by `ctf_status.py` from the README.md of every repo in the "
|
|
||||||
f"GitLab challenges group — {len(ov.challenges)} challenges*", "",
|
|
||||||
" · ".join(f"{STATUS_ICON[s]} **{s or 'unknown'}** {total[s]}"
|
|
||||||
for s in stages if total[s] or s), ""]
|
|
||||||
|
|
||||||
ready = ov.testable
|
|
||||||
L += [f"## ✅ Ready for infra testing ({len(ready)} of {len(ov.challenges)})", "",
|
|
||||||
"*every challenge at Testing Ready or beyond whose repo backs that up — "
|
|
||||||
"these can be deployed and tested against the infrastructure now*", ""]
|
|
||||||
if ready:
|
|
||||||
L += ["| | Challenge | Category | Difficulty | Status | Event | Author | Tester |",
|
|
||||||
"| --- | --- | --- | --- | --- | --- | --- | --- |"]
|
|
||||||
for ch in ready:
|
|
||||||
L.append("| " + " | ".join(md_cell(x) for x in (
|
|
||||||
STATUS_ICON[ch.stage], f"[{ch.title}]({ch.repo.web_url})",
|
|
||||||
ch.category or "?", ch.difficulty or "?", ch.stage, ch.get("Event"),
|
|
||||||
ch.get("Author"), ch.get("Tester") if ch.has_tester else "—")) + " |")
|
|
||||||
else:
|
|
||||||
L.append("*nothing yet*")
|
|
||||||
L.append("")
|
|
||||||
|
|
||||||
if ov.blocked:
|
|
||||||
L += [f"## {BLOCKED_ICON} Claims Testing Ready+, but the repo disagrees "
|
|
||||||
f"({len(ov.blocked)})", "",
|
|
||||||
"*the README says so, but description.yml or the repo shows it is not "
|
|
||||||
"finished — kept out of challs.txt until this is fixed*", ""]
|
|
||||||
for ch in ov.blocked:
|
|
||||||
L.append(f"- {STATUS_ICON[ch.stage]} [{md_cell(ch.title)}]({ch.repo.web_url}) "
|
|
||||||
f"({ch.stage}): " + "; ".join(ch.blockers))
|
|
||||||
L.append("")
|
|
||||||
|
|
||||||
L += ["## Status by category", "",
|
|
||||||
"| Category | " + " | ".join(f"{STATUS_ICON[s]} {s or '?'}" for s in stages)
|
|
||||||
+ " | total |",
|
|
||||||
"| --- |" + " ---: |" * (len(stages) + 1)]
|
|
||||||
for cat in ov.categories:
|
|
||||||
chs = groups.get(cat, [])
|
|
||||||
cnt = Counter(ch.stage for ch in chs)
|
|
||||||
L.append(f"| {cat} | " + " | ".join(str(cnt[s] or "·") for s in stages)
|
|
||||||
+ f" | **{len(chs)}** |")
|
|
||||||
L.append("| **total** | " + " | ".join(f"**{total[s]}**" for s in stages)
|
|
||||||
+ f" | **{len(ov.challenges)}** |")
|
|
||||||
|
|
||||||
diffs = ov.difficulties
|
|
||||||
L += ["", "## Category × difficulty", "",
|
|
||||||
"*one dot per challenge, coloured by status: "
|
|
||||||
+ " ".join(f"{STATUS_ICON[s]} {s}" for s in STATUS_ORDER)
|
|
||||||
+ f" {BLOCKED_ICON} blocked*", "",
|
|
||||||
"| Category | " + " | ".join(diffs) + " | total |",
|
|
||||||
"| --- |" + " --- |" * len(diffs) + " ---: |"]
|
|
||||||
for cat in ov.categories:
|
|
||||||
chs = groups.get(cat, [])
|
|
||||||
cells = [icons([ch for ch in chs if (ch.difficulty or "?") == d]) or "·" for d in diffs]
|
|
||||||
L.append(f"| {cat} | " + " | ".join(cells) + f" | **{len(chs)}** |")
|
|
||||||
dtot = Counter(ch.difficulty or "?" for ch in ov.challenges)
|
|
||||||
L.append("| **total** | " + " | ".join(f"**{dtot[d]}**" for d in diffs)
|
|
||||||
+ f" | **{len(ov.challenges)}** |")
|
|
||||||
|
|
||||||
L += ["", "## Challenges"]
|
|
||||||
for cat in ov.categories:
|
|
||||||
chs = groups.get(cat)
|
|
||||||
if not chs:
|
|
||||||
continue
|
|
||||||
L += ["", f"### {cat} ({len(chs)})", "",
|
|
||||||
"| | Challenge | Difficulty | Status | Event | Author | Tester | Last activity |",
|
|
||||||
"| --- | --- | --- | --- | --- | --- | --- | --- |"]
|
|
||||||
for ch in chs:
|
|
||||||
status = ch.stage or "?"
|
|
||||||
if ch.status_raw.lower() != status.lower():
|
|
||||||
status += f" *({ch.status_raw})*"
|
|
||||||
cat_raw = ch.get("Category", "")
|
|
||||||
name = f"[{md_cell(ch.title)}]({ch.repo.web_url})"
|
|
||||||
if cat_raw.lower() != cat:
|
|
||||||
name += f" *({cat_raw})*"
|
|
||||||
L.append("| " + " | ".join(md_cell(x) for x in (
|
|
||||||
ch.icon, name, ch.difficulty or ch.get("Difficulty"), status,
|
|
||||||
ch.get("Event"), ch.get("Author"),
|
|
||||||
ch.get("Tester") if ch.has_tester else "—", ch.repo.last_activity)) + " |")
|
|
||||||
|
|
||||||
L += ["", "## Needs attention", ""]
|
|
||||||
issues = [ch for ch in sorted(ov.challenges, key=lambda c: c.repo.path.lower()) if ch.problems]
|
|
||||||
for ch in issues:
|
|
||||||
L.append(f"- [{md_cell(ch.title)}]({ch.repo.web_url}): " + "; ".join(ch.problems))
|
|
||||||
for ch in ov.blank:
|
|
||||||
why = "no README" if ch.repo.readme is None else "README is still the blank template"
|
|
||||||
L.append(f"- [{ch.repo.path}]({ch.repo.web_url}): {why}")
|
|
||||||
if not issues and not ov.blank:
|
|
||||||
L.append("*nothing — every README is filled in. suspicious.*")
|
|
||||||
return "\n".join(L) + "\n"
|
|
||||||
|
|
||||||
|
|
||||||
# ------------------------------------------------------ hedgedoc (write) ----
|
|
||||||
|
|
||||||
# engine.io v3 packet types
|
|
||||||
EIO_OPEN, EIO_CLOSE, EIO_PING, EIO_PONG, EIO_MSG = "0", "1", "2", "3", "4"
|
|
||||||
# socket.io packet types, as the second character of an EIO_MSG packet
|
|
||||||
SIO_CONNECT, SIO_DISCONNECT, SIO_EVENT, SIO_ERROR = "0", "1", "2", "4"
|
|
||||||
|
|
||||||
|
|
||||||
def u16len(s: str) -> int:
|
|
||||||
"""Length in UTF-16 code units — what JavaScript's `String.length` counts, and
|
|
||||||
therefore what engine.io payload framing and OT operations count. Matters as
|
|
||||||
soon as the markdown holds an emoji (🔴 is one Python char but two units)."""
|
|
||||||
return len(s.encode("utf-16-le")) // 2
|
|
||||||
|
|
||||||
|
|
||||||
def encode_payload(packets: tuple) -> bytes:
|
|
||||||
return "".join(f"{u16len(p)}:{p}" for p in packets).encode("utf-8")
|
|
||||||
|
|
||||||
|
|
||||||
def decode_payload(text: str) -> list[str]:
|
|
||||||
buf = text.encode("utf-16-le")
|
|
||||||
out, i = [], 0
|
|
||||||
while i < len(buf):
|
|
||||||
j = i
|
|
||||||
while buf[j:j + 2] != b":\x00":
|
|
||||||
j += 2
|
|
||||||
if j >= len(buf):
|
|
||||||
raise RuntimeError("malformed engine.io payload")
|
|
||||||
n = int(buf[i:j].decode("utf-16-le"))
|
|
||||||
j += 2
|
|
||||||
out.append(buf[j:j + 2 * n].decode("utf-16-le"))
|
|
||||||
i = j + 2 * n
|
|
||||||
return out
|
|
||||||
|
|
||||||
|
|
||||||
class HedgeDocNote:
|
|
||||||
"""Overwrite an existing HedgeDoc 1.x note in place.
|
|
||||||
|
|
||||||
HedgeDoc 1.x has no REST endpoint that edits a note's body, so this speaks the
|
|
||||||
same engine.io/socket.io protocol the web editor uses, over XHR long-polling
|
|
||||||
(no extra dependencies beyond `requests`):
|
|
||||||
|
|
||||||
GET /<note> -> session cookie; the socket rejects us without one
|
|
||||||
GET /socket.io/ -> engine.io handshake, yields a session id
|
|
||||||
poll for 'doc' -> the note's current text and OT revision
|
|
||||||
emit 'operation' -> [-len(old), new]: delete everything, insert ours
|
|
||||||
poll for 'ack' -> the server applied it and told the other editors
|
|
||||||
|
|
||||||
The note's permission must allow us to write ("freely" for an anonymous run).
|
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(self, url: str, timeout: int = 20):
|
|
||||||
parts = urlsplit(url)
|
|
||||||
path = parts.path.rstrip("/")
|
|
||||||
if not parts.scheme or not parts.netloc or "/" not in path:
|
|
||||||
raise ValueError(f"expected a note URL like https://host/<noteId>, got {url!r}")
|
|
||||||
self.note = path.rsplit("/", 1)[-1]
|
|
||||||
prefix = path[: -len(self.note)].rstrip("/")
|
|
||||||
self.base = f"{parts.scheme}://{parts.netloc}{prefix}"
|
|
||||||
self.timeout = timeout
|
|
||||||
self.sid: str | None = None
|
|
||||||
self.s = requests.Session()
|
|
||||||
self.s.headers["Origin"] = f"{parts.scheme}://{parts.netloc}"
|
|
||||||
|
|
||||||
# -- transport ---------------------------------------------------------
|
|
||||||
def _params(self) -> dict:
|
|
||||||
p = {"EIO": "3", "transport": "polling", "noteId": self.note,
|
|
||||||
"t": format(random.getrandbits(40), "x")}
|
|
||||||
if self.sid:
|
|
||||||
p["sid"] = self.sid
|
|
||||||
return p
|
|
||||||
|
|
||||||
def _open(self) -> None:
|
|
||||||
self.sid = None
|
|
||||||
self.s.get(f"{self.base}/{self.note}", timeout=self.timeout).raise_for_status()
|
|
||||||
r = self.s.get(f"{self.base}/socket.io/", params=self._params(), timeout=self.timeout)
|
|
||||||
r.raise_for_status()
|
|
||||||
pkts = decode_payload(r.text)
|
|
||||||
if not pkts or pkts[0][:1] != EIO_OPEN:
|
|
||||||
raise RuntimeError(f"unexpected engine.io handshake: {pkts[:1]}")
|
|
||||||
self.sid = json.loads(pkts[0][1:])["sid"]
|
|
||||||
|
|
||||||
def _send(self, *packets: str) -> None:
|
|
||||||
r = self.s.post(f"{self.base}/socket.io/", params=self._params(),
|
|
||||||
data=encode_payload(packets),
|
|
||||||
headers={"Content-Type": "text/plain;charset=UTF-8"},
|
|
||||||
timeout=self.timeout)
|
|
||||||
r.raise_for_status()
|
|
||||||
|
|
||||||
def _events(self):
|
|
||||||
"""Poll once, answer keepalives, yield decoded socket.io events."""
|
|
||||||
r = self.s.get(f"{self.base}/socket.io/", params=self._params(), timeout=self.timeout)
|
|
||||||
r.raise_for_status()
|
|
||||||
for p in decode_payload(r.text):
|
|
||||||
if p == EIO_PING:
|
|
||||||
self._send(EIO_PONG)
|
|
||||||
elif p == EIO_CLOSE:
|
|
||||||
raise RuntimeError("hedgedoc closed the connection")
|
|
||||||
elif p[:1] == EIO_MSG:
|
|
||||||
kind, data = p[1:2], p[2:]
|
|
||||||
if kind == SIO_ERROR:
|
|
||||||
raise RuntimeError(f"hedgedoc refused the socket: {data}")
|
|
||||||
if kind == SIO_EVENT:
|
|
||||||
yield json.loads(data)
|
|
||||||
|
|
||||||
def _wait(self, name: str, polls: int = 8):
|
|
||||||
for _ in range(polls):
|
|
||||||
for ev in self._events():
|
|
||||||
if ev and ev[0] == name:
|
|
||||||
return ev[1] if len(ev) > 1 else None
|
|
||||||
raise RuntimeError(f"timed out waiting for {name!r} from hedgedoc")
|
|
||||||
|
|
||||||
def _close(self) -> None:
|
|
||||||
try:
|
|
||||||
self._send(EIO_MSG + SIO_DISCONNECT, EIO_CLOSE)
|
|
||||||
except requests.RequestException:
|
|
||||||
pass # we already got our ack; a tidy goodbye is a nicety
|
|
||||||
|
|
||||||
# -- api ---------------------------------------------------------------
|
|
||||||
def download(self) -> str:
|
|
||||||
r = self.s.get(f"{self.base}/{self.note}/download", timeout=self.timeout)
|
|
||||||
r.raise_for_status()
|
|
||||||
return r.content.decode("utf-8", "replace")
|
|
||||||
|
|
||||||
def replace(self, markdown: str) -> bool:
|
|
||||||
"""Replace the note body. False if it already said exactly this."""
|
|
||||||
self._open()
|
|
||||||
doc = self._wait("doc")
|
|
||||||
old, revision = doc["str"], doc["revision"]
|
|
||||||
if old == markdown:
|
|
||||||
self._close()
|
|
||||||
return False
|
|
||||||
# an OT operation is a list of: n>0 keep n chars, n<0 drop n, str insert
|
|
||||||
ops = ([-u16len(old)] if old else []) + ([markdown] if markdown else [])
|
|
||||||
self._send(EIO_MSG + SIO_EVENT
|
|
||||||
+ json.dumps(["operation", revision, ops, None], ensure_ascii=False))
|
|
||||||
self._wait("ack")
|
|
||||||
time.sleep(2) # give the realtime server a moment to flush the note to its DB
|
|
||||||
self._close()
|
|
||||||
return True
|
|
||||||
|
|
||||||
|
|
||||||
def update_hedgedoc(url: str, md: str, retries: int = 3, quiet: bool = False) -> bool:
|
|
||||||
"""Write `md` into the existing note at `url`, then read it back to be sure."""
|
|
||||||
for attempt in range(1, retries + 1):
|
|
||||||
try:
|
|
||||||
note = HedgeDocNote(url)
|
|
||||||
changed = note.replace(md)
|
|
||||||
if note.download() != md:
|
|
||||||
raise RuntimeError("note does not match what we wrote")
|
|
||||||
return changed
|
|
||||||
except (requests.RequestException, RuntimeError, ValueError, KeyError) as e:
|
|
||||||
if attempt == retries:
|
|
||||||
raise
|
|
||||||
if not quiet:
|
|
||||||
print(f"hedgedoc update failed ({e}) — retry {attempt + 1}/{retries}",
|
|
||||||
file=sys.stderr)
|
|
||||||
time.sleep(3 * attempt)
|
|
||||||
return False
|
|
||||||
|
|
||||||
|
|
||||||
def push_hedgedoc(base: str, md: str) -> str:
|
|
||||||
r = requests.post(base.rstrip("/") + "/new", data=md.encode(),
|
|
||||||
headers={"Content-Type": "text/markdown"}, allow_redirects=False)
|
|
||||||
r.raise_for_status()
|
|
||||||
loc = r.headers.get("Location", "")
|
|
||||||
return loc if loc.startswith("http") else base.rstrip("/") + loc
|
|
||||||
|
|
||||||
|
|
||||||
def single_run_lock(target: str):
|
|
||||||
"""Keep a slow or hung cron run from colliding with the next one. One lock per
|
"""Keep a slow or hung cron run from colliding with the next one. One lock per
|
||||||
target, so the daily challs.txt run and the HedgeDoc refresh can overlap."""
|
target, so the daily challs.txt run and the HedgeDoc refresh can overlap."""
|
||||||
fh = open(Path(tempfile.gettempdir()) / f"ctf_status-{target}.lock", "w")
|
fh = open(Path(tempfile.gettempdir()) / f"ctf_status-{target}.lock", "w")
|
||||||
@@ -829,7 +44,7 @@ def single_run_lock(target: str):
|
|||||||
fcntl.flock(fh, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
fcntl.flock(fh, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||||
except OSError:
|
except OSError:
|
||||||
sys.exit(f"another ctf_status.py {target} run is still going; skipping this one")
|
sys.exit(f"another ctf_status.py {target} run is still going; skipping this one")
|
||||||
return fh # held until the process exits
|
_held_locks.append(fh)
|
||||||
|
|
||||||
|
|
||||||
def main() -> None:
|
def main() -> None:
|
||||||
@@ -874,9 +89,10 @@ def main() -> None:
|
|||||||
sys.exit(str(e))
|
sys.exit(str(e))
|
||||||
if args.challs and not cfg.get("CHALLS_REPO_URL"):
|
if args.challs and not cfg.get("CHALLS_REPO_URL"):
|
||||||
sys.exit("--challs needs CHALLS_REPO_URL in the config")
|
sys.exit("--challs needs CHALLS_REPO_URL in the config")
|
||||||
# held for the lifetime of the process
|
if note_url:
|
||||||
locks = [single_run_lock(t) for t, on in (("hedgedoc", note_url), ("challs", args.challs))
|
single_run_lock("hedgedoc")
|
||||||
if on] # noqa: F841
|
if args.challs:
|
||||||
|
single_run_lock("challs")
|
||||||
|
|
||||||
gl = GitLab(cfg["GITLAB_GROUP_URL"], cfg["GITLAB_TOKEN"])
|
gl = GitLab(cfg["GITLAB_GROUP_URL"], cfg["GITLAB_TOKEN"])
|
||||||
try:
|
try:
|
||||||
|
|||||||
+6
-12
@@ -1,27 +1,21 @@
|
|||||||
#!/usr/bin/env sh
|
#!/usr/bin/env sh
|
||||||
# Install the 30-minute HedgeDoc refresh and the daily 04:00 challs.txt commit
|
# Install the 30-minute HedgeDoc refresh and the daily 04:00 challs.txt commit
|
||||||
# as systemd *user* units.
|
# as systemd *user* units.
|
||||||
# Run it as the user that owns ctf_status.env / .env, on the machine that should do the
|
# Run it as the user that owns .env, on the machine that should do the polling.
|
||||||
# polling. Re-running it is safe; it just overwrites the units.
|
# Re-running it is safe; it just overwrites the units.
|
||||||
set -eu
|
set -eu
|
||||||
|
|
||||||
dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P)
|
dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P)
|
||||||
units="${XDG_CONFIG_HOME:-$HOME/.config}/systemd/user"
|
units="${XDG_CONFIG_HOME:-$HOME/.config}/systemd/user"
|
||||||
|
env="$dir/.env"
|
||||||
|
|
||||||
[ -x "$dir/ctf_status.py" ] || { echo "no executable ctf_status.py in $dir" >&2; exit 1; }
|
[ -x "$dir/ctf_status.py" ] || { echo "no executable ctf_status.py in $dir" >&2; exit 1; }
|
||||||
# ctf_status.py reads ctf_status.env and .env; either (or both) will do
|
[ -f "$env" ] || { echo "no .env in $dir — copy .env.example to .env first" >&2; exit 1; }
|
||||||
set --
|
|
||||||
for f in "$dir/ctf_status.env" "$dir/.env"; do
|
|
||||||
if [ -f "$f" ]; then set -- "$@" "$f"; fi
|
|
||||||
done
|
|
||||||
[ $# -gt 0 ] || {
|
|
||||||
echo "no ctf_status.env or .env in $dir — copy ctf_status.env.example to one first" >&2
|
|
||||||
exit 1; }
|
|
||||||
# the timers run --update and --challs, which need these two
|
# the timers run --update and --challs, which need these two
|
||||||
for key in HEDGEDOC_NOTE CHALLS_REPO_URL; do
|
for key in HEDGEDOC_NOTE CHALLS_REPO_URL; do
|
||||||
grep -qi "^$key=..*" "$@" || { echo "set $key=... in $* first" >&2; exit 1; }
|
grep -qi "^$key=..*" "$env" || { echo "set $key=... in $env first" >&2; exit 1; }
|
||||||
done
|
done
|
||||||
chmod 600 "$@"
|
chmod 600 "$env"
|
||||||
|
|
||||||
mkdir -p "$units"
|
mkdir -p "$units"
|
||||||
for u in ctf-status.service ctf-status.timer ctf-challs.service ctf-challs.timer; do
|
for u in ctf-status.service ctf-status.timer ctf-challs.service ctf-challs.timer; do
|
||||||
|
|||||||
@@ -0,0 +1,13 @@
|
|||||||
|
"""The pieces behind ctf_status.py, in the order the data flows through them:
|
||||||
|
|
||||||
|
config.py read .env and the environment
|
||||||
|
gitlab.py list the group's repos, download README.md and description.yml
|
||||||
|
challenge.py turn a README header into a Challenge (status, category, ...)
|
||||||
|
checks.py decide what is wrong with a challenge, and whether to believe
|
||||||
|
a Testing Ready+ status <- the rules live here
|
||||||
|
overview.py run all of the above over every repo, group and sort the result
|
||||||
|
terminal.py the terminal summary
|
||||||
|
markdown.py the HedgeDoc note
|
||||||
|
challs.py challs.txt for the deployment repo
|
||||||
|
hedgedoc.py write the note into HedgeDoc
|
||||||
|
"""
|
||||||
@@ -0,0 +1,159 @@
|
|||||||
|
"""Reading a challenge's README header.
|
||||||
|
|
||||||
|
Only the template's header block is parsed:
|
||||||
|
|
||||||
|
### Challenge Name
|
||||||
|
- Category: crypto
|
||||||
|
- Difficulty: medium
|
||||||
|
- Status: Testing Ready
|
||||||
|
...
|
||||||
|
|
||||||
|
People fill it in loosely, so the values are normalised here. Whether they are
|
||||||
|
good enough is decided in checks.py.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import re
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
|
||||||
|
from .gitlab import Repo
|
||||||
|
|
||||||
|
FIELDS = ("Category", "Difficulty", "Event", "Author", "Tester", "Status")
|
||||||
|
HEADING_RE = re.compile(r"^\s{0,3}#{1,4}\s+(.+?)\s*$", re.M)
|
||||||
|
FIELD_RE = re.compile(r"^\s*[-*]\s*\**(?P<key>[A-Za-z ]+?)\**\s*:\s*(?P<val>.*?)\s*$", re.M)
|
||||||
|
# a value the author never touched: "[crypto, pwn, ...]" or "[Author Name]"
|
||||||
|
PLACEHOLDER_RE = re.compile(r"^\[.*\]$")
|
||||||
|
NO_TESTER_RE = re.compile(r"^(\?+|none|tester name|not found\b.*|tbd)(\s.*)?$", re.I)
|
||||||
|
|
||||||
|
STATUS_ORDER = ["Planning", "PoC Draft", "Testing Ready", "Tested", "Ready"]
|
||||||
|
STATUS_RE = re.compile(r"testing\s*ready|tested|ready|poc(?:\s*draft)?|planning", re.I)
|
||||||
|
STATUS_ICON = {"Planning": "⚪", "PoC Draft": "🟠", "Testing Ready": "🟡",
|
||||||
|
"Tested": "🔵", "Ready": "🟢", None: "❔"}
|
||||||
|
BLOCKED_ICON = "⛔"
|
||||||
|
CATEGORIES = ["crypto", "pwn", "web", "rev", "misc", "osint", "forensic", "hardware"]
|
||||||
|
DIFFICULTIES = ["intro", "easy", "medium", "hard"]
|
||||||
|
DIFF_RE = re.compile(r"intro|easy|medium|hard", re.I)
|
||||||
|
DIFF_RANGE_RE = re.compile(r"(intro|easy|medium|hard)\s*(?:-|–|/|\bto\b)\s*(intro|easy|medium|hard)", re.I)
|
||||||
|
CATEGORY_SPLIT_RE = re.compile(r"[,/&+]| and ")
|
||||||
|
|
||||||
|
|
||||||
|
def clean(v: str) -> str:
|
||||||
|
v = re.sub(r"\\+([\[\]_*])", r"\1", v) # markdown escapes, sometimes doubled
|
||||||
|
return v.replace("**", "").strip()
|
||||||
|
|
||||||
|
|
||||||
|
def is_placeholder(v: str) -> bool:
|
||||||
|
v = clean(v)
|
||||||
|
return not v or bool(PLACEHOLDER_RE.match(v))
|
||||||
|
|
||||||
|
|
||||||
|
def stage_of(raw: str) -> tuple[str | None, bool]:
|
||||||
|
"""Map a free-text status onto STATUS_ORDER -> (stage, clear).
|
||||||
|
|
||||||
|
"Tested with h4tum-chall-format" is clearly Tested. "Planning to PoC Draft"
|
||||||
|
or a half-deleted option list is not; we take the least advanced stage it
|
||||||
|
names, so nothing looks further along than it is, and call it unclear."""
|
||||||
|
if is_placeholder(raw):
|
||||||
|
return None, False
|
||||||
|
found = set()
|
||||||
|
for m in STATUS_RE.finditer(raw):
|
||||||
|
w = m.group(0).lower()
|
||||||
|
found.add("Testing Ready" if w.startswith("testing") else
|
||||||
|
"PoC Draft" if w.startswith("poc") else w.title())
|
||||||
|
if not found:
|
||||||
|
return None, False
|
||||||
|
return min(found, key=STATUS_ORDER.index), len(found) == 1
|
||||||
|
|
||||||
|
|
||||||
|
def difficulty_of(raw: str) -> str | None:
|
||||||
|
""""medium/hard", "medium-hard", "easy to medium" -> a range "medium-hard"."""
|
||||||
|
if is_placeholder(raw):
|
||||||
|
return None
|
||||||
|
m = DIFF_RANGE_RE.search(raw)
|
||||||
|
if m:
|
||||||
|
ends = sorted({m.group(1).lower(), m.group(2).lower()}, key=DIFFICULTIES.index)
|
||||||
|
return "-".join(ends) if len(ends) > 1 else ends[0]
|
||||||
|
# prose like "currently medium, maybe also an easy and a hard version": the first one
|
||||||
|
m = DIFF_RE.search(raw)
|
||||||
|
return m.group(0).lower() if m else None
|
||||||
|
|
||||||
|
|
||||||
|
def difficulty_rank(d: str | None) -> float:
|
||||||
|
if not d:
|
||||||
|
return 99
|
||||||
|
ends = d.split("-")
|
||||||
|
return sum(DIFFICULTIES.index(e) for e in ends) / len(ends)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class Challenge:
|
||||||
|
repo: Repo
|
||||||
|
name: str = ""
|
||||||
|
fields: dict = field(default_factory=dict)
|
||||||
|
# filled in by overview.collect(), see checks.py
|
||||||
|
problems: list = field(default_factory=list) # readme_problems()
|
||||||
|
blockers: list = field(default_factory=list) # repo_blockers()
|
||||||
|
|
||||||
|
def get(self, key: str, dash: str = "—") -> str:
|
||||||
|
v = clean(self.fields.get(key, ""))
|
||||||
|
return dash if is_placeholder(v) else v
|
||||||
|
|
||||||
|
# -- derived -----------------------------------------------------------
|
||||||
|
@property
|
||||||
|
def named(self) -> bool:
|
||||||
|
return bool(self.name) and not is_placeholder(self.name)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def title(self) -> str:
|
||||||
|
return self.name if self.named else self.repo.path
|
||||||
|
|
||||||
|
@property
|
||||||
|
def touched(self) -> bool:
|
||||||
|
"""Did anyone put *any* real content in the README template?"""
|
||||||
|
return self.named or any(not is_placeholder(self.fields.get(f, "")) for f in FIELDS)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def status_raw(self) -> str:
|
||||||
|
return self.get("Status")
|
||||||
|
|
||||||
|
@property
|
||||||
|
def stage(self) -> str | None:
|
||||||
|
return stage_of(self.fields.get("Status", ""))[0]
|
||||||
|
|
||||||
|
@property
|
||||||
|
def difficulty(self) -> str | None:
|
||||||
|
return difficulty_of(self.fields.get("Difficulty", ""))
|
||||||
|
|
||||||
|
@property
|
||||||
|
def category(self) -> str | None:
|
||||||
|
"""The first category named; "rev, misc" counts as rev."""
|
||||||
|
raw = self.get("Category", "")
|
||||||
|
first = CATEGORY_SPLIT_RE.split(raw)[0].strip().lower()
|
||||||
|
return first or None
|
||||||
|
|
||||||
|
@property
|
||||||
|
def claims_testable(self) -> bool:
|
||||||
|
"""The README says Testing Ready or beyond; checks.is_testable() also asks the repo."""
|
||||||
|
return bool(self.stage) and STATUS_ORDER.index(self.stage) >= STATUS_ORDER.index("Testing Ready")
|
||||||
|
|
||||||
|
@property
|
||||||
|
def icon(self) -> str:
|
||||||
|
return BLOCKED_ICON if self.blockers else STATUS_ICON[self.stage]
|
||||||
|
|
||||||
|
@property
|
||||||
|
def has_tester(self) -> bool:
|
||||||
|
t = self.get("Tester", "")
|
||||||
|
return bool(t) and not NO_TESTER_RE.match(t)
|
||||||
|
|
||||||
|
|
||||||
|
def parse_challenge(repo: Repo) -> Challenge:
|
||||||
|
c = Challenge(repo=repo)
|
||||||
|
text = repo.readme or ""
|
||||||
|
m = HEADING_RE.search(text)
|
||||||
|
if m:
|
||||||
|
c.name = clean(m.group(1))
|
||||||
|
for fm in FIELD_RE.finditer(text):
|
||||||
|
key = fm.group("key").strip().title()
|
||||||
|
if key in FIELDS and key not in c.fields:
|
||||||
|
c.fields[key] = fm.group("val")
|
||||||
|
return c
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
"""challs.txt for the deployment repo: what infra deploys."""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from .overview import Overview
|
||||||
|
|
||||||
|
|
||||||
|
def render_challs(ov: Overview) -> str:
|
||||||
|
"""One clone URL per line, for everything that is Testing Ready or beyond
|
||||||
|
and not blocked."""
|
||||||
|
return "".join(f"{c.repo.ssh_url}\n" for c in ov.testable)
|
||||||
|
|
||||||
|
|
||||||
|
def challs_message(old: str, new: str, path: str) -> str:
|
||||||
|
name = lambda u: u.rsplit("/", 1)[-1].removesuffix(".git")
|
||||||
|
was, now = set(old.split()), set(new.split())
|
||||||
|
parts = [f"+{name(u)}" for u in sorted(now - was)] + [f"-{name(u)}" for u in sorted(was - now)]
|
||||||
|
return f"Update {path} from challenge status" + (f": {' '.join(parts)}" if parts else "")
|
||||||
@@ -0,0 +1,198 @@
|
|||||||
|
"""What is wrong with a challenge, and whether its status is believed.
|
||||||
|
|
||||||
|
There are two kinds of finding:
|
||||||
|
|
||||||
|
* problems (readme_problems): the README header is incomplete or unclear.
|
||||||
|
They are listed under "needs attention"; nothing else happens.
|
||||||
|
|
||||||
|
* blockers (repo_blockers): the README claims Testing Ready or beyond, but the
|
||||||
|
repo shows the challenge is not finished. A blocked challenge keeps its
|
||||||
|
status in the counts, but is left out of "ready for infra testing" and out of
|
||||||
|
challs.txt, so it does not get deployed.
|
||||||
|
|
||||||
|
Every rule is one small function that returns what it found wrong, as a list
|
||||||
|
of messages (empty: nothing wrong). README_CHECKS and REPO_CHECKS list the rules
|
||||||
|
in the order their messages are shown. To add a rule, write a function and
|
||||||
|
put it in the right list.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
from dataclasses import dataclass
|
||||||
|
|
||||||
|
import yaml
|
||||||
|
|
||||||
|
from .challenge import CATEGORY_SPLIT_RE, Challenge, stage_of
|
||||||
|
from .gitlab import Repo
|
||||||
|
|
||||||
|
|
||||||
|
def is_testable(c: Challenge) -> bool:
|
||||||
|
"""Can infra deploy and test it now? The README says Testing Ready or beyond,
|
||||||
|
and the repo has no blockers."""
|
||||||
|
return c.claims_testable and not c.blockers
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------- README ----
|
||||||
|
# Run on every challenge whose README somebody filled in.
|
||||||
|
|
||||||
|
def status_is_clear(c: Challenge) -> list[str]:
|
||||||
|
stage, clear = stage_of(c.fields.get("Status", ""))
|
||||||
|
if stage is None:
|
||||||
|
return [f"status unknown ({c.status_raw!r})" if c.status_raw != "—" else "no status"]
|
||||||
|
if not clear:
|
||||||
|
return [f"status unclear ({c.status_raw!r}, counted as {stage})"]
|
||||||
|
return []
|
||||||
|
|
||||||
|
|
||||||
|
def has_category(c: Challenge) -> list[str]:
|
||||||
|
return [] if c.category else ["no category"]
|
||||||
|
|
||||||
|
|
||||||
|
def difficulty_is_clear(c: Challenge) -> list[str]:
|
||||||
|
if c.difficulty:
|
||||||
|
return []
|
||||||
|
raw = c.get("Difficulty")
|
||||||
|
return [f"difficulty unclear ({raw!r})" if raw != "—" else "no difficulty"]
|
||||||
|
|
||||||
|
|
||||||
|
def testable_has_tester(c: Challenge) -> list[str]:
|
||||||
|
"""Once a challenge is Testing Ready, somebody has to test it."""
|
||||||
|
return ["no tester"] if c.claims_testable and not c.has_tester else []
|
||||||
|
|
||||||
|
|
||||||
|
def has_name(c: Challenge) -> list[str]:
|
||||||
|
return [] if c.named else ["no challenge name in README"]
|
||||||
|
|
||||||
|
|
||||||
|
README_CHECKS = (
|
||||||
|
status_is_clear,
|
||||||
|
has_category,
|
||||||
|
difficulty_is_clear,
|
||||||
|
testable_has_tester,
|
||||||
|
has_name,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def readme_problems(c: Challenge) -> list[str]:
|
||||||
|
return [msg for check in README_CHECKS for msg in check(c)]
|
||||||
|
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------ repo ----
|
||||||
|
# Run only on challenges whose README claims Testing Ready or beyond. These are
|
||||||
|
# cheap signs that the challenge was never finished: passing them is no proof
|
||||||
|
# that it builds or solves.
|
||||||
|
|
||||||
|
FLAG_RE = re.compile(r"^h4tum\{[^{}]+\}$")
|
||||||
|
# what every repo starts with: the challenge-format example "netcat-check-multi-container"
|
||||||
|
TEMPLATE_ID = "netcat-check-multi-container"
|
||||||
|
TEMPLATE_NAME = "Netcat Check Multi Container"
|
||||||
|
TEMPLATE_AUTHOR = "jh"
|
||||||
|
TEMPLATE_FLAG = "h4tum{n3tc4t_1s_c0ol}"
|
||||||
|
TEMPLATE_CONTAINERS = {"cont1", "cont2"}
|
||||||
|
TEMPLATE_DIRS = {"container-1", "container-2"}
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class Description:
|
||||||
|
"""A repo's description.yml: a single `<id>: {...}` block."""
|
||||||
|
id: str
|
||||||
|
data: dict # everything under <id>
|
||||||
|
flag: dict # data["flag"], {} if missing
|
||||||
|
containers: list[dict] # data["containers"], {} entries dropped
|
||||||
|
deploy_dirs: list[str] # each container's deploy_dir, normalised; "." for the repo root
|
||||||
|
files: set[str] # every file and dir in the repo
|
||||||
|
|
||||||
|
|
||||||
|
class DescriptionError(Exception):
|
||||||
|
"""description.yml is missing or unusable; nothing else can be checked."""
|
||||||
|
|
||||||
|
|
||||||
|
def read_description(repo: Repo) -> Description:
|
||||||
|
if repo.desc is None:
|
||||||
|
raise DescriptionError("no description.yml")
|
||||||
|
try:
|
||||||
|
doc = yaml.safe_load(repo.desc)
|
||||||
|
except yaml.YAMLError as e:
|
||||||
|
raise DescriptionError(f"description.yml does not parse ({str(e).splitlines()[0]})") from None
|
||||||
|
if not isinstance(doc, dict) or len(doc) != 1 or not isinstance(next(iter(doc.values())), dict):
|
||||||
|
raise DescriptionError("description.yml is not a single `<id>: {...}` block")
|
||||||
|
cid, data = next(iter(doc.items()))
|
||||||
|
containers = [x for x in data.get("containers") or [] if isinstance(x, dict)]
|
||||||
|
return Description(
|
||||||
|
id=cid, data=data,
|
||||||
|
flag=data.get("flag") if isinstance(data.get("flag"), dict) else {},
|
||||||
|
containers=containers,
|
||||||
|
deploy_dirs=[os.path.normpath(str(x.get("deploy_dir") or ".")) for x in containers],
|
||||||
|
files=repo.files or set(),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def no_template_leftovers(c: Challenge, d: Description) -> list[str]:
|
||||||
|
"""description.yml was copied from the template; anything still from it means
|
||||||
|
nobody filled it in properly."""
|
||||||
|
leftovers = [what for what, hit in (
|
||||||
|
(f"id {d.id!r}", d.id == TEMPLATE_ID),
|
||||||
|
(f"name {d.data.get('name')!r}", d.data.get("name") == TEMPLATE_NAME),
|
||||||
|
(f"author {d.data.get('author')!r}", d.data.get("author") == TEMPLATE_AUTHOR),
|
||||||
|
("flag", d.flag.get("content") == TEMPLATE_FLAG),
|
||||||
|
("containers", any(x.get("name") in TEMPLATE_CONTAINERS for x in d.containers)
|
||||||
|
or bool(TEMPLATE_DIRS & set(d.deploy_dirs))),
|
||||||
|
) if hit]
|
||||||
|
return ["description.yml still has the template's " + ", ".join(leftovers)] if leftovers else []
|
||||||
|
|
||||||
|
|
||||||
|
def category_matches_readme(c: Challenge, d: Description) -> list[str]:
|
||||||
|
"""Its category must be one of the README's ("rev, misc" allows either)."""
|
||||||
|
cat = str(d.data.get("category") or "").strip().lower()
|
||||||
|
readme_cats = {x.strip().lower() for x in CATEGORY_SPLIT_RE.split(c.get("Category", ""))}
|
||||||
|
if c.category and cat not in readme_cats:
|
||||||
|
return [f"category {cat or 'missing'} in description.yml, README says {c.get('Category')}"]
|
||||||
|
return []
|
||||||
|
|
||||||
|
|
||||||
|
def flag_is_valid(c: Challenge, d: Description) -> list[str]:
|
||||||
|
"""The flag is h4tum{...}; a dynamic one needs a container to hand it to."""
|
||||||
|
b = []
|
||||||
|
content = str(d.flag.get("content") or "")
|
||||||
|
if not FLAG_RE.match(content):
|
||||||
|
b.append(f"flag {content} is not h4tum{{...}}" if content else "no flag in description.yml")
|
||||||
|
if d.flag.get("dynamic") and not any(isinstance(x.get("flag"), dict) for x in d.containers):
|
||||||
|
b.append("dynamic flag, but no container is handed one")
|
||||||
|
return b
|
||||||
|
|
||||||
|
|
||||||
|
def deploy_dirs_have_dockerfiles(c: Challenge, d: Description) -> list[str]:
|
||||||
|
b = []
|
||||||
|
for dd in d.deploy_dirs:
|
||||||
|
if dd != "." and dd not in d.files:
|
||||||
|
b.append(f"deploy_dir {dd} does not exist")
|
||||||
|
elif ("Dockerfile" if dd == "." else f"{dd}/Dockerfile") not in d.files:
|
||||||
|
b.append(f"no Dockerfile in {dd}")
|
||||||
|
return b
|
||||||
|
|
||||||
|
|
||||||
|
def has_solve_script(c: Challenge, d: Description) -> list[str]:
|
||||||
|
"""A solve.py anywhere in the repo."""
|
||||||
|
if any(p == "solve.py" or p.endswith("/solve.py") for p in d.files):
|
||||||
|
return []
|
||||||
|
return ["no solve.py"]
|
||||||
|
|
||||||
|
|
||||||
|
REPO_CHECKS = (
|
||||||
|
no_template_leftovers,
|
||||||
|
category_matches_readme,
|
||||||
|
flag_is_valid,
|
||||||
|
deploy_dirs_have_dockerfiles,
|
||||||
|
has_solve_script,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def repo_blockers(c: Challenge) -> list[str]:
|
||||||
|
"""Reasons not to believe the README's Testing Ready+ status. Needs
|
||||||
|
GitLab.load_files(c.repo) first."""
|
||||||
|
try:
|
||||||
|
d = read_description(c.repo)
|
||||||
|
except DescriptionError as e:
|
||||||
|
return [str(e)]
|
||||||
|
return [msg for check in REPO_CHECKS for msg in check(c, d)]
|
||||||
@@ -0,0 +1,35 @@
|
|||||||
|
"""Config from .env next to ctf_status.py; environment variables override it."""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
CFG_KEYS = ("GITLAB_GROUP_URL", "GITLAB_TOKEN")
|
||||||
|
OPT_KEYS = ("HEDGEDOC_NOTE", "HEDGEDOC_TAGS", "TITLE",
|
||||||
|
"CHALLS_REPO_URL", "CHALLS_PATH", "CHALLS_BRANCH")
|
||||||
|
# the names a plain `.env` tends to use for the same thing
|
||||||
|
ALIASES = {"GL_PAT": "GITLAB_TOKEN", "GITLAB_URL": "GITLAB_GROUP_URL"}
|
||||||
|
DEFAULT_TAGS = "Logistics, h4tum CTF 2026"
|
||||||
|
DEFAULT_TITLE = "CTF challenge tracking"
|
||||||
|
DEFAULT_CHALLS_PATH = "2026/challs.txt"
|
||||||
|
|
||||||
|
|
||||||
|
def load_env() -> dict:
|
||||||
|
cfg = {}
|
||||||
|
env_file = Path(__file__).parent.parent / ".env" # the repo, next to ctf_status.py
|
||||||
|
if env_file.exists():
|
||||||
|
for line in env_file.read_text().splitlines():
|
||||||
|
line = line.strip()
|
||||||
|
if not line or line.startswith("#") or "=" not in line:
|
||||||
|
continue
|
||||||
|
k, v = line.split("=", 1)
|
||||||
|
k = k.strip().upper()
|
||||||
|
cfg.setdefault(ALIASES.get(k, k), v.strip().strip("'\""))
|
||||||
|
for k in CFG_KEYS + OPT_KEYS + tuple(ALIASES):
|
||||||
|
if os.environ.get(k):
|
||||||
|
cfg[ALIASES.get(k, k)] = os.environ[k]
|
||||||
|
missing = [k for k in CFG_KEYS if not cfg.get(k)]
|
||||||
|
if missing:
|
||||||
|
sys.exit(f"missing config: {', '.join(missing)} (see .env.example)")
|
||||||
|
return cfg
|
||||||
@@ -0,0 +1,121 @@
|
|||||||
|
"""GitLab REST access: the challenge repos we read, and the one file we write."""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import base64
|
||||||
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from urllib.parse import quote, urlsplit
|
||||||
|
|
||||||
|
import requests
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class Repo:
|
||||||
|
path: str
|
||||||
|
web_url: str
|
||||||
|
ssh_url: str
|
||||||
|
last_activity: str # YYYY-MM-DD
|
||||||
|
readme: str | None # None: the repo has no README at all
|
||||||
|
id: int = 0
|
||||||
|
branch: str | None = None
|
||||||
|
# only fetched for challenges whose README claims Testing Ready or beyond
|
||||||
|
files: set[str] | None = None # every file and dir in the repo
|
||||||
|
desc: str | None = None # description.yml, None if there is none
|
||||||
|
|
||||||
|
|
||||||
|
class GitLab:
|
||||||
|
"""Access to one group, plus the one file we write elsewhere. The token needs
|
||||||
|
Project: Read (to list the group) and Repository: Read (to download files);
|
||||||
|
--challs additionally needs read + write on the deployment repo."""
|
||||||
|
|
||||||
|
def __init__(self, group_url: str, token: str, timeout: int = 20):
|
||||||
|
parts = urlsplit(group_url)
|
||||||
|
self.api = f"{parts.scheme}://{parts.netloc}/api/v4"
|
||||||
|
self.group = parts.path.strip("/")
|
||||||
|
self.timeout = timeout
|
||||||
|
self.s = requests.Session()
|
||||||
|
self.s.headers["PRIVATE-TOKEN"] = token
|
||||||
|
|
||||||
|
def _get(self, url: str, **params) -> requests.Response:
|
||||||
|
return self._req("GET", url, params=params)
|
||||||
|
|
||||||
|
def _req(self, method: str, url: str, ok: tuple = (), **kw) -> requests.Response:
|
||||||
|
r = self.s.request(method, url, timeout=self.timeout, **kw)
|
||||||
|
if r.status_code in ok:
|
||||||
|
return r
|
||||||
|
if r.status_code == 403:
|
||||||
|
# fine-grained tokens say exactly which permission is missing
|
||||||
|
try:
|
||||||
|
why = r.json().get("error_description") or r.text
|
||||||
|
except ValueError:
|
||||||
|
why = r.text
|
||||||
|
raise RuntimeError(f"gitlab refused {r.url}: {why}")
|
||||||
|
r.raise_for_status()
|
||||||
|
return r
|
||||||
|
|
||||||
|
def projects(self) -> list[dict]:
|
||||||
|
out, page = [], "1"
|
||||||
|
while page:
|
||||||
|
r = self._get(f"{self.api}/groups/{quote(self.group, safe='')}/projects",
|
||||||
|
include_subgroups="true", per_page="100", page=page,
|
||||||
|
order_by="path", sort="asc")
|
||||||
|
out += r.json()
|
||||||
|
page = r.headers.get("X-Next-Page", "")
|
||||||
|
# repos GitLab is about to delete are renamed "...-deletion_scheduled-<id>"
|
||||||
|
return [p for p in out if not p.get("archived") and not p.get("marked_for_deletion_on")]
|
||||||
|
|
||||||
|
def readme(self, p: dict) -> str | None:
|
||||||
|
branch, url = p.get("default_branch"), p.get("readme_url")
|
||||||
|
if not branch or not url or f"/-/blob/{branch}/" not in url:
|
||||||
|
return None
|
||||||
|
return self._raw(p["id"], url.split(f"/-/blob/{branch}/", 1)[1], branch)
|
||||||
|
|
||||||
|
def _raw(self, project_id: int, path: str, branch: str) -> str:
|
||||||
|
r = self._get(f"{self.api}/projects/{project_id}/repository/files/"
|
||||||
|
f"{quote(path, safe='')}/raw", ref=branch)
|
||||||
|
return r.content.decode("utf-8", "replace")
|
||||||
|
|
||||||
|
def repos(self) -> list[Repo]:
|
||||||
|
projects = self.projects()
|
||||||
|
|
||||||
|
def fetch(p: dict) -> Repo:
|
||||||
|
return Repo(path=p["path"], web_url=p["web_url"],
|
||||||
|
ssh_url=p["ssh_url_to_repo"],
|
||||||
|
last_activity=(p.get("last_activity_at") or "")[:10],
|
||||||
|
readme=self.readme(p), id=p["id"], branch=p.get("default_branch"))
|
||||||
|
|
||||||
|
with ThreadPoolExecutor(max_workers=8) as pool:
|
||||||
|
return list(pool.map(fetch, projects))
|
||||||
|
|
||||||
|
def load_files(self, repo: Repo) -> None:
|
||||||
|
"""Fill in repo.files and repo.desc, for checks.repo_blockers()."""
|
||||||
|
repo.files = set()
|
||||||
|
if not repo.branch:
|
||||||
|
return
|
||||||
|
page = "1"
|
||||||
|
while page:
|
||||||
|
r = self._get(f"{self.api}/projects/{repo.id}/repository/tree", ref=repo.branch,
|
||||||
|
recursive="true", per_page="100", page=page)
|
||||||
|
repo.files |= {t["path"] for t in r.json()}
|
||||||
|
page = r.headers.get("X-Next-Page", "")
|
||||||
|
if "description.yml" in repo.files:
|
||||||
|
repo.desc = self._raw(repo.id, "description.yml", repo.branch)
|
||||||
|
|
||||||
|
def write_file(self, repo_url: str, path: str, content: str, branch: str | None,
|
||||||
|
message) -> bool:
|
||||||
|
"""Commit `content` to `path` in the repo at `repo_url` (a web URL), creating
|
||||||
|
the file if needed; `message(old_content)` makes the commit message.
|
||||||
|
False if the file already said exactly this."""
|
||||||
|
proj = quote(urlsplit(repo_url).path.strip("/").split("/-/", 1)[0], safe="")
|
||||||
|
base = f"{self.api}/projects/{proj}"
|
||||||
|
if not branch:
|
||||||
|
branch = self._get(base).json()["default_branch"]
|
||||||
|
url = f"{base}/repository/files/{quote(path, safe='')}"
|
||||||
|
r = self._req("GET", url, ok=(404,), params={"ref": branch})
|
||||||
|
exists = r.status_code != 404
|
||||||
|
old = base64.b64decode(r.json()["content"]).decode("utf-8", "replace") if exists else ""
|
||||||
|
if exists and old == content:
|
||||||
|
return False
|
||||||
|
self._req("PUT" if exists else "POST", url,
|
||||||
|
json={"branch": branch, "content": content, "commit_message": message(old)})
|
||||||
|
return True
|
||||||
@@ -0,0 +1,180 @@
|
|||||||
|
"""Writing the note into HedgeDoc 1.x.
|
||||||
|
|
||||||
|
HedgeDoc 1.x has no REST endpoint that edits a note's body, so this pretends to
|
||||||
|
be the web editor. Nothing in here knows about challenges.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import random
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
from urllib.parse import urlsplit
|
||||||
|
|
||||||
|
import requests
|
||||||
|
|
||||||
|
# engine.io v3 packet types
|
||||||
|
EIO_OPEN, EIO_CLOSE, EIO_PING, EIO_PONG, EIO_MSG = "0", "1", "2", "3", "4"
|
||||||
|
# socket.io packet types, as the second character of an EIO_MSG packet
|
||||||
|
SIO_CONNECT, SIO_DISCONNECT, SIO_EVENT, SIO_ERROR = "0", "1", "2", "4"
|
||||||
|
|
||||||
|
|
||||||
|
def u16len(s: str) -> int:
|
||||||
|
"""Length in UTF-16 code units — what JavaScript's `String.length` counts, and
|
||||||
|
therefore what engine.io payload framing and OT operations count. Matters as
|
||||||
|
soon as the markdown holds an emoji (🔴 is one Python char but two units)."""
|
||||||
|
return len(s.encode("utf-16-le")) // 2
|
||||||
|
|
||||||
|
|
||||||
|
def encode_payload(packets: tuple) -> bytes:
|
||||||
|
return "".join(f"{u16len(p)}:{p}" for p in packets).encode("utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def decode_payload(text: str) -> list[str]:
|
||||||
|
buf = text.encode("utf-16-le")
|
||||||
|
out, i = [], 0
|
||||||
|
while i < len(buf):
|
||||||
|
j = i
|
||||||
|
while buf[j:j + 2] != b":\x00":
|
||||||
|
j += 2
|
||||||
|
if j >= len(buf):
|
||||||
|
raise RuntimeError("malformed engine.io payload")
|
||||||
|
n = int(buf[i:j].decode("utf-16-le"))
|
||||||
|
j += 2
|
||||||
|
out.append(buf[j:j + 2 * n].decode("utf-16-le"))
|
||||||
|
i = j + 2 * n
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
class HedgeDocNote:
|
||||||
|
"""Overwrite an existing HedgeDoc 1.x note in place.
|
||||||
|
|
||||||
|
Speaks the same engine.io/socket.io protocol the web editor uses, over XHR
|
||||||
|
long-polling (no extra dependencies beyond `requests`):
|
||||||
|
|
||||||
|
GET /<note> -> session cookie; the socket rejects us without one
|
||||||
|
GET /socket.io/ -> engine.io handshake, yields a session id
|
||||||
|
poll for 'doc' -> the note's current text and OT revision
|
||||||
|
emit 'operation' -> [-len(old), new]: delete everything, insert ours
|
||||||
|
poll for 'ack' -> the server applied it and told the other editors
|
||||||
|
|
||||||
|
The note's permission must allow us to write ("freely" for an anonymous run).
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, url: str, timeout: int = 20):
|
||||||
|
parts = urlsplit(url)
|
||||||
|
path = parts.path.rstrip("/")
|
||||||
|
if not parts.scheme or not parts.netloc or "/" not in path:
|
||||||
|
raise ValueError(f"expected a note URL like https://host/<noteId>, got {url!r}")
|
||||||
|
self.note = path.rsplit("/", 1)[-1]
|
||||||
|
prefix = path[: -len(self.note)].rstrip("/")
|
||||||
|
self.base = f"{parts.scheme}://{parts.netloc}{prefix}"
|
||||||
|
self.timeout = timeout
|
||||||
|
self.sid: str | None = None
|
||||||
|
self.s = requests.Session()
|
||||||
|
self.s.headers["Origin"] = f"{parts.scheme}://{parts.netloc}"
|
||||||
|
|
||||||
|
# -- transport ---------------------------------------------------------
|
||||||
|
def _params(self) -> dict:
|
||||||
|
p = {"EIO": "3", "transport": "polling", "noteId": self.note,
|
||||||
|
"t": format(random.getrandbits(40), "x")}
|
||||||
|
if self.sid:
|
||||||
|
p["sid"] = self.sid
|
||||||
|
return p
|
||||||
|
|
||||||
|
def _open(self) -> None:
|
||||||
|
self.sid = None
|
||||||
|
self.s.get(f"{self.base}/{self.note}", timeout=self.timeout).raise_for_status()
|
||||||
|
r = self.s.get(f"{self.base}/socket.io/", params=self._params(), timeout=self.timeout)
|
||||||
|
r.raise_for_status()
|
||||||
|
pkts = decode_payload(r.text)
|
||||||
|
if not pkts or pkts[0][:1] != EIO_OPEN:
|
||||||
|
raise RuntimeError(f"unexpected engine.io handshake: {pkts[:1]}")
|
||||||
|
self.sid = json.loads(pkts[0][1:])["sid"]
|
||||||
|
|
||||||
|
def _send(self, *packets: str) -> None:
|
||||||
|
r = self.s.post(f"{self.base}/socket.io/", params=self._params(),
|
||||||
|
data=encode_payload(packets),
|
||||||
|
headers={"Content-Type": "text/plain;charset=UTF-8"},
|
||||||
|
timeout=self.timeout)
|
||||||
|
r.raise_for_status()
|
||||||
|
|
||||||
|
def _events(self):
|
||||||
|
"""Poll once, answer keepalives, yield decoded socket.io events."""
|
||||||
|
r = self.s.get(f"{self.base}/socket.io/", params=self._params(), timeout=self.timeout)
|
||||||
|
r.raise_for_status()
|
||||||
|
for p in decode_payload(r.text):
|
||||||
|
if p == EIO_PING:
|
||||||
|
self._send(EIO_PONG)
|
||||||
|
elif p == EIO_CLOSE:
|
||||||
|
raise RuntimeError("hedgedoc closed the connection")
|
||||||
|
elif p[:1] == EIO_MSG:
|
||||||
|
kind, data = p[1:2], p[2:]
|
||||||
|
if kind == SIO_ERROR:
|
||||||
|
raise RuntimeError(f"hedgedoc refused the socket: {data}")
|
||||||
|
if kind == SIO_EVENT:
|
||||||
|
yield json.loads(data)
|
||||||
|
|
||||||
|
def _wait(self, name: str, polls: int = 8):
|
||||||
|
for _ in range(polls):
|
||||||
|
for ev in self._events():
|
||||||
|
if ev and ev[0] == name:
|
||||||
|
return ev[1] if len(ev) > 1 else None
|
||||||
|
raise RuntimeError(f"timed out waiting for {name!r} from hedgedoc")
|
||||||
|
|
||||||
|
def _close(self) -> None:
|
||||||
|
try:
|
||||||
|
self._send(EIO_MSG + SIO_DISCONNECT, EIO_CLOSE)
|
||||||
|
except requests.RequestException:
|
||||||
|
pass # we already got our ack; a tidy goodbye is a nicety
|
||||||
|
|
||||||
|
# -- api ---------------------------------------------------------------
|
||||||
|
def download(self) -> str:
|
||||||
|
r = self.s.get(f"{self.base}/{self.note}/download", timeout=self.timeout)
|
||||||
|
r.raise_for_status()
|
||||||
|
return r.content.decode("utf-8", "replace")
|
||||||
|
|
||||||
|
def replace(self, markdown: str) -> bool:
|
||||||
|
"""Replace the note body. False if it already said exactly this."""
|
||||||
|
self._open()
|
||||||
|
doc = self._wait("doc")
|
||||||
|
old, revision = doc["str"], doc["revision"]
|
||||||
|
if old == markdown:
|
||||||
|
self._close()
|
||||||
|
return False
|
||||||
|
# an OT operation is a list of: n>0 keep n chars, n<0 drop n, str insert
|
||||||
|
ops = ([-u16len(old)] if old else []) + ([markdown] if markdown else [])
|
||||||
|
self._send(EIO_MSG + SIO_EVENT
|
||||||
|
+ json.dumps(["operation", revision, ops, None], ensure_ascii=False))
|
||||||
|
self._wait("ack")
|
||||||
|
time.sleep(2) # give the realtime server a moment to flush the note to its DB
|
||||||
|
self._close()
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def update_hedgedoc(url: str, md: str, retries: int = 3, quiet: bool = False) -> bool:
|
||||||
|
"""Write `md` into the existing note at `url`, then read it back to be sure."""
|
||||||
|
for attempt in range(1, retries + 1):
|
||||||
|
try:
|
||||||
|
note = HedgeDocNote(url)
|
||||||
|
changed = note.replace(md)
|
||||||
|
if note.download() != md:
|
||||||
|
raise RuntimeError("note does not match what we wrote")
|
||||||
|
return changed
|
||||||
|
except (requests.RequestException, RuntimeError, ValueError, KeyError) as e:
|
||||||
|
if attempt == retries:
|
||||||
|
raise
|
||||||
|
if not quiet:
|
||||||
|
print(f"hedgedoc update failed ({e}) — retry {attempt + 1}/{retries}",
|
||||||
|
file=sys.stderr)
|
||||||
|
time.sleep(3 * attempt)
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def push_hedgedoc(base: str, md: str) -> str:
|
||||||
|
"""POST `md` as a *new* note; returns its URL."""
|
||||||
|
r = requests.post(base.rstrip("/") + "/new", data=md.encode(),
|
||||||
|
headers={"Content-Type": "text/markdown"}, allow_redirects=False)
|
||||||
|
r.raise_for_status()
|
||||||
|
loc = r.headers.get("Location", "")
|
||||||
|
return loc if loc.startswith("http") else base.rstrip("/") + loc
|
||||||
@@ -0,0 +1,114 @@
|
|||||||
|
"""The HedgeDoc note, as markdown."""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from collections import Counter
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
|
from .challenge import BLOCKED_ICON, STATUS_ICON, STATUS_ORDER
|
||||||
|
from .overview import Overview, icons
|
||||||
|
|
||||||
|
|
||||||
|
def md_cell(v: str) -> str:
|
||||||
|
return v.replace("|", "\\|")
|
||||||
|
|
||||||
|
|
||||||
|
def render_hedgedoc(ov: Overview, title: str, tags: str = "") -> str:
|
||||||
|
stamp = datetime.now().astimezone().strftime("%Y-%m-%d %H:%M:%S %Z")
|
||||||
|
stages = ov.stages
|
||||||
|
groups = ov.by_category()
|
||||||
|
total = Counter(ch.stage for ch in ov.challenges)
|
||||||
|
L = []
|
||||||
|
if tags:
|
||||||
|
# HedgeDoc reads this YAML block off the top of the note
|
||||||
|
L += ["---", f"tags: {tags}", "---", ""]
|
||||||
|
L += [f"# {title}", "",
|
||||||
|
f"**Last updated:** {stamp}", "",
|
||||||
|
f"*generated by `ctf_status.py` from the README.md of every repo in the "
|
||||||
|
f"GitLab challenges group — {len(ov.challenges)} challenges*", "",
|
||||||
|
" · ".join(f"{STATUS_ICON[s]} **{s or 'unknown'}** {total[s]}"
|
||||||
|
for s in stages if total[s] or s), ""]
|
||||||
|
|
||||||
|
ready = ov.testable
|
||||||
|
L += [f"## ✅ Ready for infra testing ({len(ready)} of {len(ov.challenges)})", "",
|
||||||
|
"*every challenge at Testing Ready or beyond whose repo backs that up — "
|
||||||
|
"these can be deployed and tested against the infrastructure now*", ""]
|
||||||
|
if ready:
|
||||||
|
L += ["| | Challenge | Category | Difficulty | Status | Event | Author | Tester |",
|
||||||
|
"| --- | --- | --- | --- | --- | --- | --- | --- |"]
|
||||||
|
for ch in ready:
|
||||||
|
L.append("| " + " | ".join(md_cell(x) for x in (
|
||||||
|
STATUS_ICON[ch.stage], f"[{ch.title}]({ch.repo.web_url})",
|
||||||
|
ch.category or "?", ch.difficulty or "?", ch.stage, ch.get("Event"),
|
||||||
|
ch.get("Author"), ch.get("Tester") if ch.has_tester else "—")) + " |")
|
||||||
|
else:
|
||||||
|
L.append("*nothing yet*")
|
||||||
|
L.append("")
|
||||||
|
|
||||||
|
if ov.blocked:
|
||||||
|
L += [f"## {BLOCKED_ICON} Claims Testing Ready+, but the repo disagrees "
|
||||||
|
f"({len(ov.blocked)})", "",
|
||||||
|
"*the README says so, but description.yml or the repo shows it is not "
|
||||||
|
"finished — kept out of challs.txt until this is fixed*", ""]
|
||||||
|
for ch in ov.blocked:
|
||||||
|
L.append(f"- {STATUS_ICON[ch.stage]} [{md_cell(ch.title)}]({ch.repo.web_url}) "
|
||||||
|
f"({ch.stage}): " + "; ".join(ch.blockers))
|
||||||
|
L.append("")
|
||||||
|
|
||||||
|
L += ["## Status by category", "",
|
||||||
|
"| Category | " + " | ".join(f"{STATUS_ICON[s]} {s or '?'}" for s in stages)
|
||||||
|
+ " | total |",
|
||||||
|
"| --- |" + " ---: |" * (len(stages) + 1)]
|
||||||
|
for cat in ov.categories:
|
||||||
|
chs = groups.get(cat, [])
|
||||||
|
cnt = Counter(ch.stage for ch in chs)
|
||||||
|
L.append(f"| {cat} | " + " | ".join(str(cnt[s] or "·") for s in stages)
|
||||||
|
+ f" | **{len(chs)}** |")
|
||||||
|
L.append("| **total** | " + " | ".join(f"**{total[s]}**" for s in stages)
|
||||||
|
+ f" | **{len(ov.challenges)}** |")
|
||||||
|
|
||||||
|
diffs = ov.difficulties
|
||||||
|
L += ["", "## Category × difficulty", "",
|
||||||
|
"*one dot per challenge, coloured by status: "
|
||||||
|
+ " ".join(f"{STATUS_ICON[s]} {s}" for s in STATUS_ORDER)
|
||||||
|
+ f" {BLOCKED_ICON} blocked*", "",
|
||||||
|
"| Category | " + " | ".join(diffs) + " | total |",
|
||||||
|
"| --- |" + " --- |" * len(diffs) + " ---: |"]
|
||||||
|
for cat in ov.categories:
|
||||||
|
chs = groups.get(cat, [])
|
||||||
|
cells = [icons([ch for ch in chs if (ch.difficulty or "?") == d]) or "·" for d in diffs]
|
||||||
|
L.append(f"| {cat} | " + " | ".join(cells) + f" | **{len(chs)}** |")
|
||||||
|
dtot = Counter(ch.difficulty or "?" for ch in ov.challenges)
|
||||||
|
L.append("| **total** | " + " | ".join(f"**{dtot[d]}**" for d in diffs)
|
||||||
|
+ f" | **{len(ov.challenges)}** |")
|
||||||
|
|
||||||
|
L += ["", "## Challenges"]
|
||||||
|
for cat in ov.categories:
|
||||||
|
chs = groups.get(cat)
|
||||||
|
if not chs:
|
||||||
|
continue
|
||||||
|
L += ["", f"### {cat} ({len(chs)})", "",
|
||||||
|
"| | Challenge | Difficulty | Status | Event | Author | Tester | Last activity |",
|
||||||
|
"| --- | --- | --- | --- | --- | --- | --- | --- |"]
|
||||||
|
for ch in chs:
|
||||||
|
status = ch.stage or "?"
|
||||||
|
if ch.status_raw.lower() != status.lower():
|
||||||
|
status += f" *({ch.status_raw})*"
|
||||||
|
cat_raw = ch.get("Category", "")
|
||||||
|
name = f"[{md_cell(ch.title)}]({ch.repo.web_url})"
|
||||||
|
if cat_raw.lower() != cat:
|
||||||
|
name += f" *({cat_raw})*"
|
||||||
|
L.append("| " + " | ".join(md_cell(x) for x in (
|
||||||
|
ch.icon, name, ch.difficulty or ch.get("Difficulty"), status,
|
||||||
|
ch.get("Event"), ch.get("Author"),
|
||||||
|
ch.get("Tester") if ch.has_tester else "—", ch.repo.last_activity)) + " |")
|
||||||
|
|
||||||
|
L += ["", "## Needs attention", ""]
|
||||||
|
issues = [ch for ch in sorted(ov.challenges, key=lambda c: c.repo.path.lower()) if ch.problems]
|
||||||
|
for ch in issues:
|
||||||
|
L.append(f"- [{md_cell(ch.title)}]({ch.repo.web_url}): " + "; ".join(ch.problems))
|
||||||
|
for ch in ov.blank:
|
||||||
|
why = "no README" if ch.repo.readme is None else "README is still the blank template"
|
||||||
|
L.append(f"- [{ch.repo.path}]({ch.repo.web_url}): {why}")
|
||||||
|
if not issues and not ov.blank:
|
||||||
|
L.append("*nothing — every README is filled in. suspicious.*")
|
||||||
|
return "\n".join(L) + "\n"
|
||||||
@@ -0,0 +1,78 @@
|
|||||||
|
"""Every challenge in the group, checked, grouped and sorted for the reports."""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from collections import defaultdict
|
||||||
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
|
from dataclasses import dataclass
|
||||||
|
|
||||||
|
from .challenge import CATEGORIES, DIFFICULTIES, STATUS_ORDER, Challenge, difficulty_rank, parse_challenge
|
||||||
|
from .checks import is_testable, readme_problems, repo_blockers
|
||||||
|
from .gitlab import GitLab
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class Overview:
|
||||||
|
challenges: list # repos with a README somebody filled in
|
||||||
|
blank: list # repos with no README, or an untouched template
|
||||||
|
|
||||||
|
@property
|
||||||
|
def categories(self) -> list[str]:
|
||||||
|
extra = sorted({c.category or "?" for c in self.challenges} - set(CATEGORIES))
|
||||||
|
return CATEGORIES + extra
|
||||||
|
|
||||||
|
@property
|
||||||
|
def difficulties(self) -> list[str]:
|
||||||
|
seen = {c.difficulty for c in self.challenges if c.difficulty}
|
||||||
|
unknown = ["?"] if any(not c.difficulty for c in self.challenges) else []
|
||||||
|
return sorted(set(DIFFICULTIES) | seen, key=difficulty_rank) + unknown
|
||||||
|
|
||||||
|
@property
|
||||||
|
def stages(self) -> list:
|
||||||
|
"""The five stages, plus None ("?") only if some status is unreadable."""
|
||||||
|
return STATUS_ORDER + ([None] if any(not c.stage for c in self.challenges) else [])
|
||||||
|
|
||||||
|
@property
|
||||||
|
def testable(self) -> list:
|
||||||
|
"""Everything infra can already deploy and test: Testing Ready or beyond."""
|
||||||
|
return sorted((c for c in self.challenges if is_testable(c)), key=self._ready_order)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def blocked(self) -> list:
|
||||||
|
"""Testing Ready or beyond according to the README, but not the repo."""
|
||||||
|
return sorted((c for c in self.challenges if c.claims_testable and c.blockers),
|
||||||
|
key=self._ready_order)
|
||||||
|
|
||||||
|
def _ready_order(self, c: Challenge) -> tuple:
|
||||||
|
return (self.categories.index(c.category or "?"),
|
||||||
|
difficulty_rank(c.difficulty), c.title.lower())
|
||||||
|
|
||||||
|
def by_category(self) -> dict[str, list]:
|
||||||
|
out = defaultdict(list)
|
||||||
|
for c in self.challenges:
|
||||||
|
out[c.category or "?"].append(c)
|
||||||
|
for chs in out.values():
|
||||||
|
chs.sort(key=lambda c: (-STATUS_ORDER.index(c.stage) if c.stage else 1,
|
||||||
|
difficulty_rank(c.difficulty), c.title.lower()))
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def collect(gl: GitLab) -> Overview:
|
||||||
|
chs, blank = [], []
|
||||||
|
for repo in gl.repos():
|
||||||
|
c = parse_challenge(repo)
|
||||||
|
(chs if c.touched else blank).append(c)
|
||||||
|
for c in chs:
|
||||||
|
c.problems = readme_problems(c)
|
||||||
|
|
||||||
|
# only a Testing Ready+ claim is worth the extra downloads to check the repo
|
||||||
|
def check(c: Challenge) -> None:
|
||||||
|
gl.load_files(c.repo)
|
||||||
|
c.blockers = repo_blockers(c)
|
||||||
|
|
||||||
|
with ThreadPoolExecutor(max_workers=8) as pool:
|
||||||
|
list(pool.map(check, [c for c in chs if c.claims_testable]))
|
||||||
|
return Overview(chs, sorted(blank, key=lambda c: c.repo.path.lower()))
|
||||||
|
|
||||||
|
|
||||||
|
def icons(chs: list) -> str:
|
||||||
|
return "".join(c.icon for c in chs)
|
||||||
@@ -0,0 +1,63 @@
|
|||||||
|
"""The summary printed when ctf_status.py runs in a terminal."""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from collections import Counter
|
||||||
|
|
||||||
|
from .challenge import BLOCKED_ICON, STATUS_ICON, STATUS_ORDER
|
||||||
|
from .overview import Overview, icons
|
||||||
|
|
||||||
|
BOLD, DIM, RED, YEL, GRN, CYA, RST = (
|
||||||
|
"\033[1m", "\033[2m", "\033[31m", "\033[33m", "\033[32m", "\033[36m", "\033[0m")
|
||||||
|
|
||||||
|
|
||||||
|
def print_terminal(ov: Overview, c: bool) -> None:
|
||||||
|
def col(x): return x if c else ""
|
||||||
|
stages = ov.stages
|
||||||
|
groups = ov.by_category()
|
||||||
|
|
||||||
|
total = Counter(ch.stage for ch in ov.challenges)
|
||||||
|
print(f"\n{col(BOLD)}=== {len(ov.challenges)} challenges ==={col(RST)} "
|
||||||
|
+ " ".join(f"{STATUS_ICON[s]} {s or '?'}:{total[s]}" for s in stages if total[s]))
|
||||||
|
|
||||||
|
ready = ov.testable
|
||||||
|
print(f"\n{col(BOLD)}{col(GRN)}=== ready for infra testing ({len(ready)}) ==={col(RST)}")
|
||||||
|
for ch in ready:
|
||||||
|
print(f" {STATUS_ICON[ch.stage]} {ch.title[:34]:<35}{ch.category or '?':<10}"
|
||||||
|
f"{ch.difficulty or '?':<13}{col(DIM)}{ch.repo.web_url}{col(RST)}")
|
||||||
|
|
||||||
|
if ov.blocked:
|
||||||
|
print(f"\n{col(BOLD)}{col(RED)}=== {BLOCKED_ICON} claims {STATUS_ORDER[2]}+, "
|
||||||
|
f"but the repo disagrees ({len(ov.blocked)}) ==={col(RST)}")
|
||||||
|
for ch in ov.blocked:
|
||||||
|
print(f" {STATUS_ICON[ch.stage]} {ch.title[:34]:<35}{col(DIM)}"
|
||||||
|
+ "; ".join(ch.blockers) + col(RST))
|
||||||
|
|
||||||
|
print(f"\n{col(BOLD)}=== status by category ==={col(RST)}")
|
||||||
|
print(f" {'':<10}" + "".join(f"{s or '?':>14}" for s in stages) + f"{'total':>7}")
|
||||||
|
for cat in ov.categories:
|
||||||
|
chs = groups.get(cat, [])
|
||||||
|
cnt = Counter(ch.stage for ch in chs)
|
||||||
|
tint = "" if chs else DIM
|
||||||
|
print(f" {col(tint)}{cat:<10}" + "".join(f"{cnt[s] or '·':>14}" for s in stages)
|
||||||
|
+ f"{len(chs):>7}{col(RST)}")
|
||||||
|
|
||||||
|
print(f"\n{col(BOLD)}=== category × difficulty ==={col(RST)} "
|
||||||
|
+ " ".join(f"{STATUS_ICON[s]} {s}" for s in STATUS_ORDER) + f" {BLOCKED_ICON} blocked")
|
||||||
|
diffs = ov.difficulties
|
||||||
|
print(f" {'':<10}" + "".join(f"{d:<13}" for d in diffs))
|
||||||
|
for cat in ov.categories:
|
||||||
|
chs = groups.get(cat, [])
|
||||||
|
cells = [icons([ch for ch in chs if (ch.difficulty or "?") == d]) for d in diffs]
|
||||||
|
# emoji are two columns wide in a terminal
|
||||||
|
print(f" {cat:<10}" + "".join(f"{x or '·'}{' ' * (13 - (2 * len(x) if x else 1))}"
|
||||||
|
for x in cells))
|
||||||
|
|
||||||
|
print(f"\n{col(BOLD)}=== needs attention ==={col(RST)}")
|
||||||
|
for ch in sorted(ov.challenges, key=lambda c: c.repo.path.lower()):
|
||||||
|
if ch.problems:
|
||||||
|
print(f" {col(YEL)}{ch.repo.path:<32}{col(RST)} {col(DIM)}"
|
||||||
|
+ "; ".join(ch.problems) + col(RST))
|
||||||
|
for ch in ov.blank:
|
||||||
|
why = "no README" if ch.repo.readme is None else "README is still the blank template"
|
||||||
|
print(f" {col(RED)}{ch.repo.path:<32}{col(RST)} {col(DIM)}{why}{col(RST)}")
|
||||||
|
print()
|
||||||
Reference in New Issue
Block a user