202 lines
7.5 KiB
Python
202 lines
7.5 KiB
Python
"""What is wrong with a challenge, and whether its status is believed.
|
|
|
|
There are two kinds of finding:
|
|
|
|
* problems (readme_problems): the README header is incomplete or unclear.
|
|
They are listed under "needs attention"; nothing else happens.
|
|
|
|
* blockers (repo_blockers): the README claims Testing Ready or beyond, but the
|
|
repo shows the challenge is not finished. A blocked challenge keeps its
|
|
status in the counts, but is left out of "ready for infra testing" and out of
|
|
challs.txt, so it does not get deployed. challenges.yml can exempt a challenge
|
|
from individual repo checks.
|
|
|
|
Every rule is one small function that returns what it found wrong, as a list
|
|
of messages (empty: nothing wrong). README_CHECKS and REPO_CHECKS list the rules
|
|
in the order their messages are shown. To add a rule, write a function and
|
|
put it in the right list.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import re
|
|
from dataclasses import dataclass
|
|
|
|
import yaml
|
|
|
|
from .challenge import CATEGORY_SPLIT_RE, Challenge, stage_of
|
|
from .gitlab import Repo
|
|
|
|
|
|
def is_testable(c: Challenge) -> bool:
|
|
"""Can infra deploy and test it now? It is included in this CTF, the README
|
|
says Testing Ready or beyond, and the repo has no blockers."""
|
|
return c.included and c.claims_testable and not c.blockers
|
|
|
|
|
|
# ---------------------------------------------------------------- README ----
|
|
# Run on every included challenge whose README somebody filled in.
|
|
|
|
def status_is_clear(c: Challenge) -> list[str]:
|
|
stage, clear = stage_of(c.fields.get("Status", ""))
|
|
if stage is None:
|
|
return [f"status unknown ({c.status_raw!r})" if c.status_raw != "—" else "no status"]
|
|
if not clear:
|
|
return [f"status unclear ({c.status_raw!r}, counted as {stage})"]
|
|
return []
|
|
|
|
|
|
def has_category(c: Challenge) -> list[str]:
|
|
return [] if c.category else ["no category"]
|
|
|
|
|
|
def difficulty_is_clear(c: Challenge) -> list[str]:
|
|
if c.difficulty:
|
|
return []
|
|
raw = c.get("Difficulty")
|
|
return [f"difficulty unclear ({raw!r})" if raw != "—" else "no difficulty"]
|
|
|
|
|
|
def testable_has_tester(c: Challenge) -> list[str]:
|
|
"""Once a challenge is Testing Ready, somebody has to test it."""
|
|
return ["no tester"] if c.claims_testable and not c.has_tester else []
|
|
|
|
|
|
def has_name(c: Challenge) -> list[str]:
|
|
return [] if c.named else ["no challenge name in README"]
|
|
|
|
|
|
README_CHECKS = (
|
|
status_is_clear,
|
|
has_category,
|
|
difficulty_is_clear,
|
|
testable_has_tester,
|
|
has_name,
|
|
)
|
|
|
|
|
|
def readme_problems(c: Challenge) -> list[str]:
|
|
return [msg for check in README_CHECKS for msg in check(c)]
|
|
|
|
|
|
# ------------------------------------------------------------------ repo ----
|
|
# Run only on challenges whose README claims Testing Ready or beyond. These are
|
|
# cheap signs that the challenge was never finished: passing them is no proof
|
|
# that it builds or solves.
|
|
|
|
FLAG_RE = re.compile(r"^h4tum\{[^{}]+\}$")
|
|
# what every repo starts with: the challenge-format example "netcat-check-multi-container"
|
|
TEMPLATE_ID = "netcat-check-multi-container"
|
|
TEMPLATE_NAME = "Netcat Check Multi Container"
|
|
TEMPLATE_AUTHOR = "jh"
|
|
TEMPLATE_FLAG = "h4tum{n3tc4t_1s_c0ol}"
|
|
TEMPLATE_CONTAINERS = {"cont1", "cont2"}
|
|
TEMPLATE_DIRS = {"container-1", "container-2"}
|
|
|
|
|
|
@dataclass
|
|
class Description:
|
|
"""A repo's description.yml: a single `<id>: {...}` block."""
|
|
id: str
|
|
data: dict # everything under <id>
|
|
flag: dict # data["flag"], {} if missing
|
|
containers: list[dict] # data["containers"], {} entries dropped
|
|
deploy_dirs: list[str] # each container's deploy_dir, normalised; "." for the repo root
|
|
files: set[str] # every file and dir in the repo
|
|
|
|
|
|
class DescriptionError(Exception):
|
|
"""description.yml is missing or unusable; nothing else can be checked."""
|
|
|
|
|
|
def read_description(repo: Repo) -> Description:
|
|
if repo.desc is None:
|
|
raise DescriptionError("no description.yml")
|
|
try:
|
|
doc = yaml.safe_load(repo.desc)
|
|
except yaml.YAMLError as e:
|
|
raise DescriptionError(f"description.yml does not parse ({str(e).splitlines()[0]})") from None
|
|
if not isinstance(doc, dict) or len(doc) != 1 or not isinstance(next(iter(doc.values())), dict):
|
|
raise DescriptionError("description.yml is not a single `<id>: {...}` block")
|
|
cid, data = next(iter(doc.items()))
|
|
containers = [x for x in data.get("containers") or [] if isinstance(x, dict)]
|
|
return Description(
|
|
id=cid, data=data,
|
|
flag=data.get("flag") if isinstance(data.get("flag"), dict) else {},
|
|
containers=containers,
|
|
deploy_dirs=[os.path.normpath(str(x.get("deploy_dir") or ".")) for x in containers],
|
|
files=repo.files or set(),
|
|
)
|
|
|
|
|
|
def no_template_leftovers(c: Challenge, d: Description) -> list[str]:
|
|
"""description.yml was copied from the template; anything still from it means
|
|
nobody filled it in properly."""
|
|
leftovers = [what for what, hit in (
|
|
(f"id {d.id!r}", d.id == TEMPLATE_ID),
|
|
(f"name {d.data.get('name')!r}", d.data.get("name") == TEMPLATE_NAME),
|
|
(f"author {d.data.get('author')!r}", d.data.get("author") == TEMPLATE_AUTHOR),
|
|
("flag", d.flag.get("content") == TEMPLATE_FLAG),
|
|
("containers", any(x.get("name") in TEMPLATE_CONTAINERS for x in d.containers)
|
|
or bool(TEMPLATE_DIRS & set(d.deploy_dirs))),
|
|
) if hit]
|
|
return ["description.yml still has the template's " + ", ".join(leftovers)] if leftovers else []
|
|
|
|
|
|
def category_matches_readme(c: Challenge, d: Description) -> list[str]:
|
|
"""Its category must be one of the README's ("rev, misc" allows either)."""
|
|
cat = str(d.data.get("category") or "").strip().lower()
|
|
readme_cats = {x.strip().lower() for x in CATEGORY_SPLIT_RE.split(c.get("Category", ""))}
|
|
if c.category and cat not in readme_cats:
|
|
return [f"category {cat or 'missing'} in description.yml, README says {c.get('Category')}"]
|
|
return []
|
|
|
|
|
|
def flag_is_valid(c: Challenge, d: Description) -> list[str]:
|
|
"""The flag is h4tum{...}; a dynamic one needs a container to hand it to."""
|
|
b = []
|
|
content = str(d.flag.get("content") or "")
|
|
if not FLAG_RE.match(content):
|
|
# never print the content: a wrong prefix or stray whitespace still leaves a real flag
|
|
b.append("flag in description.yml is not h4tum{...}" if content else "no flag in description.yml")
|
|
if d.flag.get("dynamic") and not any(isinstance(x.get("flag"), dict) for x in d.containers):
|
|
b.append("dynamic flag, but no container is handed one")
|
|
return b
|
|
|
|
|
|
def deploy_dirs_have_dockerfiles(c: Challenge, d: Description) -> list[str]:
|
|
b = []
|
|
for dd in d.deploy_dirs:
|
|
if dd != "." and dd not in d.files:
|
|
b.append(f"deploy_dir {dd} does not exist")
|
|
elif ("Dockerfile" if dd == "." else f"{dd}/Dockerfile") not in d.files:
|
|
b.append(f"no Dockerfile in {dd}")
|
|
return b
|
|
|
|
|
|
def has_solve_script(c: Challenge, d: Description) -> list[str]:
|
|
"""A solve.py anywhere in the repo."""
|
|
if any(p == "solve.py" or p.endswith("/solve.py") for p in d.files):
|
|
return []
|
|
return ["no solve.py"]
|
|
|
|
|
|
REPO_CHECKS = (
|
|
no_template_leftovers,
|
|
category_matches_readme,
|
|
flag_is_valid,
|
|
deploy_dirs_have_dockerfiles,
|
|
has_solve_script,
|
|
)
|
|
|
|
|
|
def repo_blockers(c: Challenge) -> list[str]:
|
|
"""Reasons not to believe the README's Testing Ready+ status. Needs
|
|
GitLab.load_files(c.repo) first."""
|
|
try:
|
|
d = read_description(c.repo)
|
|
except DescriptionError as e:
|
|
return [str(e)]
|
|
return [msg for check in REPO_CHECKS if check.__name__ not in c.skip_checks
|
|
for msg in check(c, d)]
|