challenge_status_tracker/tracker/checks.py

"""What is wrong with a challenge, and whether its status is believed.

There are two kinds of finding:

* problems (readme_problems): the README header is incomplete or unclear.
  They are listed under "needs attention"; nothing else happens.

* blockers (repo_blockers): the README claims Testing Ready or beyond, but the
  repo shows the challenge is not finished. A blocked challenge keeps its
  status in the counts, but is left out of "ready for infra testing" and out of
  challs.txt, so it does not get deployed.

Every rule is one small function that returns what it found wrong, as a list
of messages (empty: nothing wrong). README_CHECKS and REPO_CHECKS list the rules
in the order their messages are shown. To add a rule, write a function and
put it in the right list.
"""
from __future__ import annotations

import os
import re
from dataclasses import dataclass

import yaml

from .challenge import CATEGORY_SPLIT_RE, Challenge, stage_of
from .gitlab import Repo


def is_testable(c: Challenge) -> bool:
    """Can infra deploy and test it now? The README says Testing Ready or beyond,
    and the repo has no blockers."""
    return c.claims_testable and not c.blockers


# ---------------------------------------------------------------- README ----
# Run on every challenge whose README somebody filled in.

def status_is_clear(c: Challenge) -> list[str]:
    stage, clear = stage_of(c.fields.get("Status", ""))
    if stage is None:
        return [f"status unknown ({c.status_raw!r})" if c.status_raw != "—" else "no status"]
    if not clear:
        return [f"status unclear ({c.status_raw!r}, counted as {stage})"]
    return []


def has_category(c: Challenge) -> list[str]:
    return [] if c.category else ["no category"]


def difficulty_is_clear(c: Challenge) -> list[str]:
    if c.difficulty:
        return []
    raw = c.get("Difficulty")
    return [f"difficulty unclear ({raw!r})" if raw != "—" else "no difficulty"]


def testable_has_tester(c: Challenge) -> list[str]:
    """Once a challenge is Testing Ready, somebody has to test it."""
    return ["no tester"] if c.claims_testable and not c.has_tester else []


def has_name(c: Challenge) -> list[str]:
    return [] if c.named else ["no challenge name in README"]


README_CHECKS = (
    status_is_clear,
    has_category,
    difficulty_is_clear,
    testable_has_tester,
    has_name,
)


def readme_problems(c: Challenge) -> list[str]:
    return [msg for check in README_CHECKS for msg in check(c)]


# ------------------------------------------------------------------ repo ----
# Run only on challenges whose README claims Testing Ready or beyond. These are
# cheap signs that the challenge was never finished: passing them is no proof
# that it builds or solves.

FLAG_RE = re.compile(r"^h4tum\{[^{}]+\}$")
# what every repo starts with: the challenge-format example "netcat-check-multi-container"
TEMPLATE_ID = "netcat-check-multi-container"
TEMPLATE_NAME = "Netcat Check Multi Container"
TEMPLATE_AUTHOR = "jh"
TEMPLATE_FLAG = "h4tum{n3tc4t_1s_c0ol}"
TEMPLATE_CONTAINERS = {"cont1", "cont2"}
TEMPLATE_DIRS = {"container-1", "container-2"}


@dataclass
class Description:
    """A repo's description.yml: a single `<id>: {...}` block."""
    id: str
    data: dict              # everything under <id>
    flag: dict              # data["flag"], {} if missing
    containers: list[dict]  # data["containers"], {} entries dropped
    deploy_dirs: list[str]  # each container's deploy_dir, normalised; "." for the repo root
    files: set[str]         # every file and dir in the repo


class DescriptionError(Exception):
    """description.yml is missing or unusable; nothing else can be checked."""


def read_description(repo: Repo) -> Description:
    if repo.desc is None:
        raise DescriptionError("no description.yml")
    try:
        doc = yaml.safe_load(repo.desc)
    except yaml.YAMLError as e:
        raise DescriptionError(f"description.yml does not parse ({str(e).splitlines()[0]})") from None
    if not isinstance(doc, dict) or len(doc) != 1 or not isinstance(next(iter(doc.values())), dict):
        raise DescriptionError("description.yml is not a single `<id>: {...}` block")
    cid, data = next(iter(doc.items()))
    containers = [x for x in data.get("containers") or [] if isinstance(x, dict)]
    return Description(
        id=cid, data=data,
        flag=data.get("flag") if isinstance(data.get("flag"), dict) else {},
        containers=containers,
        deploy_dirs=[os.path.normpath(str(x.get("deploy_dir") or ".")) for x in containers],
        files=repo.files or set(),
    )


def no_template_leftovers(c: Challenge, d: Description) -> list[str]:
    """description.yml was copied from the template; anything still from it means
    nobody filled it in properly."""
    leftovers = [what for what, hit in (
        (f"id {d.id!r}", d.id == TEMPLATE_ID),
        (f"name {d.data.get('name')!r}", d.data.get("name") == TEMPLATE_NAME),
        (f"author {d.data.get('author')!r}", d.data.get("author") == TEMPLATE_AUTHOR),
        ("flag", d.flag.get("content") == TEMPLATE_FLAG),
        ("containers", any(x.get("name") in TEMPLATE_CONTAINERS for x in d.containers)
                       or bool(TEMPLATE_DIRS & set(d.deploy_dirs))),
    ) if hit]
    return ["description.yml still has the template's " + ", ".join(leftovers)] if leftovers else []


def category_matches_readme(c: Challenge, d: Description) -> list[str]:
    """Its category must be one of the README's ("rev, misc" allows either)."""
    cat = str(d.data.get("category") or "").strip().lower()
    readme_cats = {x.strip().lower() for x in CATEGORY_SPLIT_RE.split(c.get("Category", ""))}
    if c.category and cat not in readme_cats:
        return [f"category {cat or 'missing'} in description.yml, README says {c.get('Category')}"]
    return []


def flag_is_valid(c: Challenge, d: Description) -> list[str]:
    """The flag is h4tum{...}; a dynamic one needs a container to hand it to."""
    b = []
    content = str(d.flag.get("content") or "")
    if not FLAG_RE.match(content):
        b.append(f"flag {content} is not h4tum{{...}}" if content else "no flag in description.yml")
    if d.flag.get("dynamic") and not any(isinstance(x.get("flag"), dict) for x in d.containers):
        b.append("dynamic flag, but no container is handed one")
    return b


def deploy_dirs_have_dockerfiles(c: Challenge, d: Description) -> list[str]:
    b = []
    for dd in d.deploy_dirs:
        if dd != "." and dd not in d.files:
            b.append(f"deploy_dir {dd} does not exist")
        elif ("Dockerfile" if dd == "." else f"{dd}/Dockerfile") not in d.files:
            b.append(f"no Dockerfile in {dd}")
    return b


def has_solve_script(c: Challenge, d: Description) -> list[str]:
    """A solve.py anywhere in the repo."""
    if any(p == "solve.py" or p.endswith("/solve.py") for p in d.files):
        return []
    return ["no solve.py"]


REPO_CHECKS = (
    no_template_leftovers,
    category_matches_readme,
    flag_is_valid,
    deploy_dirs_have_dockerfiles,
    has_solve_script,
)


def repo_blockers(c: Challenge) -> list[str]:
    """Reasons not to believe the README's Testing Ready+ status. Needs
    GitLab.load_files(c.repo) first."""
    try:
        d = read_description(c.repo)
    except DescriptionError as e:
        return [str(e)]
    return [msg for check in REPO_CHECKS for msg in check(c, d)]