analysis: add deterministic inventory scripts and raw data
run_all.sh reproduces every file under analysis/data/ from zero: it clones the in-scope components if missing, exports group issue metadata from git.lab, and regenerates the inventories. Reruns are diff-clean -- no wall-clock time enters an output; 'days since' is measured against the management repo's HEAD date. The management repo is inventoried at main, not at the analysis branch, so this analysis does not observe its own commits. inv_repo.py aborts the run if anything outside analysis/ was modified.
This commit is contained in:
Thore Cimbal
@@ -0,0 +1,152 @@
|
||||
"""Shared helpers for the neckbeard field-test inventory scripts.
|
||||
|
||||
Determinism rules obeyed by every consumer of this module:
|
||||
|
||||
* No wall-clock time enters any output file. Where an output needs a
|
||||
"days since" figure, it is measured against REFERENCE_DATE, which is
|
||||
the committer date of the management repo's HEAD -- a property of the
|
||||
tree under analysis, not of the moment the script runs.
|
||||
* Every collection written out is sorted by an explicit key.
|
||||
* Paths in outputs are repo-relative, never absolute.
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
|
||||
# Repo layout. WORKSPACE holds the management clone plus a components/
|
||||
# directory; override with NB_WORKSPACE when the checkout lives elsewhere.
|
||||
MGMT_REPO = Path(__file__).resolve().parents[2]
|
||||
WORKSPACE = Path(os.environ.get("NB_WORKSPACE", MGMT_REPO.parent))
|
||||
COMPONENTS_DIR = WORKSPACE / "components"
|
||||
DATA_DIR = MGMT_REPO / "analysis" / "data"
|
||||
|
||||
GITLAB_HOST = "https://git.lab"
|
||||
GROUP = "axion1337.chat"
|
||||
|
||||
# Frozen in analysis/SCOPE.md, confirmed by the human at the Phase-0 STOP.
|
||||
COMPONENTS = [
|
||||
("threadnet-call", "ThreadNet Call"),
|
||||
("thread-net-git", "ThreadNet Git"),
|
||||
("threadnet-operating", "ThreadNet Operating"),
|
||||
("axion1337.chat-gitops", "ThreadNet Server Suite"),
|
||||
("ThreadNet-Web", "ThreadNet Web"),
|
||||
]
|
||||
|
||||
# Named out of scope as analysis targets by the human. References into
|
||||
# them are still recorded, tagged points-outside-scope.
|
||||
OUT_OF_SCOPE_PROJECTS = ["game-operating", "gameserver"]
|
||||
OUT_OF_SCOPE_SUBGROUPS = ["vendor", "Archiv"]
|
||||
|
||||
# Tracked-but-vendored paths, excluded from trees and scans.
|
||||
VENDORED = re.compile(r"(^|/)(node_modules|dist|build|\.yarn|vendor)/")
|
||||
|
||||
DOC_SUFFIXES = {".md", ".rst"}
|
||||
|
||||
|
||||
# The management repo is measured at its default branch, never at the
|
||||
# analysis branch: otherwise this analysis observes its own commits and
|
||||
# its own analysis/ files and reports them as project reality.
|
||||
BASELINE_REF = {"management": "main"}
|
||||
ANALYSIS_DIR = "analysis/"
|
||||
|
||||
|
||||
def repos():
|
||||
"""(name, path) for the management repo and every in-scope component."""
|
||||
out = [("management", MGMT_REPO)]
|
||||
out += [(slug, COMPONENTS_DIR / slug) for slug, _ in COMPONENTS]
|
||||
return out
|
||||
|
||||
|
||||
def ref_of(name):
|
||||
return BASELINE_REF.get(name, "HEAD")
|
||||
|
||||
|
||||
def assert_untouched():
|
||||
"""Fail loudly if this session modified anything outside analysis/.
|
||||
|
||||
The mandate forbids changing any existing file. That is checked here
|
||||
mechanically rather than trusted, on every run.
|
||||
"""
|
||||
ref = ref_of("management")
|
||||
diff = git(MGMT_REPO, "diff", "--name-only", ref, "--", ".",
|
||||
f":(exclude){ANALYSIS_DIR}")
|
||||
untracked = git(MGMT_REPO, "ls-files", "--others", "--exclude-standard",
|
||||
"--", ".", f":(exclude){ANALYSIS_DIR}")
|
||||
offending = [p for p in (diff + untracked).split("\n") if p.strip()]
|
||||
if offending:
|
||||
raise SystemExit(
|
||||
"ABORT: this session changed files outside analysis/, which the "
|
||||
"field-test mandate forbids:\n " + "\n ".join(offending))
|
||||
|
||||
|
||||
def git(repo, *args):
|
||||
"""Run git in repo, return stdout as text. Raises on non-zero exit."""
|
||||
res = subprocess.run(
|
||||
["git", "-C", str(repo), *args],
|
||||
capture_output=True, text=True, errors="replace",
|
||||
)
|
||||
if res.returncode != 0:
|
||||
raise RuntimeError(f"git {' '.join(args)} in {repo}: {res.stderr.strip()}")
|
||||
return res.stdout
|
||||
|
||||
|
||||
def tracked_files(name, repo, skip_vendored=True):
|
||||
"""Sorted repo-relative paths tracked at the repo's baseline ref.
|
||||
|
||||
Reads the ref, not the index, so the analysis branch's own files stay
|
||||
out of the inventory.
|
||||
"""
|
||||
files = [f for f in
|
||||
git(repo, "ls-tree", "-r", "-z", "--name-only", ref_of(name)).split("\0")
|
||||
if f]
|
||||
files = [f for f in files if not f.startswith(ANALYSIS_DIR)]
|
||||
if skip_vendored:
|
||||
files = [f for f in files if not VENDORED.search(f)]
|
||||
return sorted(files)
|
||||
|
||||
|
||||
def last_commit(name, repo, path):
|
||||
"""(iso_date, author_name, short_sha) of the newest commit touching path."""
|
||||
out = git(repo, "log", "-1", "--format=%ad\t%an\t%h", "--date=short",
|
||||
ref_of(name), "--", path)
|
||||
if not out.strip():
|
||||
return ("", "", "")
|
||||
return tuple(out.strip().split("\t"))
|
||||
|
||||
|
||||
def _reference_date():
|
||||
out = git(MGMT_REPO, "log", "-1", "--format=%cd", "--date=short",
|
||||
ref_of("management"))
|
||||
y, m, d = (int(x) for x in out.strip().split("-"))
|
||||
return date(y, m, d)
|
||||
|
||||
|
||||
REFERENCE_DATE = _reference_date()
|
||||
|
||||
|
||||
def days_since(iso_date):
|
||||
"""Whole days from iso_date to REFERENCE_DATE; '' when the date is empty."""
|
||||
if not iso_date:
|
||||
return ""
|
||||
y, m, d = (int(x) for x in iso_date.split("-"))
|
||||
return (REFERENCE_DATE - date(y, m, d)).days
|
||||
|
||||
|
||||
def cell(text):
|
||||
"""Make a value safe for a TSV cell: no tabs, no newlines, trimmed."""
|
||||
return str(text).replace("\t", " ").replace("\r", " ").replace("\n", " ").strip()
|
||||
|
||||
|
||||
def write_tsv(name, header, rows):
|
||||
"""Write sorted rows to analysis/data/<name>, tab separated."""
|
||||
DATA_DIR.mkdir(parents=True, exist_ok=True)
|
||||
path = DATA_DIR / name
|
||||
with path.open("w", encoding="utf-8") as fh:
|
||||
fh.write("\t".join(header) + "\n")
|
||||
for row in rows:
|
||||
fh.write("\t".join(cell(v) for v in row) + "\n")
|
||||
print(f" {name}: {len(rows)} rows")
|
||||
return path
|
||||
@@ -0,0 +1,99 @@
|
||||
"""Export group issues from git.lab to analysis/data/gitlab_issues.json.
|
||||
|
||||
Metadata only -- titles, labels, dates, state -- never descriptions or
|
||||
comment bodies. The token is read from the path the management repo's
|
||||
CLAUDE.md sanctions (~/.config/gitlab-lab/token) and is never printed,
|
||||
logged or written anywhere.
|
||||
|
||||
All issues of the group are exported, open and closed alike: the group is
|
||||
small enough that a full export beats an arbitrary "recently closed"
|
||||
cutoff, and a full export is trivially reproducible. Issues of projects
|
||||
the human placed out of scope are exported too and tagged
|
||||
in_scope=false -- they are evidence about where work lives.
|
||||
|
||||
Skipped without failing the run when the token file is missing or git.lab
|
||||
is unreachable; run_all.sh then proceeds without the ticket dimension.
|
||||
"""
|
||||
|
||||
import json
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
from common import (
|
||||
COMPONENTS, DATA_DIR, GITLAB_HOST, GROUP, OUT_OF_SCOPE_PROJECTS,
|
||||
)
|
||||
|
||||
TOKEN_FILE = Path.home() / ".config" / "gitlab-lab" / "token"
|
||||
IN_SCOPE = {slug for slug, _ in COMPONENTS} | {"management"}
|
||||
|
||||
# Exactly the fields the analysis needs. Anything else -- above all
|
||||
# `description` -- is dropped before it reaches disk.
|
||||
FIELDS = ["id", "iid", "project_id", "title", "state", "labels",
|
||||
"created_at", "updated_at", "closed_at", "due_date"]
|
||||
|
||||
|
||||
def api(token, path):
|
||||
"""Yield every page of a GitLab list endpoint."""
|
||||
url = f"{GITLAB_HOST}/api/v4/{path}"
|
||||
while url:
|
||||
req = urllib.request.Request(url, headers={"PRIVATE-TOKEN": token})
|
||||
with urllib.request.urlopen(req, timeout=60) as resp:
|
||||
yield from json.load(resp)
|
||||
url = None
|
||||
for link in resp.headers.get("Link", "").split(","):
|
||||
if 'rel="next"' in link:
|
||||
url = link.split(";")[0].strip().strip("<>")
|
||||
|
||||
|
||||
def slim(issue, projects):
|
||||
out = {k: issue.get(k) for k in FIELDS}
|
||||
out["labels"] = sorted(out["labels"] or [])
|
||||
out["project"] = projects.get(issue["project_id"], str(issue["project_id"]))
|
||||
out["in_scope"] = out["project"] in IN_SCOPE
|
||||
milestone = issue.get("milestone")
|
||||
out["milestone"] = milestone["title"] if milestone else None
|
||||
assignee = issue.get("assignee")
|
||||
out["assignee"] = assignee["username"] if assignee else None
|
||||
return out
|
||||
|
||||
|
||||
def main():
|
||||
if not TOKEN_FILE.exists():
|
||||
print(f" gitlab_issues.json: SKIPPED, no token at {TOKEN_FILE}")
|
||||
return 0
|
||||
token = TOKEN_FILE.read_text().strip()
|
||||
|
||||
try:
|
||||
projects = {
|
||||
p["id"]: p["path"]
|
||||
for p in api(token, f"groups/{GROUP}/projects?per_page=100&archived=false")
|
||||
}
|
||||
issues = list(api(token, f"groups/{GROUP}/issues?per_page=100&state=all&scope=all"))
|
||||
except (urllib.error.URLError, TimeoutError) as exc:
|
||||
print(f" gitlab_issues.json: SKIPPED, git.lab unreachable ({exc.reason})")
|
||||
return 0
|
||||
|
||||
payload = {
|
||||
"source": f"{GITLAB_HOST}/api/v4/groups/{GROUP}/issues?state=all",
|
||||
"note": "metadata only; descriptions and comments deliberately not exported",
|
||||
"projects": {
|
||||
path: {"id": pid, "in_scope": path in IN_SCOPE,
|
||||
"declared_out_of_scope": path in OUT_OF_SCOPE_PROJECTS}
|
||||
for pid, path in sorted(projects.items(), key=lambda kv: kv[1])
|
||||
},
|
||||
"issues": sorted((slim(i, projects) for i in issues), key=lambda i: i["id"]),
|
||||
}
|
||||
|
||||
out = DATA_DIR / "gitlab_issues.json"
|
||||
DATA_DIR.mkdir(parents=True, exist_ok=True)
|
||||
out.write_text(json.dumps(payload, indent=2, ensure_ascii=False,
|
||||
sort_keys=True) + "\n", encoding="utf-8")
|
||||
print(f" gitlab_issues.json: {len(payload['issues'])} issues, "
|
||||
f"{len(projects)} projects")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,83 @@
|
||||
"""Extract checkable claims from the management repo's docs -> claims.tsv.
|
||||
|
||||
Phase 1 only *extracts*; nothing here judges whether a claim is true.
|
||||
Verification happens in Phase 2.
|
||||
|
||||
A script cannot recognise a claim semantically, so extraction is trigger
|
||||
based: a doc line is emitted when it matches at least one pattern that
|
||||
marks it as asserting something checkable about the world (a component,
|
||||
a count, a status, a version, a path, a date, the mirror topology, an
|
||||
issue). Recall is favoured over precision -- a false positive costs a
|
||||
Phase-2 glance, a false negative loses evidence.
|
||||
|
||||
Only management-repo docs are scanned: the mandate scopes claims to the
|
||||
management repo's statements about its components.
|
||||
"""
|
||||
|
||||
import re
|
||||
|
||||
from common import DOC_SUFFIXES, MGMT_REPO, cell, days_since, last_commit, \
|
||||
tracked_files, write_tsv
|
||||
|
||||
COMPONENT_WORDS = (
|
||||
r"ThreadNet[- ](?:Web|Call|Git|Operating|Server Suite)|threadnet-call|"
|
||||
r"thread-net-git|threadnet-operating|threadnet-web|axion1337\.chat-gitops|"
|
||||
r"game-operating|gameserver|management"
|
||||
)
|
||||
|
||||
TRIGGERS = [
|
||||
("component-ref", re.compile(COMPONENT_WORDS, re.I)),
|
||||
("count", re.compile(
|
||||
r"\b(ein|zwei|drei|vier|fünf|sechs|sieben|acht|neun|zehn|\d+)\s+"
|
||||
r"(Repos?|Produkt-Repos?|Projekte?|Issues?|Commits?|Pipelines?|Themes?|"
|
||||
r"Hosts?|Mirrors?)\b", re.I)),
|
||||
("status", re.compile(
|
||||
r"\b(erledigt|offen|live|aktiv|geschlossen|leer|umgezogen|entfallen|"
|
||||
r"abgelöst|veraltet|überholt|scharf|grün|rot|tabu|kanonisch|"
|
||||
r"zurückgestellt|verifiziert|bereinigt)\b", re.I)),
|
||||
("version", re.compile(r"\bv?\d+\.\d+\.\d+\b|\b\d+\.\d+\.\d+-[\w.]+\b")),
|
||||
("path-claim", re.compile(r"`[^`]*(?:/[^`]*|\.(?:md|ya?ml|json|py|ts|toml|crt))`")),
|
||||
("date-claim", re.compile(r"\b(seit|Stand|bis|am|ab)\s+\d{4}-\d{2}-\d{2}\b", re.I)),
|
||||
("mirror-topology", re.compile(
|
||||
r"\b(Mirror|Spiegel|gespiegelt|Gitea|rohana|Push-Mirror|kanonisch|Flux-Source)\b",
|
||||
re.I)),
|
||||
("issue-ref", re.compile(r"(?:#\d+\b|/-/issues/\d+|\b[A-Z]{3,8}-\d{2}\b)")),
|
||||
]
|
||||
|
||||
FENCE_RE = re.compile(r"^\s*(```|~~~)")
|
||||
|
||||
|
||||
def main():
|
||||
rows = []
|
||||
docs = [f for f in tracked_files("management", MGMT_REPO)
|
||||
if any(f.endswith(s) for s in DOC_SUFFIXES)]
|
||||
for src in sorted(docs):
|
||||
iso, _, _ = last_commit("management", MGMT_REPO, src)
|
||||
text = (MGMT_REPO / src).read_text(encoding="utf-8")
|
||||
in_fence = False
|
||||
for lineno, line in enumerate(text.splitlines(), start=1):
|
||||
if FENCE_RE.match(line):
|
||||
in_fence = not in_fence
|
||||
continue
|
||||
stripped = line.strip()
|
||||
if len(stripped) < 12:
|
||||
continue
|
||||
hits = [name for name, rx in TRIGGERS if rx.search(stripped)]
|
||||
if not hits:
|
||||
continue
|
||||
rows.append([
|
||||
src, lineno, ";".join(hits), "yes" if in_fence else "no",
|
||||
iso, days_since(iso), cell(stripped)[:400],
|
||||
])
|
||||
|
||||
rows.sort(key=lambda r: (r[0], r[1]))
|
||||
write_tsv(
|
||||
"claims.tsv",
|
||||
["path", "line", "triggers", "in_code_block", "doc_last_commit_date",
|
||||
"doc_days_since_change", "claim_text"],
|
||||
rows,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,146 @@
|
||||
"""Link inventory across all analysed repos -> analysis/data/links.tsv.
|
||||
|
||||
Statuses are the three the mandate fixes:
|
||||
|
||||
ok target verified to exist
|
||||
broken target verified to be absent
|
||||
points-outside-scope target lies outside the analysed set
|
||||
|
||||
A fourth column, `kind`, carries the nuance the status alone cannot:
|
||||
whether the target was a relative file, a git.lab issue, a repo URL, an
|
||||
anchor, or an external URL.
|
||||
|
||||
Issue URLs resolve offline against gitlab_issues.json when that file
|
||||
exists, so a rerun needs no network. Without it they are reported as
|
||||
kind=issue-url, status=points-outside-scope, note=unresolved-no-issue-data.
|
||||
"""
|
||||
|
||||
import json
|
||||
import re
|
||||
from urllib.parse import unquote, urlsplit
|
||||
|
||||
from common import (
|
||||
COMPONENTS, DATA_DIR, DOC_SUFFIXES, GROUP, OUT_OF_SCOPE_PROJECTS,
|
||||
OUT_OF_SCOPE_SUBGROUPS, repos, tracked_files, write_tsv,
|
||||
)
|
||||
|
||||
# Inline markdown links, minus image embeds; reference definitions too.
|
||||
LINK_RE = re.compile(r"(?<!\!)\[[^\]]*\]\(\s*<?([^)\s>]+)[^)]*\)")
|
||||
# A reference definition needs a target that actually looks like one.
|
||||
# Without that guard, log lines such as "[ERROR]: Task failed: ..." in
|
||||
# imported wiki dumps parse as links to a file named "Task".
|
||||
REFDEF_RE = re.compile(r"^\s{0,3}\[[^\]]+\]:\s*<?([^\s>]*[/.:#][^\s>]*)")
|
||||
|
||||
IN_SCOPE_REPOS = {slug for slug, _ in COMPONENTS} | {"management"}
|
||||
ISSUE_URL_RE = re.compile(r"^/([^/]+(?:/[^/]+)*?)/-/issues/(\d+)")
|
||||
|
||||
|
||||
def load_issue_index():
|
||||
"""(project, iid) pairs known to exist, from a previous API export."""
|
||||
path = DATA_DIR / "gitlab_issues.json"
|
||||
if not path.exists():
|
||||
return None
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
return {(i["project"], i["iid"]) for i in data["issues"]}
|
||||
|
||||
|
||||
def classify_url(target, issue_index):
|
||||
"""-> (kind, status, note) for an absolute URL."""
|
||||
parts = urlsplit(target)
|
||||
host = parts.netloc.lower()
|
||||
if host != "git.lab":
|
||||
return ("external-url", "points-outside-scope", f"host={host}")
|
||||
|
||||
m = ISSUE_URL_RE.match(parts.path)
|
||||
if m:
|
||||
ns, iid = m.group(1), int(m.group(2))
|
||||
group, _, project = ns.rpartition("/")
|
||||
if group != GROUP or project not in IN_SCOPE_REPOS:
|
||||
return ("issue-url", "points-outside-scope", f"project={ns}")
|
||||
if issue_index is None:
|
||||
return ("issue-url", "points-outside-scope", "unresolved-no-issue-data")
|
||||
if (project, iid) in issue_index:
|
||||
return ("issue-url", "ok", f"{project}#{iid}")
|
||||
return ("issue-url", "broken", f"{project}#{iid} not in group export")
|
||||
|
||||
segs = [s for s in parts.path.split("/") if s]
|
||||
if segs and segs[0] == GROUP and len(segs) >= 2:
|
||||
project = segs[1]
|
||||
if project in OUT_OF_SCOPE_PROJECTS or project in OUT_OF_SCOPE_SUBGROUPS:
|
||||
return ("repo-url", "points-outside-scope", f"declared out of scope: {project}")
|
||||
if project in IN_SCOPE_REPOS:
|
||||
return ("repo-url", "ok", f"in-scope repo {project}")
|
||||
return ("repo-url", "points-outside-scope", f"path={parts.path or '/'}")
|
||||
|
||||
|
||||
def classify_relative(repo_name, src, target, tracked):
|
||||
"""-> (kind, status, note) for a repo-relative link."""
|
||||
path = unquote(target.split("#")[0].split("?")[0])
|
||||
if not path:
|
||||
return ("anchor", "ok", "same-document anchor")
|
||||
|
||||
base = src.rsplit("/", 1)[0] if "/" in src else ""
|
||||
joined = f"{base}/{path}" if base and not path.startswith("/") else path.lstrip("/")
|
||||
|
||||
resolved, stack = [], joined.split("/")
|
||||
for seg in stack:
|
||||
if seg in ("", "."):
|
||||
continue
|
||||
if seg == "..":
|
||||
if resolved:
|
||||
resolved.pop()
|
||||
else:
|
||||
return ("relative-file", "points-outside-scope",
|
||||
"traverses above the repo root")
|
||||
else:
|
||||
resolved.append(seg)
|
||||
cand = "/".join(resolved)
|
||||
|
||||
if cand in tracked:
|
||||
return ("relative-file", "ok", cand)
|
||||
# Directory links are legitimate targets in this doc set.
|
||||
if any(t.startswith(cand + "/") for t in tracked):
|
||||
return ("relative-file", "ok", cand + "/ (directory)")
|
||||
# Extensionless doc links ("[x](playwright#anchor)") are common in
|
||||
# imported upstream docs and resolve in the renderers that serve them.
|
||||
for suffix in sorted(DOC_SUFFIXES):
|
||||
if cand + suffix in tracked:
|
||||
return ("relative-file", "ok", f"{cand}{suffix} (extension implied)")
|
||||
return ("relative-file", "broken", f"no tracked file at {cand}")
|
||||
|
||||
|
||||
def main():
|
||||
issue_index = load_issue_index()
|
||||
rows = []
|
||||
for name, repo in repos():
|
||||
tracked = set(tracked_files(name, repo))
|
||||
docs = [f for f in tracked if any(f.endswith(s) for s in DOC_SUFFIXES)]
|
||||
for src in docs:
|
||||
try:
|
||||
text = (repo / src).read_text(encoding="utf-8")
|
||||
except (OSError, UnicodeDecodeError):
|
||||
continue
|
||||
for lineno, line in enumerate(text.splitlines(), start=1):
|
||||
targets = LINK_RE.findall(line)
|
||||
ref = REFDEF_RE.match(line)
|
||||
if ref:
|
||||
targets = targets + [ref.group(1)]
|
||||
for target in targets:
|
||||
if target.startswith(("mailto:", "tel:")):
|
||||
kind, status, note = ("mailto", "points-outside-scope", "")
|
||||
elif target.startswith("#"):
|
||||
kind, status, note = ("anchor", "ok", "same-document anchor")
|
||||
elif re.match(r"^[a-zA-Z][a-zA-Z0-9+.-]*://", target):
|
||||
kind, status, note = classify_url(target, issue_index)
|
||||
else:
|
||||
kind, status, note = classify_relative(name, src, target, tracked)
|
||||
rows.append([name, src, lineno, kind, target, status, note])
|
||||
|
||||
rows.sort(key=lambda r: (r[0], r[1], r[2], r[4]))
|
||||
write_tsv("links.tsv",
|
||||
["repo", "src_path", "line", "kind", "target", "status", "note"],
|
||||
rows)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,171 @@
|
||||
"""Per-repo inventory: file trees, doc inventory, markers, git activity.
|
||||
|
||||
Writes tree_<repo>.txt, docs_inventory.tsv, markers.tsv, git_activity.tsv.
|
||||
"""
|
||||
|
||||
import re
|
||||
from collections import defaultdict
|
||||
from datetime import date
|
||||
|
||||
from common import (
|
||||
DATA_DIR, DOC_SUFFIXES, assert_untouched, days_since, git, last_commit,
|
||||
ref_of, repos, tracked_files, write_tsv,
|
||||
)
|
||||
|
||||
MARKER_RE = re.compile(r"\b(TODO|FIXME|HACK|XXX|DEPRECATED)\b")
|
||||
# git grep's ERE does not implement \b, which silently reports zero
|
||||
# markers; the scan therefore runs in Python, where \b works.
|
||||
|
||||
MAX_SCAN_BYTES = 512 * 1024
|
||||
|
||||
|
||||
def write_trees():
|
||||
for name, repo in repos():
|
||||
files = tracked_files(name, repo)
|
||||
path = DATA_DIR / f"tree_{name}.txt"
|
||||
path.write_text("\n".join(files) + "\n", encoding="utf-8")
|
||||
print(f" tree_{name}.txt: {len(files)} files")
|
||||
|
||||
|
||||
def write_docs_inventory():
|
||||
rows = []
|
||||
for name, repo in repos():
|
||||
for f in tracked_files(name, repo):
|
||||
if not any(f.endswith(s) for s in DOC_SUFFIXES):
|
||||
continue
|
||||
size = (repo / f).stat().st_size
|
||||
iso, author, sha = last_commit(name, repo, f)
|
||||
rows.append([name, f, size, iso, author, days_since(iso), sha])
|
||||
rows.sort(key=lambda r: (r[0], r[1]))
|
||||
write_tsv(
|
||||
"docs_inventory.tsv",
|
||||
["repo", "path", "bytes", "last_commit_date", "last_author",
|
||||
"days_since_change", "last_commit"],
|
||||
rows,
|
||||
)
|
||||
|
||||
|
||||
def _blame_dates(repo, path, lines_wanted):
|
||||
"""Map line number -> (author date, short sha, author) via one blame pass."""
|
||||
try:
|
||||
out = git(repo, "blame", "--line-porcelain", "--", path)
|
||||
except RuntimeError:
|
||||
return {}
|
||||
result, sha, author, when, lineno = {}, None, None, None, None
|
||||
for line in out.splitlines():
|
||||
if re.match(r"^[0-9a-f]{40} ", line):
|
||||
parts = line.split()
|
||||
sha, lineno = parts[0], int(parts[2])
|
||||
elif line.startswith("author "):
|
||||
author = line[len("author "):]
|
||||
elif line.startswith("author-time "):
|
||||
when = date.fromtimestamp(int(line[len("author-time "):])).isoformat()
|
||||
elif line.startswith("\t"):
|
||||
if lineno in lines_wanted:
|
||||
result[lineno] = (when or "", (sha or "")[:9], author or "")
|
||||
return result
|
||||
|
||||
|
||||
def write_markers():
|
||||
rows = []
|
||||
for name, repo in repos():
|
||||
for f in tracked_files(name, repo):
|
||||
full = repo / f
|
||||
try:
|
||||
if full.stat().st_size > MAX_SCAN_BYTES:
|
||||
continue
|
||||
text = full.read_text(encoding="utf-8")
|
||||
except (OSError, UnicodeDecodeError):
|
||||
continue # binary or unreadable: not a marker carrier
|
||||
hits = {}
|
||||
for i, line in enumerate(text.splitlines(), start=1):
|
||||
m = MARKER_RE.search(line)
|
||||
if m:
|
||||
hits[i] = (m.group(1), line.strip()[:200])
|
||||
if not hits:
|
||||
continue
|
||||
blame = _blame_dates(repo, f, set(hits))
|
||||
for lineno in sorted(hits):
|
||||
marker, snippet = hits[lineno]
|
||||
when, sha, author = blame.get(lineno, ("", "", ""))
|
||||
rows.append([name, f, lineno, marker, snippet, when,
|
||||
days_since(when), sha, author])
|
||||
rows.sort(key=lambda r: (r[0], r[1], r[2]))
|
||||
write_tsv(
|
||||
"markers.tsv",
|
||||
["repo", "path", "line", "marker", "text", "blame_date",
|
||||
"days_since_blame", "blame_commit", "blame_author"],
|
||||
rows,
|
||||
)
|
||||
|
||||
|
||||
def _is_doc(path):
|
||||
return any(path.endswith(s) for s in DOC_SUFFIXES)
|
||||
|
||||
|
||||
def write_git_activity():
|
||||
rows = []
|
||||
for name, repo in repos():
|
||||
log = git(repo, "log", "--format=%x01%H\t%ad", "--date=short",
|
||||
"--name-only", ref_of(name))
|
||||
per_month = defaultdict(lambda: {"commits": 0, "docs": 0, "code": 0,
|
||||
"docs_only": 0})
|
||||
commit_days = []
|
||||
for chunk in log.split("\x01"):
|
||||
if not chunk.strip():
|
||||
continue
|
||||
head, _, body = chunk.partition("\n")
|
||||
sha, iso = head.split("\t")
|
||||
files = [ln for ln in body.splitlines() if ln.strip()]
|
||||
touched_docs = any(_is_doc(f) for f in files)
|
||||
touched_code = any(not _is_doc(f) for f in files)
|
||||
bucket = per_month[iso[:7]]
|
||||
bucket["commits"] += 1
|
||||
bucket["docs"] += 1 if touched_docs else 0
|
||||
bucket["code"] += 1 if touched_code else 0
|
||||
bucket["docs_only"] += 1 if touched_docs and not touched_code else 0
|
||||
commit_days.append(iso)
|
||||
|
||||
for month in sorted(per_month):
|
||||
b = per_month[month]
|
||||
for metric in ("commits", "docs", "code", "docs_only"):
|
||||
key = {"docs": "commits_touching_docs",
|
||||
"code": "commits_touching_code",
|
||||
"docs_only": "commits_docs_only"}.get(metric, metric)
|
||||
rows.append([name, month, key, b[metric]])
|
||||
|
||||
days = sorted({d for d in commit_days})
|
||||
totals = {
|
||||
"commits": len(commit_days),
|
||||
"commits_touching_docs": sum(b["docs"] for b in per_month.values()),
|
||||
"commits_touching_code": sum(b["code"] for b in per_month.values()),
|
||||
"commits_docs_only": sum(b["docs_only"] for b in per_month.values()),
|
||||
"active_days": len(days),
|
||||
"first_commit": days[0] if days else "",
|
||||
"last_commit": days[-1] if days else "",
|
||||
"days_since_last_commit": days_since(days[-1]) if days else "",
|
||||
}
|
||||
gap, gap_from, gap_to = 0, "", ""
|
||||
for a, b in zip(days, days[1:]):
|
||||
ya, ma, da = (int(x) for x in a.split("-"))
|
||||
yb, mb, db = (int(x) for x in b.split("-"))
|
||||
delta = (date(yb, mb, db) - date(ya, ma, da)).days
|
||||
if delta > gap:
|
||||
gap, gap_from, gap_to = delta, a, b
|
||||
totals["max_gap_days"] = gap
|
||||
totals["max_gap_from"] = gap_from
|
||||
totals["max_gap_to"] = gap_to
|
||||
for metric in sorted(totals):
|
||||
rows.append([name, "ALL", metric, totals[metric]])
|
||||
|
||||
rows.sort(key=lambda r: (r[0], r[1], r[2]))
|
||||
write_tsv("git_activity.tsv", ["repo", "scope", "metric", "value"], rows)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
assert_untouched()
|
||||
DATA_DIR.mkdir(parents=True, exist_ok=True)
|
||||
write_trees()
|
||||
write_docs_inventory()
|
||||
write_markers()
|
||||
write_git_activity()
|
||||
Executable
+68
@@ -0,0 +1,68 @@
|
||||
#!/usr/bin/env bash
|
||||
# Reproduce every raw data file of the neckbeard field test from zero.
|
||||
#
|
||||
# bash analysis/scripts/run_all.sh
|
||||
#
|
||||
# Clones the in-scope component repos next to the management repo if they
|
||||
# are missing, then regenerates everything under analysis/data/.
|
||||
# Rerunning on an unchanged tree must produce no diff -- no wall-clock
|
||||
# time enters any output (see common.py).
|
||||
#
|
||||
# Requirements: bash, git, python3 (stdlib only). Network and lab access
|
||||
# are needed for the initial clone and for gitlab_issues.json; both steps
|
||||
# skip cleanly without them, and everything else still runs.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
MGMT_REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
WORKSPACE="${NB_WORKSPACE:-$(dirname "$MGMT_REPO")}"
|
||||
COMPONENTS_DIR="$WORKSPACE/components"
|
||||
SCRIPTS="$MGMT_REPO/analysis/scripts"
|
||||
|
||||
# Scope frozen in analysis/SCOPE.md, confirmed by the human at Phase 0.
|
||||
COMPONENTS=(
|
||||
threadnet-call
|
||||
thread-net-git
|
||||
threadnet-operating
|
||||
axion1337.chat-gitops
|
||||
ThreadNet-Web
|
||||
)
|
||||
|
||||
echo "management repo: $MGMT_REPO"
|
||||
echo "workspace: $WORKSPACE"
|
||||
|
||||
echo
|
||||
echo "== components"
|
||||
mkdir -p "$COMPONENTS_DIR"
|
||||
for slug in "${COMPONENTS[@]}"; do
|
||||
target="$COMPONENTS_DIR/$slug"
|
||||
if [ -d "$target/.git" ]; then
|
||||
echo " $slug: present"
|
||||
continue
|
||||
fi
|
||||
echo " $slug: cloning"
|
||||
GIT_TERMINAL_PROMPT=0 git clone -q \
|
||||
"https://git.lab/axion1337.chat/$slug.git" "$target"
|
||||
# This analysis must never write back. Break the push path on every
|
||||
# clone it creates.
|
||||
git -C "$target" remote set-url --push origin DISABLED-no-push
|
||||
done
|
||||
|
||||
echo
|
||||
echo "== tickets"
|
||||
python3 "$SCRIPTS/fetch_gitlab_issues.py"
|
||||
|
||||
echo
|
||||
echo "== repo inventory"
|
||||
python3 "$SCRIPTS/inv_repo.py"
|
||||
|
||||
echo
|
||||
echo "== links"
|
||||
python3 "$SCRIPTS/inv_links.py"
|
||||
|
||||
echo
|
||||
echo "== claims"
|
||||
python3 "$SCRIPTS/inv_claims.py"
|
||||
|
||||
echo
|
||||
echo "done. raw data in $MGMT_REPO/analysis/data/"
|
||||
Reference in New Issue
Block a user