#!/usr/bin/env python3 """TFP public OpenADMET benchmark checker. Python 3.12 standard library. Only scores declared predictions and checks a reproducibility package's structure. It never executes submission code. Scientific/reproduction acceptance is a separate accountable review stage. MIT; see the campaign's benchmark skill. """ import csv import hashlib import io import json import math import os from pathlib import Path, PurePosixPath import stat import zipfile class InvalidSubmission(Exception): pass def finite(value, label, nonnegative=False): try: number = float(value) except (ValueError, TypeError, OverflowError): raise InvalidSubmission(label + " must be numeric") if not math.isfinite(number) or (nonnegative and number < 0): raise InvalidSubmission(label + " must be finite" + (" and nonnegative" if nonnegative else "")) return number def table(data, expected, label, max_rows=100000): try: text = data.decode("utf-8-sig") if "\x00" in text: raise ValueError("NUL byte") reader = csv.DictReader(io.StringIO(text, newline=""), strict=True) if reader.fieldnames != expected: raise InvalidSubmission(label + " columns must be exactly " + repr(expected)) result = {} for row in reader: if len(result) >= max_rows or None in row or None in row.values(): raise InvalidSubmission(label + " has an invalid row or too many rows") key = row["molecule_id"] if not key or key != key.strip() or key in result: raise InvalidSubmission(label + " has a blank, padded, or duplicate molecule_id") result[key] = row if not result: raise InvalidSubmission(label + " must not be empty") return result except (UnicodeError, csv.Error, ValueError) as exc: raise InvalidSubmission(label + " is not valid UTF-8 CSV: " + str(exc)) def json_object(data, label): def unique(pairs): value = {} for key, item in pairs: if key in value: raise ValueError("duplicate JSON key: " + key) value[key] = item return value try: value = json.loads(data.decode("utf-8"), object_pairs_hook=unique, parse_constant=lambda x: (_ for _ in ()).throw(ValueError(x))) if not isinstance(value, dict): raise ValueError("expected an object") return value except (UnicodeError, ValueError) as exc: raise InvalidSubmission(label + " is not valid JSON: " + str(exc)) def validate_method(data, config, train_hash): report = json_object(data, "method") expected = { "schema_version": "tfp.openadmet.method/v1", "task_id": config["task_id"], "training_data_policy": "bound_train_only", "evaluation_label_use": "scoring_only", "pretrained_model_used": False, "prediction_units": config["prediction_units"], "training_file_sha256": train_hash, } for key, value in expected.items(): if key not in report or report[key] != value or (key == "pretrained_model_used" and report[key] is not False): raise InvalidSubmission("method." + key + " disagrees with the benchmark policy or reference") for key in ["model_name", "training_description", "validation_description", "reproduction_command", "environment", "compute", "agent_assistance", "limitations", "code_license"]: if not isinstance(report.get(key), str) or not report[key].strip() or len(report[key]) > 20000: raise InvalidSubmission("method." + key + " must be a nonempty string of at most 20000 characters") if not isinstance(report.get("seeds"), list) or not report["seeds"] or len(report["seeds"]) > 100: raise InvalidSubmission("method.seeds must contain 1 to 100 integer seeds") if any(type(seed) is not int or not 0 <= seed < 2**32 for seed in report["seeds"]): raise InvalidSubmission("method.seeds entries must be unsigned 32-bit integers") # Claims are declarations, not evidence of scientific independence. return report def validate_archive(data): try: with zipfile.ZipFile(io.BytesIO(data)) as archive: entries = archive.infolist() if not 1 <= len(entries) <= 200: raise InvalidSubmission("reproduction ZIP must contain 1 to 200 entries") seen = set() total = 0 for item in entries: name = item.filename parts = PurePosixPath(name).parts if (not name or "\\" in name or "\x00" in name or name.startswith("/") or any(part in ("..", ".") for part in name.split("/") if part) or ":" in name or name in seen or len(name) > 240): raise InvalidSubmission("reproduction ZIP contains unsafe or duplicate paths") seen.add(name) mode = item.external_attr >> 16 if stat.S_ISLNK(mode) or (stat.S_IFMT(mode) not in (0, stat.S_IFREG, stat.S_IFDIR)): raise InvalidSubmission("reproduction ZIP may contain only regular files and directories") if item.flag_bits & 1 or item.compress_type not in (zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED): raise InvalidSubmission("reproduction ZIP must be unencrypted with stored/deflate compression") total += item.file_size if item.file_size > 32 * 1024**2 or total > 64 * 1024**2: raise InvalidSubmission("reproduction ZIP exceeds the 32 MiB/file or 64 MiB expanded limit") # Read and CRC-check every member, but never extract or execute it. actual = 0 with archive.open(item) as handle: while True: chunk = handle.read(65536) if not chunk: break actual += len(chunk) if actual > item.file_size or actual > 32 * 1024**2: raise InvalidSubmission("reproduction ZIP has an inconsistent expanded size") for required in ("README.md", "run.py", "requirements.txt", "LICENSE"): if required not in seen or archive.getinfo(required).is_dir(): raise InvalidSubmission("reproduction ZIP is missing root file " + required) return {"files": len(entries), "uncompressed_bytes": total} except (zipfile.BadZipFile, RuntimeError, NotImplementedError, EOFError) as exc: raise InvalidSubmission("reproduction artifact is not a supported ZIP: " + str(exc)) def score_predictions(config, tests, labels, predictions): if set(predictions) != set(tests): missing = len(set(tests) - set(predictions)) extra = len(set(predictions) - set(tests)) raise InvalidSubmission(f"prediction IDs must match test exactly ({missing} missing, {extra} extra)") if set(labels) != set(tests): raise RuntimeError("trusted label/test IDs disagree") per_endpoint = {} for endpoint in config["targets"]: errors = [] values = {} for key in tests: values[key] = finite(predictions[key][endpoint], endpoint + " prediction", config["nonnegative"]) if predictions[key]["smiles"] != tests[key]["smiles"]: raise InvalidSubmission("prediction SMILES must match test strings exactly: " + key) if labels[key]["smiles"] != tests[key]["smiles"]: raise RuntimeError("trusted label/test SMILES disagree") for key in sorted(tests): raw = labels[key][endpoint] if raw == "": continue # Sparse assay values are never imputed as zero. actual = finite(raw, "trusted label", config["nonnegative"]) predicted = values[key] if config["transform"] == "log10_one_plus": actual = math.log1p(actual) / math.log(10) predicted = math.log1p(predicted) / math.log(10) error = abs(predicted - actual) if not math.isfinite(error): raise InvalidSubmission("prediction magnitude causes nonfinite error") errors.append(error) if not errors: raise RuntimeError("trusted labels contain no observations for " + endpoint) try: mean = math.fsum(errors) / len(errors) except OverflowError: raise InvalidSubmission("prediction magnitudes overflow the metric") if not math.isfinite(mean): raise InvalidSubmission("prediction magnitudes overflow the metric") per_endpoint[endpoint] = {"mae": mean, "observed_labels": len(errors), "unscored_missing": len(tests) - len(errors)} value = math.fsum(item["mae"] / len(per_endpoint) for item in per_endpoint.values()) return value, per_endpoint def main(): out_dir = Path(os.environ.get("TFP_OUTPUT_DIR", "/output")) out_dir.mkdir(parents=True, exist_ok=True) try: manifest = json.loads(Path(os.environ.get("TFP_MANIFEST", "/input/manifest.json")).read_text()) def read(kind, key, limit): matches = [entry for entry in manifest[kind] if entry.get("key" if kind == "reference" else "role") == key] if len(matches) != 1: raise RuntimeError("manifest must bind exactly one " + key) path = Path(matches[0]["path"]) if path.stat().st_size > limit: if kind == "artifacts": raise InvalidSubmission(key + " exceeds the size limit") raise RuntimeError("trusted reference exceeds limit") return path.read_bytes() config = json_object(read("reference", "benchmark_config", 262144), "benchmark_config") if config["transform"] not in ("identity", "log10_one_plus"): raise RuntimeError("unknown trusted score transform") fields = ["molecule_id", "smiles"] + config["targets"] train = read("reference", "train", 8 * 1024**2) tests = table(read("reference", "test", 8 * 1024**2), ["molecule_id", "smiles"], "trusted test") labels = table(read("reference", "labels", 8 * 1024**2), fields, "trusted labels") predictions = table(read("artifacts", "predictions", 8 * 1024**2), fields, "predictions") validate_method(read("artifacts", "method", 262144), config, hashlib.sha256(train).hexdigest()) archive = validate_archive(read("artifacts", "reproduction", 32 * 1024**2)) value, endpoints = score_predictions(config, tests, labels, predictions) evidence = {"task_id": config["task_id"], "public_retrospective": True, "score": value, "metric": config["metric_name"], "prediction_rows": len(predictions), "endpoints": endpoints, "reproduction_archive": archive, "scope": "Scoring and structural checks only; independent reproduction and scientific acceptance require review."} (out_dir / "evidence").mkdir(exist_ok=True) (out_dir / "evidence" / "metrics.json").write_text(json.dumps(evidence, indent=2, allow_nan=False) + "\n") result = {"result": "verified", "score": value, "reason": "Prediction coverage, units declaration and reproducibility-package structure passed; public retrospective score computed. Scientific review remains required."} except InvalidSubmission as exc: result = {"result": "refuted", "reason": str(exc)[:3900]} except Exception as exc: result = {"result": "inconclusive", "reason": "Could not evaluate trusted inputs or execution: " + str(exc)[:3700]} (out_dir / "result.json").write_text(json.dumps(result, allow_nan=False) + "\n") print(result["reason"]) if __name__ == "__main__": main()