#!/usr/bin/env python3
"""Second SO4AG 0.1 reader (Python, standard library only, Apache-2.0).
Written separately from the reference implementation in validator/ and shares no code with it.
It reads a source from SO4AG JSON, from a JSON-LD document or from an HTML page with embedded JSON-LD,
and answers one question: may this claim be used for this task at this time?
python so4ag_reader.py conformance
run the conformance suite
python so4ag_reader.py ask --property [--scope k=v ...] [--now ISO] [--review]
"""
import hashlib
import json
import re
import sys
from datetime import datetime, timezone
from pathlib import Path
SOURCE_TYPES = ("so4ag:Source", "https://soqver.com/spec/so4ag/0.1/terms#Source")
# ---------- canonical JSON (RFC 8785 for the values SO4AG uses) and claim digest ----------
def canonical(v):
if isinstance(v, dict):
keys = sorted(v, key=lambda k: k.encode("utf-16-be"))
return "{" + ",".join(json.dumps(k, ensure_ascii=False) + ":" + canonical(v[k]) for k in keys) + "}"
if isinstance(v, list):
return "[" + ",".join(canonical(x) for x in v) + "]"
if isinstance(v, float) and v == int(v) and abs(v) < 1e15:
return str(int(v))
return json.dumps(v, ensure_ascii=False)
def claim_digest(claim):
body = {k: v for k, v in claim.items() if k != "attestations"}
return "sha256:" + hashlib.sha256(canonical(body).encode("utf-8")).hexdigest()
# ---------- time ----------
def instant(text):
t = datetime.fromisoformat(text.replace("Z", "+00:00"))
return t if t.tzinfo else t.replace(tzinfo=timezone.utc)
# ---------- the consumer rules ----------
def scope_keys_of(property_uri, profiles):
for p in profiles:
ns = p["namespace"]
if property_uri.startswith(ns) and property_uri[len(ns):] in p["properties"]:
return set(p.get("scope_keys", {}))
return set()
def evaluate(claim, task, now, profiles):
"""Returns (usable, reasons, value). `disputed` is reported but does not block."""
reasons = []
if task.get("property") and task["property"] != claim["property"]:
reasons.append("property_mismatch")
if task.get("subject") and task["subject"] != claim["subject"]:
reasons.append("subject_mismatch")
understood = set(task["understoodScopeKeys"]) if "understoodScopeKeys" in task else scope_keys_of(claim["property"], profiles)
asked = task.get("scope") or {}
for key, limit in claim["scope"].items():
if key not in understood:
reasons.append("scope_not_comparable:" + key)
elif key not in asked:
reasons.append("scope_not_in_task:" + key)
elif limit != "any" and scope_text(asked[key]) != limit:
reasons.append("scope_mismatch:" + key)
if now < instant(claim["valid_from"]):
reasons.append("not_yet_valid")
if now >= instant(claim["valid_until"]):
reasons.append("expired")
if claim.get("dispute"):
reasons.append("disputed")
if claim["knowledge_state"] != "known":
reasons.append(claim["knowledge_state"])
if task.get("requireReview"):
digest = claim_digest(claim)
trusted = task.get("trustedIssuers")
accepted = any(
a.get("claim_digest") == digest
and a.get("issuer_relation") == "independent"
and instant(a["expires_at"]) > now
and (trusted is None or a.get("issuer") in trusted)
for a in claim.get("attestations", [])
)
if not accepted:
reasons.append("no_valid_independent_attestation")
usable = all(r == "disputed" for r in reasons)
return usable, sorted(reasons), (claim.get("value") if usable else None)
def scope_text(v):
if isinstance(v, bool):
return "true" if v else "false"
if isinstance(v, float) and v == int(v):
return str(int(v))
return str(v)
# ---------- reading JSON-LD ----------
def identifiers(node_list):
return [{"scheme": i["propertyID"], "value": i["value"]} for i in node_list]
def value_from_node(n):
if isinstance(n, bool):
return {"type": "boolean", "value": n}
if "@value" in n:
out = {"type": "text", "value": n["@value"]}
if "@language" in n:
out["lang"] = n["@language"]
return out
kind = n.get("@type")
if kind == "MonetaryAmount":
out = {"type": "money", "amount": n["minValue"] if "minValue" in n else n["value"], "currency": n["currency"]}
if "so4ag:vatIncluded" in n:
out["vat_included"] = n["so4ag:vatIncluded"]
if "minValue" in n:
out["bound"] = "minimum"
elif "so4ag:bound" in n:
out["bound"] = n["so4ag:bound"]
return out
if kind == "QuantitativeValue":
return {"type": "quantity", "value": n["value"], "unit": n["unitText"]}
if kind == "DefinedTerm":
return {"type": "enum", "value": n["termCode"]}
if kind == "ItemList":
return {"type": "enum_set", "values": [x["termCode"] for x in n["itemListElement"]]}
if kind == "Organization":
out = {"type": "organization", "name": n["name"]}
if "identifier" in n:
out["identifiers"] = identifiers(n["identifier"])
if "url" in n:
out["url"] = n["url"]
return out
if kind == "Place":
out = {"type": "place", "name": n["name"]}
address = n.get("address") or {}
if "streetAddress" in address:
out["address"] = address["streetAddress"]
if "addressCountry" in address:
out["country"] = address["addressCountry"]
if "geo" in n:
out["geo"] = {"lat": n["geo"]["latitude"], "lon": n["geo"]["longitude"]}
if "openingHours" in n:
out["opening_hours"] = n["openingHours"]
return out
raise ValueError("unknown value node")
def copy_fields(src, pairs):
return {to: src[frm] for frm, to in pairs if frm in src}
def claim_from_node(pv, subject):
c = {"id": pv["@id"], "subject": subject, "property": pv["propertyID"],
"knowledge_state": pv["so4ag:knowledgeState"], "assertion_basis": pv["so4ag:assertionBasis"],
"valid_from": pv["validFrom"], "valid_until": pv["validThrough"],
"scope": {s["propertyID"]: s["value"] for s in pv.get("so4ag:scope", [])}}
if "so4ag:revision" in pv:
c["revision"] = pv["so4ag:revision"]
if "value" in pv:
c["value"] = value_from_node(pv["value"])
if "so4ag:evidence" in pv:
c["evidence"] = [copy_fields(e, [("url", "url"), ("so4ag:observedAt", "observed_at"), ("so4ag:kind", "kind"),
("so4ag:issuer", "issuer"), ("so4ag:quote", "quote"), ("so4ag:digest", "digest")])
for e in pv["so4ag:evidence"]]
if "so4ag:note" in pv:
c["note"] = pv["so4ag:note"]
if "so4ag:dispute" in pv:
d = pv["so4ag:dispute"]
c["dispute"] = copy_fields(d, [("so4ag:since", "since"), ("so4ag:note", "note")])
if "so4ag:conflictingClaim" in d:
c["dispute"]["conflicting_claims"] = [x["@id"] for x in d["so4ag:conflictingClaim"]]
if "so4ag:attestation" in pv:
c["attestations"] = []
for a in pv["so4ag:attestation"]:
item = copy_fields(a, [("@id", "id"), ("so4ag:issuerRelation", "issuer_relation"), ("so4ag:method", "method"),
("so4ag:claimDigest", "claim_digest"), ("so4ag:issuedAt", "issued_at"),
("so4ag:expiresAt", "expires_at"), ("so4ag:statusUrl", "status_url"),
("so4ag:credentialUrl", "credential_url")])
if "so4ag:issuer" in a:
item["issuer"] = a["so4ag:issuer"]["@id"]
c["attestations"].append(item)
if "so4ag:extensions" in pv:
c["extensions"] = pv["so4ag:extensions"]["@value"]
return c
def claims_from_jsonld(doc):
graph = doc.get("@graph")
if not isinstance(graph, list) or not any(n.get("additionalType") in SOURCE_TYPES for n in graph):
raise ValueError("not a SO4AG source")
claims = []
for node in graph:
if node.get("additionalType") in SOURCE_TYPES:
continue
for pv in node.get("additionalProperty", []):
if "so4ag:knowledgeState" in pv:
claims.append(claim_from_node(pv, node["@id"]))
return claims
DEFAULT_KEYS = ("subject", "assertion_basis", "valid_from", "valid_until", "evidence", "scope")
def expand02(doc):
"""SO4AG 0.2 -> claims written out in full: file defaults applied, null = the claim has no such field."""
defaults = doc.get("defaults") or {}
out = []
for claim in doc.get("claims", []):
c = dict(claim)
for k in DEFAULT_KEYS:
if k in c and c[k] is None:
del c[k]
elif k not in c and k in defaults:
c[k] = json.loads(json.dumps(defaults[k]))
c.setdefault("scope", {})
out.append(c)
return out
def claims_from_text(text):
"""SO4AG JSON (0.1 or 0.2), JSON-LD, or an HTML page with JSON-LD blocks."""
stripped = text.lstrip()
if stripped.startswith("{"):
doc = json.loads(text)
if doc.get("so4ag") == "0.2-draft":
return expand02(doc)
return doc["claims"] if "claims" in doc else claims_from_jsonld(doc)
for block in re.findall(r'', text, re.S | re.I):
try:
doc = json.loads(block)
except ValueError:
continue
for d in doc if isinstance(doc, list) else [doc]:
try:
return claims_from_jsonld(d)
except (ValueError, AttributeError):
pass
raise ValueError("no SO4AG source found")
# ---------- command line ----------
def load_profiles(root):
return [json.loads(p.read_text(encoding="utf-8")) for p in sorted(Path(root).glob("*.json"))]
def run_conformance(folder):
folder = Path(folder)
profiles = load_profiles(folder.parent.parent / "profiles")
failed = total = 0
def check(ok, label):
nonlocal failed, total
total += 1
if not ok:
failed += 1
print("FAIL", label)
digests = json.loads((folder / "digests.json").read_text(encoding="utf-8"))
sources = {}
for name in ("shop", "rental"):
native = json.loads((folder / f"{name}.json").read_text(encoding="utf-8"))["claims"]
from_ld = claims_from_text((folder / f"{name}.jsonld").read_text(encoding="utf-8"))
html = '"
from_html = claims_from_text(html)
sources[name + ".json"] = {"so4ag": native, "jsonld": from_ld, "html": from_html}
for form, claims in sources[name + ".json"].items():
check(len(claims) == len(native), f"{name} {form}: number of claims")
for c in claims:
check(claim_digest(c) == digests.get(c["id"]), f"{name} {form}: digest of {c['id']}")
cases = json.loads((folder / "evaluation.json").read_text(encoding="utf-8"))["cases"]
for case in cases:
for form, claims in sources[case["manifest"]].items():
claim = next(c for c in claims if c["id"] == case["claim"])
usable, reasons, value = evaluate(claim, case["task"], instant(case["now"]), profiles)
want = case["expect"]
check(usable == want["usable"] and reasons == want["reasons"] and value == want["value"],
f"{case['id']} ({form}) {case['title']}: got {usable} {reasons}")
print(f"{total - failed}/{total} checks passed ({len(cases)} evaluation cases x 3 input forms, digests of every claim)")
return 1 if failed else 0
def run_ask(argv):
path, prop, scope, now, review = argv[0], None, {}, datetime.now(timezone.utc), False
i = 1
while i < len(argv):
if argv[i] == "--property":
prop = argv[i + 1]; i += 2
elif argv[i] == "--scope":
k, _, v = argv[i + 1].partition("="); scope[k] = v; i += 2
elif argv[i] == "--now":
now = instant(argv[i + 1]); i += 2
elif argv[i] == "--review":
review = True; i += 1
else:
raise SystemExit("unknown argument " + argv[i])
here = Path(__file__).resolve().parent
profiles = load_profiles(here.parent.parent / "spec" / "profiles")
claims = claims_from_text(Path(path).read_text(encoding="utf-8"))
out = []
for c in claims:
if prop and c["property"] != prop and not c["property"].endswith("#" + prop):
continue
task = {"property": c["property"], "scope": scope, "requireReview": review}
usable, reasons, value = evaluate(c, task, now, profiles)
out.append({"claim": c["id"], "usable": usable, "reasons": reasons, "value": value, "scope": c["scope"], "valid_until": c["valid_until"]})
out.sort(key=lambda r: not r["usable"])
print(json.dumps(out, ensure_ascii=False, indent=2))
return 0
if __name__ == "__main__":
if len(sys.argv) >= 3 and sys.argv[1] == "conformance":
sys.exit(run_conformance(sys.argv[2]))
if len(sys.argv) >= 3 and sys.argv[1] == "ask":
sys.exit(run_ask(sys.argv[2:]))
print(__doc__)
sys.exit(2)