{ "surface": "1edtech", "deliverable": "surface_qc", "generatedAt": "2026-05-20T22:43:09.228Z", "surfaceVerdict": { "status": "pass", "statement": "Advance to Alpha. The 1EdTech QTI surface is mutually consistent enough to become Alpha's upstream truth.", "rollbackRecommendation": "No product rollback. P-001 is process cleanup only: append closure or supersession records for stale issue-ledger rows." }, "customerEvalTarget": { "persona": "Discerning 1EdTech compliance lead and platform staff engineer who has read the approved architecture, data dictionary, customer website, and implementation.", "jobToBeDone": "Decide in one sitting whether the four approved 1EdTech QTI deliverables are mutually consistent, honestly tested, and safe to advance to Alpha.", "benchmark": "CNCF Certified Kubernetes Software Conformance program, which makes conformance confirmable by a reproducible test tool and reviewer-approved results.", "benchmarkUrl": "https://www.cncf.io/certification/software-conformance/" }, "upstream": [ { "id": "architecture", "title": "1EdTech Architecture", "status": "approved", "attempt": 2, "canonicalUrl": "https://platform3-avjryotuw-andymontgomery-9773s-projects.vercel.app", "localPath": "/Users/andymontgomery/projects/platform3/loop/qti/artifacts/1edtech/architecture", "approvedAt": "2026-05-20T20:30:45Z" }, { "id": "data_dictionary", "title": "1EdTech Data Dictionary", "status": "approved", "attempt": 1, "canonicalUrl": "https://platform3-198nmblg3-andymontgomery-9773s-projects.vercel.app", "localPath": "/Users/andymontgomery/projects/platform3/loop/qti/artifacts/1edtech/data_dictionary", "approvedAt": "2026-05-20T20:58:00Z" }, { "id": "customer_website", "title": "1EdTech Customer Website", "status": "approved", "attempt": 2, "canonicalUrl": "https://platform3-fscem3woq-andymontgomery-9773s-projects.vercel.app", "localPath": "/Users/andymontgomery/projects/platform3/loop/qti/artifacts/1edtech/customer_website", "approvedAt": "2026-05-20T21:33:13Z" }, { "id": "implementation", "title": "1EdTech Implementation", "status": "approved", "attempt": 1, "canonicalUrl": null, "localPath": "/Users/andymontgomery/projects/platform3/loop/qti/artifacts/1edtech/implementation/impl", "approvedAt": "2026-05-20T22:36:35Z" } ], "metrics": [ [ "Surface verdict", "Pass", "Advance the 1EdTech QTI surface to Alpha as upstream truth." ], [ "Architecture ITDs", "24", "Stable anchors include ITD-017 conformance evidence and ITD-024 learner-data privacy." ], [ "Data dictionary", "13 tables / 127 fields", "All persistence objects are documented with provenance, invalid-value guidance, and ITD links." ], [ "Customer endpoints", "8", "The same operation IDs appear in the customer site, OpenAPI, provenance, HTTP handler, and tests." ], [ "Implementation tests", "13/13", "Node contract suite passed during this QC run." ], [ "Conformance smoke", "1,276 assertions", "Full local bundle run passed with 1,011 passed, 265 skipped, 0 failed, and 0 error assertions." ], [ "Bundle hash", "sha256:fd194bdb...", "The implementation records a real source-bundle hash, not the old placeholder." ], [ "Anchor misses", "0", "Architecture, data dictionary, and customer website local hash links resolve." ] ], "deliverables": [ { "id": "architecture", "title": "Architecture", "verdict": "Pass", "status": "pass", "evidence": [ "Approved attempt 2 after the reviewer required learner-data privacy, hosting-state consistency, and removal of self-grading language.", "Architecture traceability contains 24 ITDs, 13 qti.* tables, 8 API operations, 6 learner-sensitive field rules, and 5 privacy rules.", "ITD-017 defines conformance evidence; ITD-024 defines pseudonymous learner data and redaction. Both are carried into the downstream artifacts." ], "rollback": "none" }, { "id": "data-dictionary", "title": "Data Dictionary", "verdict": "Pass", "status": "pass", "evidence": [ "Approved attempt 1 with 13 table sections, 127 field rows, and 10 allowed-value sets.", "qti.conformance_run and qti.conformance_assertion document profile, bundle hash, runner version, assertion key, artifact/spec references, status, and diagnostics.", "Learner-sensitive fields such as candidate_ref, response_state, template_state, outcome_state, and processing_trace are tied back to ITD-024." ], "rollback": "none" }, { "id": "customer-website", "title": "Customer Website", "verdict": "Pass", "status": "pass", "evidence": [ "Approved attempt 2 after adding top-level Authentication and Errors sections plus inline request and response schema tables for every endpoint.", "The runConformance endpoint promises: Run the bundled QTI example corpus through validation, round-trip, and processing assertions.", "The conformance workflow states that evidence is persisted in qti.conformance_run and qti.conformance_assertion rows with reproducible bundle and runner metadata." ], "rollback": "none" }, { "id": "implementation", "title": "Implementation", "verdict": "Pass", "status": "pass", "evidence": [ "Rollback attempt 1 was approved at 2026-05-20T22:36:35Z after replacing the fabricated conformance success path with src/conformance/qtiConformanceRunner.mjs.", "npm test passed 13/13 tests, including runConformance fail-fast behavior, real bundle hash checks, qti.conformance_assertion persistence, OpenAPI ref resolution, privacy redaction, and all eight HTTP operations.", "Full local source-bundle smoke passed: 1,276 total assertions; 316 validation checks; 316 round-trip checks; 62 processing assertions passed; 0 failed; 0 error." ], "rollback": "none" } ], "checks": [ { "id": "state-chain", "title": "Approved Artifact Chain", "status": "pass", "result": "state.json, transcript.md, and approved summaries agree that all four upstream deliverables are approved before this surface QC attempt.", "evidence": [ "state.json current deliverable is 1edtech/surface_qc attempt 1; upstream architecture, data_dictionary, customer_website, and implementation are approved.", "implementation/summary.json attempt is 1 after rollback, matching transcript approval at 2026-05-20T22:36:35Z.", "The report uses the deployment-specific canonical URLs from state.json and approved summaries, not the shared platform3.vercel.app alias." ] }, { "id": "table-field-chain", "title": "Table And Field Coverage", "status": "pass", "result": "The persistence model is consistent across architecture traceability, the data dictionary, implementation provenance, and migration DDL.", "evidence": [ "Architecture traceability lists 13 qti.* tables; the data dictionary lists 13 tables and 127 fields; implementation provenance lists the same 13 tables.", "migrations/001_qti_core.sql creates qti.conformance_run and qti.conformance_assertion and comments their meaning, status enum, indexes, and diagnostics.", "tests/schemaContract.test.mjs verifies the migration carries every approved data-dictionary table and critical field." ] }, { "id": "api-chain", "title": "API Contract Coverage", "status": "pass", "result": "The 8 customer-facing operation IDs align across the approved customer website, OpenAPI, HTTP server, provenance, and tests.", "evidence": [ "Operation IDs: ingestContentPackage, getDeliveryJson, saveAuthoringJson, exportXml, startDeliverySession, submitAttempt, deleteCandidateRuntimeData, runConformance.", "tests/httpApi.test.mjs exercises auth, tenant matching, ingest idempotency, delivery ETag, authoring If-Match, session start, attempt submit, privacy deletion, and runConformance.", "openapi/qti-boundary.openapi.yaml file references resolve inside the implementation artifact, including generated/spec/qti-object-graph.schema.json." ] }, { "id": "conformance-chain", "title": "Conformance Execution And Evidence", "status": "pass", "result": "The previous rollback blocker is closed: runConformance now executes a source-bundle runner and persists granular assertion evidence.", "evidence": [ "Customer website quote: Run the bundled QTI example corpus through validation, round-trip, and processing assertions.", "src/conformance/qtiConformanceRunner.mjs requires QTI_SPEC_BUNDLE_DIR, hashes the source bundle, walks examples/qtiv3-examples, evaluates profile/validation/round-trip/processing assertions, and returns failed or error when evidence is not clean.", "src/repository/postgresQtiRepository.mjs inserts qti.conformance_run and one qti.conformance_assertion row per runner assertion; src/repository/inMemoryQtiRepository.mjs keeps equivalent in-memory evidence.", "tests/conformanceRunner.test.mjs proves missing bundle returns HTTP 500, real bundle hashes match sha256:<64 hex>, runner errors are non-passed, and Postgres assertion inserts occur.", "Full local smoke with external/qti/spec_bundle returned status passed, bundle hash sha256:fd194bdbcf495cded55f73fd4bca0244f8f9c8f78b4ea7cd8627c89cb5f4f131, and 1,276 assertion rows." ] }, { "id": "privacy-chain", "title": "Learner Data Privacy", "status": "pass", "result": "ITD-024 is implemented in the dictionary, SQL constraints, HTTP behavior, runtime storage, and tests.", "evidence": [ "Data dictionary marks candidate_ref, response_state, template_state, outcome_state, and processing_trace as learner-sensitive fields.", "migrations/001_qti_core.sql enforces candidate_ref UUID shape and JSON object shapes.", "tests/privacyAndTrace.test.mjs rejects direct PII and secrets, redacts processing traces and Problem.detail, and protects learner runtime deletion with authorization checks." ] }, { "id": "findability-chain", "title": "Hosted Documentation Findability", "status": "pass", "result": "The three upstream documentation sites have stable local deep links and driver-recorded hosted verification.", "evidence": [ "Local hash-link check: architecture ids=37 links=157 missing=0; data dictionary ids=217 links=178 missing=0; customer website ids=292 links=384 missing=0.", "Driver summaries record unauthenticated 200 verification for architecture at 2026-05-20T20:27:07Z, data dictionary at 2026-05-20T20:54:02Z, and customer website at 2026-05-20T21:29:28Z.", "Codex sandbox DNS could not resolve Vercel hosts during this run, so this report does not claim a fresh network fetch; the driver will deploy and verify this QC site." ] }, { "id": "issue-ledger-chain", "title": "Issue Ledger Consistency", "status": "warn", "result": "issues.jsonl is append-only and still contains earlier open rows, but every product blocker listed there has a later approved fix in transcript.md and attempt results.", "evidence": [ "Earlier architecture, customer_website, and implementation findings remain in issues.jsonl with open status.", "Transcript and eval result files show the corresponding attempts were later approved after fixes.", "The old surface_qc F-001 is superseded by the approved implementation rollback attempt and by the current passing conformance runner evidence." ] } ], "findings": [ { "id": "P-001", "severity": "process", "title": "Issue ledger closure records are stale", "affects": "loop/qti/issues.jsonl", "quote": "Earlier rows in issues.jsonl retain open status even though transcript.md records later reviewer approvals.", "evidence": [ "This does not contradict the product surface because state.json, transcript.md, summary.json files, and eval results identify the approved current attempts.", "It does make supervision noisier because a future reader must inspect transcript.md to know those issue rows were superseded." ], "recommendation": "Append closure or supersession rows after approval, or update the driver to write a closure record when a later attempt addresses a recorded issue.", "rollback": "none" } ], "verification": [ { "command": "npm test", "cwd": "loop/qti/artifacts/1edtech/implementation/impl", "result": "pass", "outcome": "13 tests passed, 0 failed. Covered conformance runner, HTTP API, privacy/trace redaction, schema contracts, OpenAPI references, and bundled schema references." }, { "command": "node --input-type=module -e 'import { runQtiConformance } ...'", "cwd": "loop/qti/artifacts/1edtech/implementation/impl", "result": "pass", "outcome": "QTI_SPEC_BUNDLE_DIR smoke over external/qti/spec_bundle returned status passed, 1,276 assertions, 1,011 passed, 265 skipped, 0 failed, 0 error." }, { "command": "JSON parse checks", "cwd": "repo root", "result": "pass", "outcome": "architecture-traceability.json, qti-data-dictionary.json, customer-website-source.json, search-index.json, implementation provenance.json, implementation summary.json, and generated surface-qc-evidence.json parse as JSON." }, { "command": "Static hash-link check", "cwd": "repo root", "result": "pass", "outcome": "Architecture, data dictionary, and customer website local hash links all resolve with 0 missing anchors." }, { "command": "curl -I -L approved Vercel URLs", "cwd": "repo root", "result": "not-run", "outcome": "Codex sandbox DNS could not resolve Vercel hosts. The report relies on driver-recorded prior 200 checks and leaves this deliverable's canonical_url null for driver deployment and unauthenticated fetch." } ], "sourceFiles": [ "loop/qti/state.json", "loop/qti/transcript.md", "loop/qti/issues.jsonl", "loop/qti/evals/1edtech/surface_qc/customer_eval.json", "loop/qti/evals/1edtech/surface_qc/rubric.json", "loop/qti/artifacts/1edtech/architecture/summary.json", "loop/qti/artifacts/1edtech/architecture/site/index.html", "loop/qti/artifacts/1edtech/architecture/site/architecture-traceability.json", "loop/qti/artifacts/1edtech/data_dictionary/summary.json", "loop/qti/artifacts/1edtech/data_dictionary/site/index.html", "loop/qti/artifacts/1edtech/data_dictionary/site/qti-data-dictionary.json", "loop/qti/artifacts/1edtech/customer_website/summary.json", "loop/qti/artifacts/1edtech/customer_website/site/index.html", "loop/qti/artifacts/1edtech/customer_website/site/customer-website-source.json", "loop/qti/artifacts/1edtech/customer_website/site/search-index.json", "loop/qti/artifacts/1edtech/implementation/summary.json", "loop/qti/artifacts/1edtech/implementation/impl/README.md", "loop/qti/artifacts/1edtech/implementation/impl/provenance.json", "loop/qti/artifacts/1edtech/implementation/impl/src/conformance/qtiConformanceRunner.mjs", "loop/qti/artifacts/1edtech/implementation/impl/src/http/qtiHttpServer.mjs", "loop/qti/artifacts/1edtech/implementation/impl/src/repository/inMemoryQtiRepository.mjs", "loop/qti/artifacts/1edtech/implementation/impl/src/repository/postgresQtiRepository.mjs", "loop/qti/artifacts/1edtech/implementation/impl/tests/conformanceRunner.test.mjs", "loop/qti/artifacts/1edtech/implementation/impl/tests/httpApi.test.mjs", "loop/qti/artifacts/1edtech/implementation/impl/tests/privacyAndTrace.test.mjs", "loop/qti/artifacts/1edtech/implementation/impl/tests/schemaContract.test.mjs", "external/qti/docs/conformance-and-documentation.md", "external/qti/generated/conformance-full/conformance-report.md", "external/qti/generated/conformance/processing-coverage.md", "external/qti/spec_bundle/README.md", "external/qti/spec_bundle/MANIFEST.md" ] }