Add preregistration consistency assistant

preregistration-consistency-assistant/README.md

@@ -0,0 +1,25 @@
+# Preregistration consistency assistant
+
+This module checks whether AI review output should be released when a study has a preregistration plan.
+
+It focuses on one AI research assistant slice: spotting preregistration conflicts before the assistant gives authors reviewer-facing guidance. It does not call external registries, run AI models, or process private manuscripts.
+
+## What it checks
+
+- preregistration id
+- primary outcome reporting
+- unlogged primary outcome changes
+- unexplained analysis plan changes
+- unplanned subgroup claims
+- sample size shortfalls
+- secondary outcome promotion
+- missing null result discussion
+
+## Run it
+
+```bash
+node preregistration-consistency-assistant/test.js
+node preregistration-consistency-assistant/demo.js
+```
+
+The demo writes JSON and Markdown artifacts to `artifacts/`.

preregistration-consistency-assistant/artifacts/preregistration-report.md

@@ -0,0 +1,27 @@
+# Preregistration consistency assistant report
+
+## STUDY-01: Sleep intervention trial
+Decision: RELEASE_ASSISTANT_REVIEW
+Next step: release the assistant review with preregistration receipt
+
+## STUDY-02: Microbiome cohort analysis
+Decision: FLAG_FOR_AUTHOR_CHECK
+Next step: ask authors to address warnings before final release
+Warnings:
+- sample size is below the preregistered plan
+- secondary outcomes are promoted above the primary outcome
+- null results are not mentioned
+
+## STUDY-03: Cognitive training trial
+Decision: HOLD_ASSISTANT_REVIEW
+Next step: hold AI review output until preregistration conflicts are resolved
+Blockers:
+- missing preregistration id
+- primary outcome is not reported
+- primary outcome changed without amendment
+- analysis plan changed without explanation
+- unplanned subgroup claim is not marked exploratory
+Warnings:
+- sample size is below the preregistered plan
+- secondary outcomes are promoted above the primary outcome
+- null results are not mentioned

preregistration-consistency-assistant/artifacts/preregistration-results.json

@@ -0,0 +1,40 @@
+[
+  {
+    "studyId": "STUDY-01",
+    "title": "Sleep intervention trial",
+    "decision": "RELEASE_ASSISTANT_REVIEW",
+    "blockers": [],
+    "warnings": [],
+    "nextStep": "release the assistant review with preregistration receipt"
+  },
+  {
+    "studyId": "STUDY-02",
+    "title": "Microbiome cohort analysis",
+    "decision": "FLAG_FOR_AUTHOR_CHECK",
+    "blockers": [],
+    "warnings": [
+      "sample size is below the preregistered plan",
+      "secondary outcomes are promoted above the primary outcome",
+      "null results are not mentioned"
+    ],
+    "nextStep": "ask authors to address warnings before final release"
+  },
+  {
+    "studyId": "STUDY-03",
+    "title": "Cognitive training trial",
+    "decision": "HOLD_ASSISTANT_REVIEW",
+    "blockers": [
+      "missing preregistration id",
+      "primary outcome is not reported",
+      "primary outcome changed without amendment",
+      "analysis plan changed without explanation",
+      "unplanned subgroup claim is not marked exploratory"
+    ],
+    "warnings": [
+      "sample size is below the preregistered plan",
+      "secondary outcomes are promoted above the primary outcome",
+      "null results are not mentioned"
+    ],
+    "nextStep": "hold AI review output until preregistration conflicts are resolved"
+  }
+]

preregistration-consistency-assistant/demo.js

@@ -0,0 +1,13 @@
+const fs = require("fs");
+const path = require("path");
+const studies = require("./sample-data.json");
+const { evaluateStudies, renderMarkdownReport } = require("./index");
+
+const artifactsDir = path.join(__dirname, "artifacts");
+fs.mkdirSync(artifactsDir, { recursive: true });
+
+const results = evaluateStudies(studies);
+fs.writeFileSync(path.join(artifactsDir, "preregistration-results.json"), JSON.stringify(results, null, 2));
+fs.writeFileSync(path.join(artifactsDir, "preregistration-report.md"), renderMarkdownReport(results));
+
+console.log(renderMarkdownReport(results));

preregistration-consistency-assistant/index.js

@@ -0,0 +1,64 @@
+function evaluateStudy(study) {
+  const blockers = [];
+  const warnings = [];
+
+  if (!study.preregistrationId) blockers.push("missing preregistration id");
+  if (!study.primaryOutcomeReported) blockers.push("primary outcome is not reported");
+  if (study.primaryOutcomeChanged && !study.amendmentLogged) blockers.push("primary outcome changed without amendment");
+  if (study.analysisPlanChanged && !study.deviationExplained) blockers.push("analysis plan changed without explanation");
+  if (study.unplannedSubgroupClaim && !study.markedExploratory) blockers.push("unplanned subgroup claim is not marked exploratory");
+  if (study.sampleSize < study.plannedSampleSize) warnings.push("sample size is below the preregistered plan");
+  if (study.secondaryOutcomesPromoted) warnings.push("secondary outcomes are promoted above the primary outcome");
+  if (!study.nullResultsMentioned) warnings.push("null results are not mentioned");
+
+  let decision = "RELEASE_ASSISTANT_REVIEW";
+  if (blockers.length) {
+    decision = "HOLD_ASSISTANT_REVIEW";
+  } else if (warnings.length) {
+    decision = "FLAG_FOR_AUTHOR_CHECK";
+  }
+
+  return {
+    studyId: study.id,
+    title: study.title,
+    decision,
+    blockers,
+    warnings,
+    nextStep: nextStepFor(decision),
+  };
+}
+
+function nextStepFor(decision) {
+  if (decision === "RELEASE_ASSISTANT_REVIEW") return "release the assistant review with preregistration receipt";
+  if (decision === "FLAG_FOR_AUTHOR_CHECK") return "ask authors to address warnings before final release";
+  return "hold AI review output until preregistration conflicts are resolved";
+}
+
+function evaluateStudies(studies) {
+  return studies.map(evaluateStudy);
+}
+
+function renderMarkdownReport(results) {
+  const lines = ["# Preregistration consistency assistant report", ""];
+  for (const result of results) {
+    lines.push(`## ${result.studyId}: ${result.title}`);
+    lines.push(`Decision: ${result.decision}`);
+    lines.push(`Next step: ${result.nextStep}`);
+    if (result.blockers.length) {
+      lines.push("Blockers:");
+      for (const blocker of result.blockers) lines.push(`- ${blocker}`);
+    }
+    if (result.warnings.length) {
+      lines.push("Warnings:");
+      for (const warning of result.warnings) lines.push(`- ${warning}`);
+    }
+    lines.push("");
+  }
+  return lines.join("\n");
+}
+
+module.exports = {
+  evaluateStudies,
+  evaluateStudy,
+  renderMarkdownReport,
+};

preregistration-consistency-assistant/sample-data.json

@@ -0,0 +1,50 @@
+[
+  {
+    "id": "STUDY-01",
+    "title": "Sleep intervention trial",
+    "preregistrationId": "OSF-123",
+    "primaryOutcomeReported": true,
+    "primaryOutcomeChanged": false,
+    "amendmentLogged": false,
+    "analysisPlanChanged": false,
+    "deviationExplained": false,
+    "unplannedSubgroupClaim": false,
+    "markedExploratory": false,
+    "sampleSize": 240,
+    "plannedSampleSize": 220,
+    "secondaryOutcomesPromoted": false,
+    "nullResultsMentioned": true
+  },
+  {
+    "id": "STUDY-02",
+    "title": "Microbiome cohort analysis",
+    "preregistrationId": "OSF-778",
+    "primaryOutcomeReported": true,
+    "primaryOutcomeChanged": false,
+    "amendmentLogged": false,
+    "analysisPlanChanged": false,
+    "deviationExplained": false,
+    "unplannedSubgroupClaim": false,
+    "markedExploratory": false,
+    "sampleSize": 141,
+    "plannedSampleSize": 180,
+    "secondaryOutcomesPromoted": true,
+    "nullResultsMentioned": false
+  },
+  {
+    "id": "STUDY-03",
+    "title": "Cognitive training trial",
+    "preregistrationId": "",
+    "primaryOutcomeReported": false,
+    "primaryOutcomeChanged": true,
+    "amendmentLogged": false,
+    "analysisPlanChanged": true,
+    "deviationExplained": false,
+    "unplannedSubgroupClaim": true,
+    "markedExploratory": false,
+    "sampleSize": 86,
+    "plannedSampleSize": 120,
+    "secondaryOutcomesPromoted": true,
+    "nullResultsMentioned": false
+  }
+]

preregistration-consistency-assistant/test.js

@@ -0,0 +1,15 @@
+const assert = require("assert");
+const studies = require("./sample-data.json");
+const { evaluateStudies } = require("./index");
+
+const results = evaluateStudies(studies);
+const byId = Object.fromEntries(results.map((result) => [result.studyId, result]));
+
+assert.strictEqual(byId["STUDY-01"].decision, "RELEASE_ASSISTANT_REVIEW");
+assert.strictEqual(byId["STUDY-02"].decision, "FLAG_FOR_AUTHOR_CHECK");
+assert.ok(byId["STUDY-02"].warnings.some((warning) => warning.includes("sample size")));
+assert.strictEqual(byId["STUDY-03"].decision, "HOLD_ASSISTANT_REVIEW");
+assert.ok(byId["STUDY-03"].blockers.some((blocker) => blocker.includes("preregistration")));
+assert.ok(byId["STUDY-03"].blockers.some((blocker) => blocker.includes("subgroup")));
+
+console.log("preregistration consistency assistant tests passed");