{"release_id":"20261029T170018Z-80c62152e5a8","digest":"6e382464b75c6d7a4328a4963ec9b82eab47fa0e82435a75476330e9e8ed1cdc","generated_at":"2026-10-29T17:00:18Z","reference_date":"2026-10-29","source_cutoff":"2026-10-28","evaluation_cutoff":"2026-10-28","policy_version":"2026-10-28.7","schema_version":1,"fixture":false,"part":6,"parts":6,"claims":[{"url":"https://punditclaims.ai/claims/marcus-does-ai-really-need-a-paradigm-shift-282b4f","standing":"current","assessment_label":"Supported","versions":[{"assessment_id":"marcus-does-ai-really-need-a-paradigm-shift-282b4f-a2","status":"current","url":"https://punditclaims.ai/claims/marcus-does-ai-really-need-a-paradigm-shift-282b4f#assessment-marcus-does-ai-really-need-a-paradigm-shift-282b4f-a2"}],"claim":{"claim_id":"marcus-does-ai-really-need-a-paradigm-shift-282b4f","subject_id":"gary-marcus","revision":1,"content_hash":"3d54c3b1893bf6fb4682ad44405436e19727438df015fe93a0e62c83fd04b2ec","normalized_proposition":"BIG-bench shows many capabilities improve with scale but not all, which bears out Marcus's 'Deep learning is hitting a wall' essay.","kind":"observation","target":"large language models","conditions":[{"text":"per BIG-bench findings","type":"qualifier","from_source":true}],"original_horizon_text":null,"topic_tags":["scaling","capabilities"],"concerns_providers":[],"selection_stratum":"declared-lead","occurrences":[{"id":"marcus-does-ai-really-need-a-paradigm-shift-282b4f-o1","statement_date":"2022-06-11","statement_date_precision":"day","excerpt":"The basic finding was: loads of things scale, but not at all ( as foreseen both by Kurenkov in his essay last year and in my much lampooned but basically accurate essay, Deep learning is hitting a wall).","locator":"paragraph 36","document":{"url":"https://garymarcus.substack.com/p/does-ai-really-need-a-paradigm-shift","title":"Does AI really need a paradigm shift?","publisher":"Marcus on AI","published_at":"2022-06-11T17:32:54Z","date_uncertain":false,"date_provenance":"JSON-LD datePublished"},"observed_at":"2026-10-29T00:05:17Z","observed_via":"live","archive_url":null,"access_state":"full_public","snapshot_hash":"9716f559ab7b333c80f9fb7209db58ce8a0fde42f3f553914472aa7b9ff74804","receipts":1}],"first_statement_date":"2022-06-11","related":[],"current":{"assessment_id":"marcus-does-ai-really-need-a-paradigm-shift-282b4f-a2","payload":{"claim_id":"marcus-does-ai-really-need-a-paradigm-shift-282b4f","subject_id":"gary-marcus","normalized_proposition":"BIG-bench shows many capabilities improve with scale but not all, which bears out Marcus's 'Deep learning is hitting a wall' essay.","kind":"observation","conditions":["per BIG-bench findings"],"original_horizon_text":null,"statement":{"excerpt":"The basic finding was: loads of things scale, but not at all ( as foreseen both by Kurenkov in his essay last year and in my much lampooned but basically accurate essay, Deep learning is hitting a wall).","locator":"paragraph 36","url":"https://garymarcus.substack.com/p/does-ai-really-need-a-paradigm-shift","date":"2022-06-11","document_title":"Does AI really need a paradigm shift?","observed_at":"2026-10-29T00:05:17Z"},"criteria":{"from_speaker":["The basic finding was","loads of things scale, but not at all","as foreseen","Deep learning is hitting a wall"],"our_interpretation":["The surrounding sentence about progress on some measures but not others makes not all the intended reading of the apparent typo not at all.","The test concerns the original June 2022 BIG-bench paper and its evaluated model sizes.","The reference to the earlier essay is assessed as corroboration of its limited contention that improvement was uneven across measures. It is not treated as proof of every argument in that essay or of a permanent scaling barrier."],"support_condition":"The original BIG-bench report finds broad improvements with model scale alongside tasks or measures that do not improve over the tested range.","contradiction_condition":"The original report finds improvement on all tested measures, or fails to report the claimed coexistence of improving and non-improving performance."},"axis":"historical_observation","outcome":"supported","horizon_state":"not_applicable","evaluation_as_of":"2026-10-28","rationale":"The June 9, 2022 BIG-bench report explicitly states that some tasks improve with scale while others show no improvement over the tested range [e1]. This supports the limited claim of uneven measured progress across tasks [e1].","limitations":["The finding does not establish a permanent scaling barrier or what happens beyond the tested model sizes [e1].","It corroborates uneven measured progress, not every argument in Marcus's earlier essay [e1]."],"evidence":[{"id":"e1","relation":"supports","excerpt":"no improvement with scale","locator":"Section 3.4, Linearity and breakthroughness: categorizing scaling behaviors","url":"https://arxiv.org/html/2206.04615v1","publisher":"Srivastava et al.; arXiv","event_date":"2022-06-09","available_at":"2022-06-09","explanation":"The original report directly states the coexistence Marcus describes. Sections 3.4.2 and 6.1 also explain that metric choice can conceal gradual progress and that some limitations may change with larger models or improved data."}],"what_would_resolve":null,"retrospective":true,"plain_reading":true},"payload_hash":"f051d99d80625bcad62b8e4e9877719d72baa2d4ae52941b8833cd4080f4b566","policy_version":"2026-10-28.7","recorded_at":"2026-10-29T07:10:59Z","drafter":{"provider":"openai","model":"gpt-6-astra"},"reviewers":[{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-does-ai-really-need-a-paradigm-shift-282b4f-4x-review-anthropic-1"},{"provider":"openai","model":"gpt-6-astra","run_id":"marcus-does-ai-really-need-a-paradigm-shift-282b4f-4x-review-openai-1"}],"final_text":{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-does-ai-really-need-a-paradigm-shift-282b4f-4x-final-anthropic-2"},"status":"current","notice":null,"superseded_by":null,"evidence_meta":[{"id":"e1","source_type":"preprint","audited":null,"derived":null,"metric":null}],"public_from":null,"public_until":null},"history":[],"review":{"review_due":false,"triggers":[],"last_completed_review_at":"2026-10-29T07:10:59Z","next_review_due_at":null,"horizon_now":"not_applicable"},"event_ids":["marcus-does-ai-really-need-a-paradigm-shift-282b4f-assessment-published-a2"],"last_substantive_change_at":"2026-10-29T17:00:18Z"}},{"url":"https://punditclaims.ai/claims/marcus-does-ai-really-need-a-paradigm-shift-3bb4b4","standing":"current","assessment_label":"Unresolved","versions":[{"assessment_id":"marcus-does-ai-really-need-a-paradigm-shift-3bb4b4-a2","status":"current","url":"https://punditclaims.ai/claims/marcus-does-ai-really-need-a-paradigm-shift-3bb4b4#assessment-marcus-does-ai-really-need-a-paradigm-shift-3bb4b4-a2"}],"claim":{"claim_id":"marcus-does-ai-really-need-a-paradigm-shift-3bb4b4","subject_id":"gary-marcus","revision":2,"content_hash":"5fcab7de7e80d33cc24a33337be910016d082a952e32feec30559cf732ddb475","normalized_proposition":"Whenever AGI arrives, an explicit type/token distinction will be crucial to it.","kind":"conditional_forecast","target":"explicit type/token distinction in AGI","conditions":[{"text":"whenever AGI comes","type":"antecedent","from_source":true}],"original_horizon_text":null,"topic_tags":["agi-timelines","capabilities"],"concerns_providers":[],"selection_stratum":"declared-lead","occurrences":[{"id":"marcus-does-ai-really-need-a-paradigm-shift-3bb4b4-o1","statement_date":"2022-06-11","statement_date_precision":"day","excerpt":"👉an explicit type/token distinction will be crucial","locator":"prediction 4 of 4 (paragraph 58)","document":{"url":"https://garymarcus.substack.com/p/does-ai-really-need-a-paradigm-shift","title":"Does AI really need a paradigm shift?","publisher":"Marcus on AI","published_at":"2022-06-11T17:32:54Z","date_uncertain":false,"date_provenance":"JSON-LD datePublished"},"observed_at":"2026-10-29T00:05:17Z","observed_via":"live","archive_url":null,"access_state":"full_public","snapshot_hash":"9716f559ab7b333c80f9fb7209db58ce8a0fde42f3f553914472aa7b9ff74804","receipts":1}],"first_statement_date":"2022-06-11","related":[],"current":{"assessment_id":"marcus-does-ai-really-need-a-paradigm-shift-3bb4b4-a2","payload":{"claim_id":"marcus-does-ai-really-need-a-paradigm-shift-3bb4b4","subject_id":"gary-marcus","normalized_proposition":"Whenever AGI arrives, an explicit type/token distinction will be crucial to it.","kind":"conditional_forecast","conditions":["whenever AGI comes"],"original_horizon_text":null,"statement":{"excerpt":"👉an explicit type/token distinction will be crucial","locator":"prediction 4 of 4 (paragraph 58)","url":"https://garymarcus.substack.com/p/does-ai-really-need-a-paradigm-shift","date":"2022-06-11","document_title":"Does AI really need a paradigm shift?","observed_at":"2026-10-29T00:05:17Z"},"criteria":{"from_speaker":["whenever AGI comes","an explicit type/token distinction","will be crucial","tied not to time but to architecture"],"our_interpretation":["Treat arrival as a demonstrated system with broadly accepted human-level general cognitive capability. Broad acceptance is an evaluator safeguard.","Read type/token distinction as an explicit separation between a general kind or category and particular instances of it.","Language-model tokenization alone does not establish this distinction in the system's reasoning.","Read crucial as central dependence supported by component tests or equivalent evidence.","Test the first broadly accepted AGI; dependence in that system would support the arrival prediction without proving universal necessity."],"support_condition":"The AGI condition is established, and technical evidence shows an explicit kind-versus-instance representation whose removal substantially undermines the arriving system's general capability.","contradiction_condition":"The AGI condition is established, and complete architectural evidence or controlled tests show that the system achieves general capability without an explicit type/token distinction."},"axis":"forecast_outcome","outcome":"unresolved","horizon_state":"open_horizon","evaluation_as_of":"2026-10-28","rationale":"OpenAI's September 3, 2026 announcement reports GPT-6 Astra's capabilities [e1]. It does not establish broadly accepted AGI or show whether general capability depends on an explicit distinction between kinds and individual instances [e1]. This evidence therefore leaves the architectural prediction unresolved [e1].","limitations":["The announcement provides neither independent confirmation of AGI nor tests of dependence on an explicit type/token distinction [e1].","Its lack of qualifying architectural evidence does not establish that the distinction is absent [e1]."],"evidence":[{"id":"e1","relation":"context","excerpt":"We’re introducing GPT‑6 Astra","locator":"Introduction, first paragraph after update notices","url":"https://openai.com/index/gpt-6-astra","publisher":"OpenAI","event_date":"2026-09-03","available_at":"2026-09-03","explanation":"This is a developer's capability claim for a candidate system. It does not independently establish AGI, disclose an explicit kind-versus-instance mechanism, or show whether such a mechanism is crucial."}],"what_would_resolve":"A dated, broadly accepted AGI demonstration, with public architectural evidence and controlled tests or equivalent evidence showing central dependence on an explicit kind-versus-instance distinction, or successful general capability without it.","retrospective":true,"plain_reading":true},"payload_hash":"ae4d518d9c10654b8b600b544fd5e02029542403f94f25854a762ec3d7a036ca","policy_version":"2026-10-28.7","recorded_at":"2026-10-29T07:10:59Z","drafter":{"provider":"openai","model":"gpt-6-astra"},"reviewers":[{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-does-ai-really-need-a-paradigm-shift-282b4f-4x-review-anthropic-1"},{"provider":"openai","model":"gpt-6-astra","run_id":"marcus-does-ai-really-need-a-paradigm-shift-282b4f-4x-review-openai-1"}],"final_text":{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-does-ai-really-need-a-paradigm-shift-282b4f-4x-final-anthropic-2"},"status":"current","notice":null,"superseded_by":null,"evidence_meta":[{"id":"e1","source_type":"company_disclosure","audited":null,"derived":null,"metric":null}],"public_from":null,"public_until":null},"history":[],"review":{"review_due":false,"triggers":[],"last_completed_review_at":"2026-10-29T07:10:59Z","next_review_due_at":null,"horizon_now":"open_horizon"},"event_ids":["marcus-does-ai-really-need-a-paradigm-shift-3bb4b4-assessment-published-a2"],"last_substantive_change_at":"2026-10-29T17:00:18Z"}},{"url":"https://punditclaims.ai/claims/marcus-does-ai-really-need-a-paradigm-shift-410d51","standing":"current","assessment_label":"Can’t be checked","versions":[{"assessment_id":"marcus-does-ai-really-need-a-paradigm-shift-410d51-a1","status":"current","url":"https://punditclaims.ai/claims/marcus-does-ai-really-need-a-paradigm-shift-410d51#assessment-marcus-does-ai-really-need-a-paradigm-shift-410d51-a1"}],"claim":{"claim_id":"marcus-does-ai-really-need-a-paradigm-shift-410d51","subject_id":"gary-marcus","revision":1,"content_hash":"939c6f78f5c5fa50b90aef1478a21b3025583f618bf28295a843b3539a781fef","normalized_proposition":"The failures of large-scale systems, after a heavily funded test of the scaling hypothesis, are evidence that the field should seriously consider alternatives to scaling.","kind":"normative","target":"AI research community","conditions":[{"text":"given failures of large scale systems","type":"qualifier","from_source":true}],"original_horizon_text":null,"topic_tags":["scaling"],"concerns_providers":[],"selection_stratum":"declared-lead","occurrences":[{"id":"marcus-does-ai-really-need-a-paradigm-shift-410d51-o1","statement_date":"2022-06-11","statement_date_precision":"day","excerpt":"I say that we have given the scaling hypothesis a really good look (with a larger budget than all but a handful of projects in history), and as such the failures of large scale systems are evidence (not proof) that we ought to seriously consider alternatives, e.g., here:","locator":"paragraph 12","document":{"url":"https://garymarcus.substack.com/p/does-ai-really-need-a-paradigm-shift","title":"Does AI really need a paradigm shift?","publisher":"Marcus on AI","published_at":"2022-06-11T17:32:54Z","date_uncertain":false,"date_provenance":"JSON-LD datePublished"},"observed_at":"2026-10-29T00:05:17Z","observed_via":"live","archive_url":null,"access_state":"full_public","snapshot_hash":"9716f559ab7b333c80f9fb7209db58ce8a0fde42f3f553914472aa7b9ff74804","receipts":1}],"first_statement_date":"2022-06-11","related":[],"current":{"assessment_id":"marcus-does-ai-really-need-a-paradigm-shift-410d51-a1","payload":{"claim_id":"marcus-does-ai-really-need-a-paradigm-shift-410d51","subject_id":"gary-marcus","normalized_proposition":"The failures of large-scale systems, after a heavily funded test of the scaling hypothesis, are evidence that the field should seriously consider alternatives to scaling.","kind":"normative","conditions":["given failures of large scale systems"],"original_horizon_text":null,"statement":{"excerpt":"I say that we have given the scaling hypothesis a really good look (with a larger budget than all but a handful of projects in history), and as such the failures of large scale systems are evidence (not proof) that we ought to seriously consider alternatives, e.g., here:","locator":"paragraph 12","url":"https://garymarcus.substack.com/p/does-ai-really-need-a-paradigm-shift","date":"2022-06-11","document_title":"Does AI really need a paradigm shift?","observed_at":"2026-10-29T00:05:17Z"},"criteria":{"from_speaker":["given failures of large scale systems"],"our_interpretation":[],"support_condition":"Not checked against evidence: both screens found that public evidence could not settle this claim.","contradiction_condition":"Not checked against evidence: both screens found that public evidence could not settle this claim."},"axis":"not_applicable","outcome":"not_adjudicable","horizon_state":"not_applicable","evaluation_as_of":"2026-10-28","rationale":"Not checked against evidence. Two independent screens, one from each provider, both found this a recommendation or a value judgment.","limitations":[],"evidence":[],"what_would_resolve":null,"retrospective":true,"uncheckable":["value_judgment"]},"payload_hash":"9dd4241990aecfff50d983bd3f72251d708baf99efcc302c2ef97f1e3dd69b16","policy_version":"2026-10-28.7","recorded_at":"2026-10-29T01:10:54Z","drafter":{"provider":"anthropic","model":"claude-opus-5-5"},"reviewers":[{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-does-ai-really-need-a-paradigm-shift-screen-anthropic-1"},{"provider":"openai","model":"gpt-6-astra","run_id":"marcus-does-ai-really-need-a-paradigm-shift-screen-openai-1"}],"final_text":null,"status":"current","notice":null,"superseded_by":null,"evidence_meta":[],"public_from":null,"public_until":null},"history":[],"review":{"review_due":false,"triggers":[],"last_completed_review_at":"2026-10-29T01:10:54Z","next_review_due_at":null,"horizon_now":"not_applicable"},"event_ids":["marcus-does-ai-really-need-a-paradigm-shift-410d51-assessment-published-a1"],"last_substantive_change_at":"2026-10-29T17:00:18Z"}},{"url":"https://punditclaims.ai/claims/marcus-does-ai-really-need-a-paradigm-shift-7639f9","standing":"current","assessment_label":"Unresolved","versions":[{"assessment_id":"marcus-does-ai-really-need-a-paradigm-shift-7639f9-a2","status":"current","url":"https://punditclaims.ai/claims/marcus-does-ai-really-need-a-paradigm-shift-7639f9#assessment-marcus-does-ai-really-need-a-paradigm-shift-7639f9-a2"}],"claim":{"claim_id":"marcus-does-ai-really-need-a-paradigm-shift-7639f9","subject_id":"gary-marcus","revision":2,"content_hash":"2897d3c9ba5955523fb4e16c4ebbf88cead89bd62112b51eb6e71a1293916fe1","normalized_proposition":"Whenever AGI arrives, explicit cognitive models will be crucial to it.","kind":"conditional_forecast","target":"explicit cognitive models in AGI","conditions":[{"text":"whenever AGI comes","type":"antecedent","from_source":true}],"original_horizon_text":null,"topic_tags":["agi-timelines","capabilities"],"concerns_providers":[],"selection_stratum":"declared-lead","occurrences":[{"id":"marcus-does-ai-really-need-a-paradigm-shift-7639f9-o1","statement_date":"2022-06-11","statement_date_precision":"day","excerpt":"👉explicit cognitive models will be crucial","locator":"prediction 2 of 4 (paragraph 56)","document":{"url":"https://garymarcus.substack.com/p/does-ai-really-need-a-paradigm-shift","title":"Does AI really need a paradigm shift?","publisher":"Marcus on AI","published_at":"2022-06-11T17:32:54Z","date_uncertain":false,"date_provenance":"JSON-LD datePublished"},"observed_at":"2026-10-29T00:05:17Z","observed_via":"live","archive_url":null,"access_state":"full_public","snapshot_hash":"9716f559ab7b333c80f9fb7209db58ce8a0fde42f3f553914472aa7b9ff74804","receipts":1}],"first_statement_date":"2022-06-11","related":[],"current":{"assessment_id":"marcus-does-ai-really-need-a-paradigm-shift-7639f9-a2","payload":{"claim_id":"marcus-does-ai-really-need-a-paradigm-shift-7639f9","subject_id":"gary-marcus","normalized_proposition":"Whenever AGI arrives, explicit cognitive models will be crucial to it.","kind":"conditional_forecast","conditions":["whenever AGI comes"],"original_horizon_text":null,"statement":{"excerpt":"👉explicit cognitive models will be crucial","locator":"prediction 2 of 4 (paragraph 56)","url":"https://garymarcus.substack.com/p/does-ai-really-need-a-paradigm-shift","date":"2022-06-11","document_title":"Does AI really need a paradigm shift?","observed_at":"2026-10-29T00:05:17Z"},"criteria":{"from_speaker":["whenever AGI comes","explicit cognitive models","will be crucial","tied not to time but to architecture"],"our_interpretation":["Treat arrival as a demonstrated system with broadly accepted human-level general cognitive capability. Broad acceptance is an evaluator safeguard.","Read explicit cognitive models as identifiable representations of entities, relations, and states that the system uses to understand situations and reason about them.","A researcher's model of the system's cognition does not itself qualify; the representations must operate within the system.","Read crucial as central dependence demonstrated through component tests or equivalent evidence.","Test the architecture of the first broadly accepted AGI; support for that case would not prove necessity for every possible AGI."],"support_condition":"The AGI condition is established, and public architectural and component evidence shows that explicit cognitive models are central to the arriving system's general capability.","contradiction_condition":"The AGI condition is established, and complete architectural evidence or controlled tests show that the system attains and retains general capability without explicit cognitive models."},"axis":"forecast_outcome","outcome":"unresolved","horizon_state":"open_horizon","evaluation_as_of":"2026-10-28","rationale":"IBM's July 14, 2026 update states that there is no academic consensus on exactly what qualifies as AGI [e1]. This background source establishes neither a qualifying AGI arrival nor dependence on explicit cognitive models [e1]. It therefore leaves the architectural prediction unresolved [e1].","limitations":["The July background summary does not establish AGI's status through October 28, 2026 [e1].","It provides no inspection or component tests of explicit cognitive models within a qualifying system [e1]."],"evidence":[{"id":"e1","relation":"context","excerpt":"no consensus","locator":"What is artificial general intelligence (AGI)?, paragraph 2","url":"https://www.ibm.com/think/topics/artificial-general-intelligence","publisher":"IBM","event_date":"2026-07-14","available_at":"2026-07-14","explanation":"This explains why AGI's arrival cannot be inferred from the label alone. It is background, not an independent capability assessment or evidence about cognitive-model necessity."}],"what_would_resolve":"A dated, broadly accepted AGI demonstration, with public documentation identifying internal cognitive models and controlled tests or equivalent evidence showing central dependence on them, or successful general capability without them.","retrospective":true,"plain_reading":true},"payload_hash":"0c45a66cc443060e08fb7dae3be898b8e0e3438e6eff7452d0eb6c494d013bae","policy_version":"2026-10-28.7","recorded_at":"2026-10-29T07:10:59Z","drafter":{"provider":"openai","model":"gpt-6-astra"},"reviewers":[{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-does-ai-really-need-a-paradigm-shift-282b4f-4x-review-anthropic-1"},{"provider":"openai","model":"gpt-6-astra","run_id":"marcus-does-ai-really-need-a-paradigm-shift-282b4f-4x-review-openai-1"}],"final_text":{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-does-ai-really-need-a-paradigm-shift-282b4f-4x-final-anthropic-2"},"status":"current","notice":null,"superseded_by":null,"evidence_meta":[{"id":"e1","source_type":"secondary_summary","audited":null,"derived":null,"metric":null}],"public_from":null,"public_until":null},"history":[],"review":{"review_due":false,"triggers":[],"last_completed_review_at":"2026-10-29T07:10:59Z","next_review_due_at":null,"horizon_now":"open_horizon"},"event_ids":["marcus-does-ai-really-need-a-paradigm-shift-7639f9-assessment-published-a2"],"last_substantive_change_at":"2026-10-29T17:00:18Z"}},{"url":"https://punditclaims.ai/claims/marcus-does-ai-really-need-a-paradigm-shift-b3d895","standing":"current","assessment_label":"Can’t be checked","versions":[{"assessment_id":"marcus-does-ai-really-need-a-paradigm-shift-b3d895-a1","status":"current","url":"https://punditclaims.ai/claims/marcus-does-ai-really-need-a-paradigm-shift-b3d895#assessment-marcus-does-ai-really-need-a-paradigm-shift-b3d895-a1"}],"claim":{"claim_id":"marcus-does-ai-really-need-a-paradigm-shift-b3d895","subject_id":"gary-marcus","revision":1,"content_hash":"ed0c3467774a5f2e6895aec28ed65a782ee9f52916ad7f516feb64913a10eff5","normalized_proposition":"A paradigm shift may be needed, but it could take a different form than the neurosymbolic one Marcus advocates.","kind":"forecast","target":"AI paradigm shift","conditions":[],"original_horizon_text":null,"topic_tags":["agi-timelines"],"concerns_providers":[],"selection_stratum":"declared-lead","occurrences":[{"id":"marcus-does-ai-really-need-a-paradigm-shift-b3d895-o1","statement_date":"2022-06-11","statement_date_precision":"day","excerpt":"That said, I, of course, could be right about foreseeing the need for a paradigm shift, but wrong about what that paradigm shift turns out to be.","locator":"paragraph 70","document":{"url":"https://garymarcus.substack.com/p/does-ai-really-need-a-paradigm-shift","title":"Does AI really need a paradigm shift?","publisher":"Marcus on AI","published_at":"2022-06-11T17:32:54Z","date_uncertain":false,"date_provenance":"JSON-LD datePublished"},"observed_at":"2026-10-29T00:05:17Z","observed_via":"live","archive_url":null,"access_state":"full_public","snapshot_hash":"9716f559ab7b333c80f9fb7209db58ce8a0fde42f3f553914472aa7b9ff74804","receipts":1}],"first_statement_date":"2022-06-11","related":[],"current":{"assessment_id":"marcus-does-ai-really-need-a-paradigm-shift-b3d895-a1","payload":{"claim_id":"marcus-does-ai-really-need-a-paradigm-shift-b3d895","subject_id":"gary-marcus","normalized_proposition":"A paradigm shift may be needed, but it could take a different form than the neurosymbolic one Marcus advocates.","kind":"forecast","conditions":[],"original_horizon_text":null,"statement":{"excerpt":"That said, I, of course, could be right about foreseeing the need for a paradigm shift, but wrong about what that paradigm shift turns out to be.","locator":"paragraph 70","url":"https://garymarcus.substack.com/p/does-ai-really-need-a-paradigm-shift","date":"2022-06-11","document_title":"Does AI really need a paradigm shift?","observed_at":"2026-10-29T00:05:17Z"},"criteria":{"from_speaker":[],"our_interpretation":[],"support_condition":"Not checked against evidence: both screens found that public evidence could not settle this claim.","contradiction_condition":"Not checked against evidence: both screens found that public evidence could not settle this claim."},"axis":"not_applicable","outcome":"not_adjudicable","horizon_state":"not_applicable","evaluation_as_of":"2026-10-28","rationale":"Not checked against evidence. Two independent screens, one from each provider, both found this stated as a chance or a possibility, or at medium or low confidence.","limitations":[],"evidence":[],"what_would_resolve":null,"retrospective":true,"uncheckable":["probability_language"]},"payload_hash":"83c9af24cc8b5913c86c956b45dffaaf783f0ba381e3296f55a85e0bcc52befa","policy_version":"2026-10-28.7","recorded_at":"2026-10-29T01:10:54Z","drafter":{"provider":"anthropic","model":"claude-opus-5-5"},"reviewers":[{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-does-ai-really-need-a-paradigm-shift-screen-anthropic-1"},{"provider":"openai","model":"gpt-6-astra","run_id":"marcus-does-ai-really-need-a-paradigm-shift-screen-openai-1"}],"final_text":null,"status":"current","notice":null,"superseded_by":null,"evidence_meta":[],"public_from":null,"public_until":null},"history":[],"review":{"review_due":false,"triggers":[],"last_completed_review_at":"2026-10-29T01:10:54Z","next_review_due_at":null,"horizon_now":"not_applicable"},"event_ids":["marcus-does-ai-really-need-a-paradigm-shift-b3d895-assessment-published-a1"],"last_substantive_change_at":"2026-10-29T17:00:18Z"}},{"url":"https://punditclaims.ai/claims/marcus-does-ai-really-need-a-paradigm-shift-b5bb62","standing":"current","assessment_label":"Supported","versions":[{"assessment_id":"marcus-does-ai-really-need-a-paradigm-shift-b5bb62-a1","status":"current","url":"https://punditclaims.ai/claims/marcus-does-ai-really-need-a-paradigm-shift-b5bb62#assessment-marcus-does-ai-really-need-a-paradigm-shift-b5bb62-a1"}],"claim":{"claim_id":"marcus-does-ai-really-need-a-paradigm-shift-b5bb62","subject_id":"gary-marcus","revision":2,"content_hash":"9b1af26fbee594258341ef3b3541f25d18429b697d8d5d423e5f06da27a0a65d","normalized_proposition":"The BIG-bench paper looked at scaling on many measures and found progress on some but not others, and the measures that did not scale in it are what Marcus calls the 'wall'.","kind":"observation","target":"deep learning scaling","conditions":[{"text":"in the BIG-bench paper's findings","type":"scope","from_source":true},{"text":"progress on some but not others","type":"scope","from_source":true}],"original_horizon_text":null,"topic_tags":["scaling"],"concerns_providers":[],"selection_stratum":"declared-lead","occurrences":[{"id":"marcus-does-ai-really-need-a-paradigm-shift-b5bb62-o1","statement_date":"2022-06-11","statement_date_precision":"day","excerpt":"The massive paper looks at scaling on many measures, and sees progress on some– but not others. The stuff that isn’t scaling is the “wall”.","locator":"paragraph 37","document":{"url":"https://garymarcus.substack.com/p/does-ai-really-need-a-paradigm-shift","title":"Does AI really need a paradigm shift?","publisher":"Marcus on AI","published_at":"2022-06-11T17:32:54Z","date_uncertain":false,"date_provenance":"JSON-LD datePublished"},"observed_at":"2026-10-29T00:05:17Z","observed_via":"live","archive_url":null,"access_state":"full_public","snapshot_hash":"9716f559ab7b333c80f9fb7209db58ce8a0fde42f3f553914472aa7b9ff74804","receipts":1}],"first_statement_date":"2022-06-11","related":[],"current":{"assessment_id":"marcus-does-ai-really-need-a-paradigm-shift-b5bb62-a1","payload":{"claim_id":"marcus-does-ai-really-need-a-paradigm-shift-b5bb62","subject_id":"gary-marcus","normalized_proposition":"The BIG-bench paper looked at scaling on many measures and found progress on some but not others, and the measures that did not scale in it are what Marcus calls the 'wall'.","kind":"observation","conditions":["in the BIG-bench paper's findings","progress on some but not others"],"original_horizon_text":null,"statement":{"excerpt":"The massive paper looks at scaling on many measures, and sees progress on some– but not others. The stuff that isn’t scaling is the “wall”.","locator":"paragraph 37","url":"https://garymarcus.substack.com/p/does-ai-really-need-a-paradigm-shift","date":"2022-06-11","document_title":"Does AI really need a paradigm shift?","observed_at":"2026-10-29T00:05:17Z"},"criteria":{"from_speaker":["The massive paper looks at scaling on many measures","sees progress on some– but not others","The stuff that isn’t scaling is the “wall”"],"our_interpretation":["The paper is the original June 2022 BIG-bench report.","Progress means improvement in the reported task metrics across evaluated model sizes.","Marcus explicitly labels the non-improving measures the wall. In this sentence, that label does not independently establish that they can never improve."],"support_condition":"The original paper evaluates many tasks and reports improving scores on some while other scores remain flat over the tested scales.","contradiction_condition":"The original paper does not report that pattern, or every evaluated task measure improves with scale."},"axis":"historical_observation","outcome":"supported","horizon_state":"not_applicable","evaluation_as_of":"2026-10-28","rationale":"The original BIG-bench paper explicitly reports the pattern Marcus describes. His use of wall names the measures that did not improve in those experiments. The evidence supports that historical description, while leaving future scalability open.","limitations":["The finding applies to the evaluated models, scales and metrics.","Some metrics can conceal gradual progress.","This assessment does not establish an unavoidable or permanent wall."],"evidence":[{"id":"e1","relation":"supports","excerpt":"no improvement with scale","locator":"Section 3.4, Linearity and breakthroughness: categorizing scaling behaviors","url":"https://arxiv.org/html/2206.04615v1","publisher":"Srivastava et al.; arXiv","event_date":"2022-06-09","available_at":"2022-06-09","explanation":"The report directly confirms improving and non-improving task scores across evaluated scales. Its discussion of smoother metrics limits what those flat scores imply about underlying capabilities."}],"what_would_resolve":null,"retrospective":true,"plain_reading":true},"payload_hash":"9ea28de6b0fb618045a79207f884521e98139abbb5cf68f799abf803d23f0ff7","policy_version":"2026-10-28.7","recorded_at":"2026-10-29T07:00:46Z","drafter":{"provider":"openai","model":"gpt-6-astra"},"reviewers":[{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-does-ai-really-need-a-paradigm-shift-828976-4x-review-anthropic-1"},{"provider":"openai","model":"gpt-6-astra","run_id":"marcus-does-ai-really-need-a-paradigm-shift-828976-4x-review-openai-1"}],"final_text":{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-does-ai-really-need-a-paradigm-shift-828976-4x-final-anthropic-1"},"status":"current","notice":null,"superseded_by":null,"evidence_meta":[{"id":"e1","source_type":"preprint","audited":null,"derived":null,"metric":null}],"public_from":null,"public_until":null},"history":[],"review":{"review_due":false,"triggers":[],"last_completed_review_at":"2026-10-29T07:00:46Z","next_review_due_at":null,"horizon_now":"not_applicable"},"event_ids":["marcus-does-ai-really-need-a-paradigm-shift-b5bb62-assessment-published-a1"],"last_substantive_change_at":"2026-10-29T17:00:18Z"}},{"url":"https://punditclaims.ai/claims/marcus-does-ai-really-need-a-paradigm-shift-b9155e","standing":"current","assessment_label":"Can’t be checked","versions":[{"assessment_id":"marcus-does-ai-really-need-a-paradigm-shift-b9155e-a1","status":"current","url":"https://punditclaims.ai/claims/marcus-does-ai-really-need-a-paradigm-shift-b9155e#assessment-marcus-does-ai-really-need-a-paradigm-shift-b9155e-a1"}],"claim":{"claim_id":"marcus-does-ai-really-need-a-paradigm-shift-b9155e","subject_id":"gary-marcus","revision":2,"content_hash":"bfa4ccb1f7b626caaf8d6e36459b5a6a96899826ae9d93be2ab07dad683e652a","normalized_proposition":"AI research needs new approaches rather than scale alone.","kind":"normative","target":"AI research","conditions":[],"original_horizon_text":null,"topic_tags":["scaling"],"concerns_providers":[],"selection_stratum":"declared-lead","occurrences":[{"id":"marcus-does-ai-really-need-a-paradigm-shift-b9155e-o1","statement_date":"2022-06-11","statement_date_precision":"day","excerpt":"That’s why we need new approaches, exactly as I have been arguing.","locator":"paragraph 38","document":{"url":"https://garymarcus.substack.com/p/does-ai-really-need-a-paradigm-shift","title":"Does AI really need a paradigm shift?","publisher":"Marcus on AI","published_at":"2022-06-11T17:32:54Z","date_uncertain":false,"date_provenance":"JSON-LD datePublished"},"observed_at":"2026-10-29T00:05:17Z","observed_via":"live","archive_url":null,"access_state":"full_public","snapshot_hash":"9716f559ab7b333c80f9fb7209db58ce8a0fde42f3f553914472aa7b9ff74804","receipts":1}],"first_statement_date":"2022-06-11","related":[],"current":{"assessment_id":"marcus-does-ai-really-need-a-paradigm-shift-b9155e-a1","payload":{"claim_id":"marcus-does-ai-really-need-a-paradigm-shift-b9155e","subject_id":"gary-marcus","normalized_proposition":"AI research needs new approaches rather than scale alone.","kind":"normative","conditions":[],"original_horizon_text":null,"statement":{"excerpt":"That’s why we need new approaches, exactly as I have been arguing.","locator":"paragraph 38","url":"https://garymarcus.substack.com/p/does-ai-really-need-a-paradigm-shift","date":"2022-06-11","document_title":"Does AI really need a paradigm shift?","observed_at":"2026-10-29T00:05:17Z"},"criteria":{"from_speaker":[],"our_interpretation":[],"support_condition":"Not checked against evidence: both screens found that public evidence could not settle this claim.","contradiction_condition":"Not checked against evidence: both screens found that public evidence could not settle this claim."},"axis":"not_applicable","outcome":"not_adjudicable","horizon_state":"not_applicable","evaluation_as_of":"2026-10-28","rationale":"Not checked against evidence. Two independent screens, one from each provider, both found this a recommendation or a value judgment.","limitations":[],"evidence":[],"what_would_resolve":null,"retrospective":true,"uncheckable":["value_judgment"]},"payload_hash":"a1c6963f262cf9eb28fd65e7236e0d2bc4a11fa701bba5837bb2a35f120f920c","policy_version":"2026-10-28.7","recorded_at":"2026-10-29T02:32:05Z","drafter":{"provider":"anthropic","model":"claude-opus-5-5"},"reviewers":[{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-does-ai-really-need-a-paradigm-shift-screen-anthropic-2"},{"provider":"openai","model":"gpt-6-astra","run_id":"marcus-does-ai-really-need-a-paradigm-shift-screen-openai-2"}],"final_text":null,"status":"current","notice":null,"superseded_by":null,"evidence_meta":[],"public_from":null,"public_until":null},"history":[],"review":{"review_due":false,"triggers":[],"last_completed_review_at":"2026-10-29T02:32:05Z","next_review_due_at":null,"horizon_now":"not_applicable"},"event_ids":["marcus-does-ai-really-need-a-paradigm-shift-b9155e-assessment-published-a1"],"last_substantive_change_at":"2026-10-29T17:00:18Z"}},{"url":"https://punditclaims.ai/claims/marcus-does-ai-really-need-a-paradigm-shift-c220b8","standing":"current","assessment_label":"Unresolved","versions":[{"assessment_id":"marcus-does-ai-really-need-a-paradigm-shift-c220b8-a2","status":"current","url":"https://punditclaims.ai/claims/marcus-does-ai-really-need-a-paradigm-shift-c220b8#assessment-marcus-does-ai-really-need-a-paradigm-shift-c220b8-a2"}],"claim":{"claim_id":"marcus-does-ai-really-need-a-paradigm-shift-c220b8","subject_id":"gary-marcus","revision":2,"content_hash":"81906cbc94ac0f2f17899904a1ae209698926a1f24641bc272f103300ad4068e","normalized_proposition":"Whenever AGI arrives, operations over variables, including storing, retrieving and comparing values, will be crucial to it.","kind":"conditional_forecast","target":"operations over variables in AGI","conditions":[{"text":"whenever AGI comes","type":"antecedent","from_source":true}],"original_horizon_text":null,"topic_tags":["agi-timelines","capabilities"],"concerns_providers":[],"selection_stratum":"declared-lead","occurrences":[{"id":"marcus-does-ai-really-need-a-paradigm-shift-c220b8-o1","statement_date":"2022-06-11","statement_date_precision":"day","excerpt":"👉operations over variables (including storing, retrieving and comparing values) will be crucial","locator":"prediction 3 of 4 (paragraph 57)","document":{"url":"https://garymarcus.substack.com/p/does-ai-really-need-a-paradigm-shift","title":"Does AI really need a paradigm shift?","publisher":"Marcus on AI","published_at":"2022-06-11T17:32:54Z","date_uncertain":false,"date_provenance":"JSON-LD datePublished"},"observed_at":"2026-10-29T00:05:17Z","observed_via":"live","archive_url":null,"access_state":"full_public","snapshot_hash":"9716f559ab7b333c80f9fb7209db58ce8a0fde42f3f553914472aa7b9ff74804","receipts":1}],"first_statement_date":"2022-06-11","related":[],"current":{"assessment_id":"marcus-does-ai-really-need-a-paradigm-shift-c220b8-a2","payload":{"claim_id":"marcus-does-ai-really-need-a-paradigm-shift-c220b8","subject_id":"gary-marcus","normalized_proposition":"Whenever AGI arrives, operations over variables, including storing, retrieving and comparing values, will be crucial to it.","kind":"conditional_forecast","conditions":["whenever AGI comes"],"original_horizon_text":null,"statement":{"excerpt":"👉operations over variables (including storing, retrieving and comparing values) will be crucial","locator":"prediction 3 of 4 (paragraph 57)","url":"https://garymarcus.substack.com/p/does-ai-really-need-a-paradigm-shift","date":"2022-06-11","document_title":"Does AI really need a paradigm shift?","observed_at":"2026-10-29T00:05:17Z"},"criteria":{"from_speaker":["whenever AGI comes","operations over variables","including storing, retrieving and comparing values","will be crucial"],"our_interpretation":["Treat arrival as a demonstrated system with broadly accepted human-level general cognitive capability. Broad acceptance is an evaluator safeguard.","Read operations over variables as reusable bindings between variables and values, with operations that store, retrieve, and compare those values in the system's reasoning.","The claim does not explicitly require a separate symbolic module. A neural implementation can qualify if it demonstrably performs the specified operations.","Incidental variables in implementation code do not establish a crucial cognitive mechanism.","Read crucial as central dependence. Test the arriving AGI without treating one implementation as proof about every possible AGI."],"support_condition":"The AGI condition is established, and architectural or mechanistic evidence shows that storing, retrieving, and comparing variable-bound values centrally enables its general capability.","contradiction_condition":"The AGI condition is established, and sufficiently complete mechanistic evidence shows that it attains general capability without the specified variable operations, or retains that capability when they are disabled."},"axis":"forecast_outcome","outcome":"unresolved","horizon_state":"open_horizon","evaluation_as_of":"2026-10-28","rationale":"On March 17, 2026, Google DeepMind announced a framework for measuring progress toward AGI and an initiative to develop evaluations [e1]. That announcement establishes neither a broadly accepted AGI demonstration nor whether variable operations centrally enable such a system's general capability, so it does not settle the conditional forecast [e1].","limitations":["The packet does not establish the AGI antecedent; this does not establish that AGI is absent.","No qualifying system has accessible mechanistic evidence establishing or excluding central dependence on the specified operations.","Ordinary memory use or success on variable-like tasks alone would not establish the required cognitive mechanism or its importance.","A candidate system-card PDF could not be inspected."],"evidence":[{"id":"e1","relation":"context","excerpt":"measure progress toward AGI","locator":"Article subtitle","url":"https://deepmind.google/blog/measuring-progress-toward-agi-a-cognitive-framework","publisher":"Google DeepMind","event_date":"2026-03-17","available_at":"2026-03-17","explanation":"This announces work on measuring progress. It does not demonstrate the antecedent or identify variable operations as necessary to an AGI."}],"what_would_resolve":"A dated, broadly accepted AGI demonstration, together with mechanistic evidence and interventions showing that reusable variable bindings and value storage, retrieval and comparison centrally enable its general capability—or that it achieves or retains that capability without those operations.","retrospective":true,"plain_reading":true},"payload_hash":"d7efe9fe42e7c309343587b2c148b9749a529d11cb1d6952c1dbf9b2f2716052","policy_version":"2026-10-28.7","recorded_at":"2026-10-29T07:11:13Z","drafter":{"provider":"openai","model":"gpt-6-astra"},"reviewers":[{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-does-ai-really-need-a-paradigm-shift-828976-4x-review-anthropic-1"},{"provider":"openai","model":"gpt-6-astra","run_id":"marcus-does-ai-really-need-a-paradigm-shift-828976-4x-review-openai-1"}],"final_text":{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-does-ai-really-need-a-paradigm-shift-828976-2x-final-anthropic-1"},"status":"current","notice":null,"superseded_by":null,"evidence_meta":[{"id":"e1","source_type":"company_disclosure","audited":null,"derived":null,"metric":null}],"public_from":null,"public_until":null},"history":[],"review":{"review_due":false,"triggers":[],"last_completed_review_at":"2026-10-29T07:11:13Z","next_review_due_at":null,"horizon_now":"open_horizon"},"event_ids":["marcus-does-ai-really-need-a-paradigm-shift-c220b8-assessment-published-a2"],"last_substantive_change_at":"2026-10-29T17:00:18Z"}},{"url":"https://punditclaims.ai/claims/marcus-does-ai-really-need-a-paradigm-shift-c94cb0","standing":"current","assessment_label":"Can’t be checked","versions":[{"assessment_id":"marcus-does-ai-really-need-a-paradigm-shift-c94cb0-a1","status":"current","url":"https://punditclaims.ai/claims/marcus-does-ai-really-need-a-paradigm-shift-c94cb0#assessment-marcus-does-ai-really-need-a-paradigm-shift-c94cb0-a1"}],"claim":{"claim_id":"marcus-does-ai-really-need-a-paradigm-shift-c94cb0","subject_id":"gary-marcus","revision":1,"content_hash":"89f64c9fa8abc6650e7a586df371fc59caa79118d4b32b9c583cd3278d8d5173","normalized_proposition":"Marcus puts a 90% probability on AI needing a paradigm shift to reach AGI.","kind":"forecast","target":"AI research toward AGI","conditions":[],"original_horizon_text":null,"topic_tags":["agi-timelines","scaling"],"concerns_providers":[],"selection_stratum":"declared-lead","occurrences":[{"id":"marcus-does-ai-really-need-a-paradigm-shift-c94cb0-o1","statement_date":"2022-06-11","statement_date_precision":"day","excerpt":"Personally I’d put the probability that we need to shift at 90%, well above the 60% that Alexander suggests, and I would put the probability that we will need to embrace symbol-manipulation as part of the mix at 80%, more than double Alexander’s 34%.","locator":"paragraph 30","document":{"url":"https://garymarcus.substack.com/p/does-ai-really-need-a-paradigm-shift","title":"Does AI really need a paradigm shift?","publisher":"Marcus on AI","published_at":"2022-06-11T17:32:54Z","date_uncertain":false,"date_provenance":"JSON-LD datePublished"},"observed_at":"2026-10-29T00:05:17Z","observed_via":"live","archive_url":null,"access_state":"full_public","snapshot_hash":"9716f559ab7b333c80f9fb7209db58ce8a0fde42f3f553914472aa7b9ff74804","receipts":1}],"first_statement_date":"2022-06-11","related":[],"current":{"assessment_id":"marcus-does-ai-really-need-a-paradigm-shift-c94cb0-a1","payload":{"claim_id":"marcus-does-ai-really-need-a-paradigm-shift-c94cb0","subject_id":"gary-marcus","normalized_proposition":"Marcus puts a 90% probability on AI needing a paradigm shift to reach AGI.","kind":"forecast","conditions":[],"original_horizon_text":null,"statement":{"excerpt":"Personally I’d put the probability that we need to shift at 90%, well above the 60% that Alexander suggests, and I would put the probability that we will need to embrace symbol-manipulation as part of the mix at 80%, more than double Alexander’s 34%.","locator":"paragraph 30","url":"https://garymarcus.substack.com/p/does-ai-really-need-a-paradigm-shift","date":"2022-06-11","document_title":"Does AI really need a paradigm shift?","observed_at":"2026-10-29T00:05:17Z"},"criteria":{"from_speaker":["90%"],"our_interpretation":[],"support_condition":"Not checked against evidence: both screens found that public evidence could not settle this claim.","contradiction_condition":"Not checked against evidence: both screens found that public evidence could not settle this claim."},"axis":"not_applicable","outcome":"not_adjudicable","horizon_state":"not_applicable","evaluation_as_of":"2026-10-28","rationale":"Not checked against evidence. Two independent screens, one from each provider, both found this stated as a chance or a possibility, or at medium or low confidence.","limitations":[],"evidence":[],"what_would_resolve":null,"retrospective":true,"uncheckable":["probability_language"]},"payload_hash":"6888f47ab5fc45efff08ea8d8b4d47a275ab75444eb06aba210ab12fa75c9d03","policy_version":"2026-10-28.7","recorded_at":"2026-10-29T01:10:54Z","drafter":{"provider":"anthropic","model":"claude-opus-5-5"},"reviewers":[{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-does-ai-really-need-a-paradigm-shift-screen-anthropic-1"},{"provider":"openai","model":"gpt-6-astra","run_id":"marcus-does-ai-really-need-a-paradigm-shift-screen-openai-1"}],"final_text":null,"status":"current","notice":null,"superseded_by":null,"evidence_meta":[],"public_from":null,"public_until":null},"history":[],"review":{"review_due":false,"triggers":[],"last_completed_review_at":"2026-10-29T01:10:54Z","next_review_due_at":null,"horizon_now":"not_applicable"},"event_ids":["marcus-does-ai-really-need-a-paradigm-shift-c94cb0-assessment-published-a1"],"last_substantive_change_at":"2026-10-29T17:00:18Z"}},{"url":"https://punditclaims.ai/claims/marcus-does-ai-really-need-a-paradigm-shift-c94cb0-2","standing":"current","assessment_label":"Can’t be checked","versions":[{"assessment_id":"marcus-does-ai-really-need-a-paradigm-shift-c94cb0-2-a1","status":"current","url":"https://punditclaims.ai/claims/marcus-does-ai-really-need-a-paradigm-shift-c94cb0-2#assessment-marcus-does-ai-really-need-a-paradigm-shift-c94cb0-2-a1"}],"claim":{"claim_id":"marcus-does-ai-really-need-a-paradigm-shift-c94cb0-2","subject_id":"gary-marcus","revision":1,"content_hash":"93cba459192d07fd068d745a90e022f41038f39c124c7bd7d33abc690bf0c9ee","normalized_proposition":"Marcus puts an 80% probability on AGI needing symbol manipulation as part of the mix.","kind":"forecast","target":"AI research toward AGI","conditions":[],"original_horizon_text":null,"topic_tags":["agi-timelines"],"concerns_providers":[],"selection_stratum":"declared-lead","occurrences":[{"id":"marcus-does-ai-really-need-a-paradigm-shift-c94cb0-2-o1","statement_date":"2022-06-11","statement_date_precision":"day","excerpt":"Personally I’d put the probability that we need to shift at 90%, well above the 60% that Alexander suggests, and I would put the probability that we will need to embrace symbol-manipulation as part of the mix at 80%, more than double Alexander’s 34%.","locator":"paragraph 30","document":{"url":"https://garymarcus.substack.com/p/does-ai-really-need-a-paradigm-shift","title":"Does AI really need a paradigm shift?","publisher":"Marcus on AI","published_at":"2022-06-11T17:32:54Z","date_uncertain":false,"date_provenance":"JSON-LD datePublished"},"observed_at":"2026-10-29T00:05:17Z","observed_via":"live","archive_url":null,"access_state":"full_public","snapshot_hash":"9716f559ab7b333c80f9fb7209db58ce8a0fde42f3f553914472aa7b9ff74804","receipts":1}],"first_statement_date":"2022-06-11","related":[],"current":{"assessment_id":"marcus-does-ai-really-need-a-paradigm-shift-c94cb0-2-a1","payload":{"claim_id":"marcus-does-ai-really-need-a-paradigm-shift-c94cb0-2","subject_id":"gary-marcus","normalized_proposition":"Marcus puts an 80% probability on AGI needing symbol manipulation as part of the mix.","kind":"forecast","conditions":[],"original_horizon_text":null,"statement":{"excerpt":"Personally I’d put the probability that we need to shift at 90%, well above the 60% that Alexander suggests, and I would put the probability that we will need to embrace symbol-manipulation as part of the mix at 80%, more than double Alexander’s 34%.","locator":"paragraph 30","url":"https://garymarcus.substack.com/p/does-ai-really-need-a-paradigm-shift","date":"2022-06-11","document_title":"Does AI really need a paradigm shift?","observed_at":"2026-10-29T00:05:17Z"},"criteria":{"from_speaker":["80%"],"our_interpretation":[],"support_condition":"Not checked against evidence: both screens found that public evidence could not settle this claim.","contradiction_condition":"Not checked against evidence: both screens found that public evidence could not settle this claim."},"axis":"not_applicable","outcome":"not_adjudicable","horizon_state":"not_applicable","evaluation_as_of":"2026-10-28","rationale":"Not checked against evidence. Two independent screens, one from each provider, both found this stated as a chance or a possibility, or at medium or low confidence.","limitations":[],"evidence":[],"what_would_resolve":null,"retrospective":true,"uncheckable":["probability_language"]},"payload_hash":"ee1526435a391faef6c0a158a0ebb5d4313022113bb9eb019b285500d7cb0772","policy_version":"2026-10-28.7","recorded_at":"2026-10-29T01:10:54Z","drafter":{"provider":"anthropic","model":"claude-opus-5-5"},"reviewers":[{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-does-ai-really-need-a-paradigm-shift-screen-anthropic-1"},{"provider":"openai","model":"gpt-6-astra","run_id":"marcus-does-ai-really-need-a-paradigm-shift-screen-openai-1"}],"final_text":null,"status":"current","notice":null,"superseded_by":null,"evidence_meta":[],"public_from":null,"public_until":null},"history":[],"review":{"review_due":false,"triggers":[],"last_completed_review_at":"2026-10-29T01:10:54Z","next_review_due_at":null,"horizon_now":"not_applicable"},"event_ids":["marcus-does-ai-really-need-a-paradigm-shift-c94cb0-2-assessment-published-a1"],"last_substantive_change_at":"2026-10-29T17:00:18Z"}},{"url":"https://punditclaims.ai/claims/marcus-does-ai-really-need-a-paradigm-shift-fb3686","standing":"current","assessment_label":"Supported","versions":[{"assessment_id":"marcus-does-ai-really-need-a-paradigm-shift-fb3686-a1","status":"current","url":"https://punditclaims.ai/claims/marcus-does-ai-really-need-a-paradigm-shift-fb3686#assessment-marcus-does-ai-really-need-a-paradigm-shift-fb3686-a1"}],"claim":{"claim_id":"marcus-does-ai-really-need-a-paradigm-shift-fb3686","subject_id":"gary-marcus","revision":1,"content_hash":"e6d0f9ca61fb59b09565acd77a8e1ba21c7cb4516456bbeeb2c66809c6713642","normalized_proposition":"Many of the harmful consequences of the webscraping/big-data approach to AI fall disproportionately on women and minorities.","kind":"observation","target":"webscraping/big data approach to AI","conditions":[],"original_horizon_text":null,"topic_tags":["safety"],"concerns_providers":[],"selection_stratum":"declared-lead","occurrences":[{"id":"marcus-does-ai-really-need-a-paradigm-shift-fb3686-o1","statement_date":"2022-06-11","statement_date_precision":"day","excerpt":"so many of the unfortunate consequences of a webscraping/big data approach to AI are disproportionately borne by women and minorities.","locator":"paragraph 19","document":{"url":"https://garymarcus.substack.com/p/does-ai-really-need-a-paradigm-shift","title":"Does AI really need a paradigm shift?","publisher":"Marcus on AI","published_at":"2022-06-11T17:32:54Z","date_uncertain":false,"date_provenance":"JSON-LD datePublished"},"observed_at":"2026-10-29T00:05:17Z","observed_via":"live","archive_url":null,"access_state":"full_public","snapshot_hash":"9716f559ab7b333c80f9fb7209db58ce8a0fde42f3f553914472aa7b9ff74804","receipts":1}],"first_statement_date":"2022-06-11","related":[],"current":{"assessment_id":"marcus-does-ai-really-need-a-paradigm-shift-fb3686-a1","payload":{"claim_id":"marcus-does-ai-really-need-a-paradigm-shift-fb3686","subject_id":"gary-marcus","normalized_proposition":"Many of the harmful consequences of the webscraping/big-data approach to AI fall disproportionately on women and minorities.","kind":"observation","conditions":[],"original_horizon_text":null,"statement":{"excerpt":"so many of the unfortunate consequences of a webscraping/big data approach to AI are disproportionately borne by women and minorities.","locator":"paragraph 19","url":"https://garymarcus.substack.com/p/does-ai-really-need-a-paradigm-shift","date":"2022-06-11","document_title":"Does AI really need a paradigm shift?","observed_at":"2026-10-29T00:05:17Z"},"criteria":{"from_speaker":["so many of the unfortunate consequences","a webscraping/big data approach to AI","disproportionately borne by women and minorities"],"our_interpretation":["The plain reading is that multiple documented kinds of harm from this approach disproportionately affect women and minority groups. It does not require every harm to affect every such group.","Harm includes exclusion from training data, degrading representation, and discriminatory model outputs.","The observation concerns the research record and systems existing before the post. There is no standard reporting quarter for this subject; the target ends on the post date and has no stated beginning.","This assesses the distribution of documented harms, rather than whether web scraping alone caused them."],"support_condition":"Primary audits of existing datasets and models document multiple forms of disproportionate harm involving women and minority groups.","contradiction_condition":"Broad, comparable audits show that the alleged disparities are absent or reversed across the relevant harms, undermining the claimed recurring pattern."},"axis":"historical_observation","outcome":"supported","horizon_state":"not_applicable","evaluation_as_of":"2026-10-28","rationale":"The pre-post research documents several relevant disparities: exclusion of minority-associated language, sexualized representations of women, and degrading racial classifications. Together they support the ordinary qualitative reading of a recurring pattern. They do not establish that women and minorities bear most total AI harm, or that every harm falls disproportionately on those groups.","limitations":["Support depends on reading many qualitatively and including representational and data-exclusion harms.","The studies cover selected systems and probes, not a representative census of AI harms.","Some measured harms disproportionately affected men."],"evidence":[{"id":"e1","relation":"supports","excerpt":"disproportionately affected","locator":"Section 5.3, paragraph reporting dialect exclusion rates","url":"https://arxiv.org/html/2104.08758v2","publisher":"Dodge et al.; EMNLP 2021, arXiv version","event_date":"2021","available_at":"2021-09-30","explanation":"The audit reports removal rates of 42% for African American English and 32% for Hispanic-aligned English, against 6.2% for White-aligned English. This documents unequal exclusion from a web-scraped training corpus, using dialect as a proxy rather than identifying authors."},{"id":"e2","relation":"supports","excerpt":"returned pornographic imagery","locator":"Section 2.1, Misogyny and stereotypes","url":"https://arxiv.org/html/2110.01963v1","publisher":"Birhane, Prabhu and Kahembwe; arXiv","event_date":"2021-09-12","available_at":"2021-10-05","explanation":"The audit documents sexualized retrieval for ordinary feminine and demographic terms, including terms referring to minority women. This supports unequal representational harm, although the exploratory searches do not estimate its population-wide prevalence."},{"id":"e3","relation":"supports","excerpt":"significant disparities","locator":"Section 7, Broader Impacts, FairFace bias probes and Table 6","url":"https://arxiv.org/html/2103.00020v1","publisher":"Radford et al.; arXiv","event_date":"2021","available_at":"2021-02-26","explanation":"CLIP's audit reports nonhuman classifications for approximately 14% of Black images, versus under 8% for every other racial group. It also qualifies any universal gender claim: crime-related misclassification affected male images more often than female images, 16.5% versus 9.8%."}],"what_would_resolve":null,"retrospective":true,"plain_reading":true},"payload_hash":"15d69aed24457310145ffd63c28c99678fb72807cf900eca7bb0c11673e55179","policy_version":"2026-10-28.7","recorded_at":"2026-10-29T06:59:19Z","drafter":{"provider":"openai","model":"gpt-6-astra"},"reviewers":[{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-does-ai-really-need-a-paradigm-shift-fb3686-1x-review-anthropic-1"},{"provider":"openai","model":"gpt-6-astra","run_id":"marcus-does-ai-really-need-a-paradigm-shift-fb3686-1x-review-openai-1"}],"final_text":{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-does-ai-really-need-a-paradigm-shift-fb3686-1x-final-anthropic-1"},"status":"current","notice":null,"superseded_by":null,"evidence_meta":[{"id":"e1","source_type":"peer_reviewed","audited":null,"derived":null,"metric":null},{"id":"e2","source_type":"preprint","audited":null,"derived":null,"metric":null},{"id":"e3","source_type":"preprint","audited":null,"derived":null,"metric":null}],"public_from":null,"public_until":null},"history":[],"review":{"review_due":false,"triggers":[],"last_completed_review_at":"2026-10-29T06:59:19Z","next_review_due_at":null,"horizon_now":"not_applicable"},"event_ids":["marcus-does-ai-really-need-a-paradigm-shift-fb3686-assessment-published-a1"],"last_substantive_change_at":"2026-10-29T17:00:18Z"}},{"url":"https://punditclaims.ai/claims/marcus-what-does-it-mean-when-an-ai-fails-112099","standing":"current","assessment_label":"Can’t be checked","versions":[{"assessment_id":"marcus-what-does-it-mean-when-an-ai-fails-112099-a1","status":"current","url":"https://punditclaims.ai/claims/marcus-what-does-it-mean-when-an-ai-fails-112099#assessment-marcus-what-does-it-mean-when-an-ai-fails-112099-a1"}],"claim":{"claim_id":"marcus-what-does-it-mean-when-an-ai-fails-112099","subject_id":"gary-marcus","revision":1,"content_hash":"37757c4e1b83d768e63456538940649450a9fcc102223f3bdc09f12e6183435a","normalized_proposition":"Scott Alexander's finding that InstructGPT got four of six previously failed prompts right may have been a fluke.","kind":"observation","target":"InstructGPT","conditions":[],"original_horizon_text":null,"topic_tags":["capabilities"],"concerns_providers":["openai"],"selection_stratum":"declared-lead","occurrences":[{"id":"marcus-what-does-it-mean-when-an-ai-fails-112099-o1","statement_date":"2022-06-07","statement_date_precision":"day","excerpt":"Scott’s more marked success on six questions may have been a fluke.","locator":"paragraph 57","document":{"url":"https://garymarcus.substack.com/p/what-does-it-mean-when-an-ai-fails","title":"What does it mean when an AI fails? A Reply to SlateStarCodex’s riff on Gary Marcus","publisher":"Marcus on AI","published_at":"2022-06-07T17:28:09Z","date_uncertain":false,"date_provenance":"JSON-LD datePublished"},"observed_at":"2026-10-29T00:05:19Z","observed_via":"live","archive_url":null,"access_state":"full_public","snapshot_hash":"fa2a39c0800cc5475acd393215c920495373f319a6dd41cc57a65d2d3d491dbc","receipts":1}],"first_statement_date":"2022-06-07","related":[],"current":{"assessment_id":"marcus-what-does-it-mean-when-an-ai-fails-112099-a1","payload":{"claim_id":"marcus-what-does-it-mean-when-an-ai-fails-112099","subject_id":"gary-marcus","normalized_proposition":"Scott Alexander's finding that InstructGPT got four of six previously failed prompts right may have been a fluke.","kind":"observation","conditions":[],"original_horizon_text":null,"statement":{"excerpt":"Scott’s more marked success on six questions may have been a fluke.","locator":"paragraph 57","url":"https://garymarcus.substack.com/p/what-does-it-mean-when-an-ai-fails","date":"2022-06-07","document_title":"What does it mean when an AI fails? A Reply to SlateStarCodex’s riff on Gary Marcus","observed_at":"2026-10-29T00:05:19Z"},"criteria":{"from_speaker":[],"our_interpretation":[],"support_condition":"Not checked against evidence: both screens found that public evidence could not settle this claim.","contradiction_condition":"Not checked against evidence: both screens found that public evidence could not settle this claim."},"axis":"not_applicable","outcome":"not_adjudicable","horizon_state":"not_applicable","evaluation_as_of":"2026-10-28","rationale":"Not checked against evidence. Two independent screens, one from each provider, both found this stated as a chance or a possibility, or at medium or low confidence.","limitations":[],"evidence":[],"what_would_resolve":null,"retrospective":true,"uncheckable":["probability_language"]},"payload_hash":"84f5ac9be97027d48b5fb18ec4c7d005fcc4aef99a834d2122d151c3759b07bd","policy_version":"2026-10-28.7","recorded_at":"2026-10-29T01:56:19Z","drafter":{"provider":"anthropic","model":"claude-opus-5-5"},"reviewers":[{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-what-does-it-mean-when-an-ai-fails-screen-anthropic-1"},{"provider":"openai","model":"gpt-6-astra","run_id":"marcus-what-does-it-mean-when-an-ai-fails-screen-openai-1"}],"final_text":null,"status":"current","notice":null,"superseded_by":null,"evidence_meta":[],"public_from":null,"public_until":null},"history":[],"review":{"review_due":false,"triggers":[],"last_completed_review_at":"2026-10-29T01:56:19Z","next_review_due_at":null,"horizon_now":"not_applicable"},"event_ids":["marcus-what-does-it-mean-when-an-ai-fails-112099-assessment-published-a1"],"last_substantive_change_at":"2026-10-29T17:00:18Z"}},{"url":"https://punditclaims.ai/claims/marcus-what-does-it-mean-when-an-ai-fails-739569","standing":"current","assessment_label":"Can’t be checked","versions":[{"assessment_id":"marcus-what-does-it-mean-when-an-ai-fails-739569-a1","status":"current","url":"https://punditclaims.ai/claims/marcus-what-does-it-mean-when-an-ai-fails-739569#assessment-marcus-what-does-it-mean-when-an-ai-fails-739569-a1"}],"claim":{"claim_id":"marcus-what-does-it-mean-when-an-ai-fails-739569","subject_id":"gary-marcus","revision":1,"content_hash":"fb17014e8aaccb4241c0c0765c0f48b40e09a72c0110fae3e93437bbcf22a833","normalized_proposition":"Because LLMs lack causal world models, they are not stable or trustworthy.","kind":"causal","target":"large language models","conditions":[],"original_horizon_text":null,"topic_tags":["reliability"],"concerns_providers":[],"selection_stratum":"declared-lead","occurrences":[{"id":"marcus-what-does-it-mean-when-an-ai-fails-739569-o1","statement_date":"2022-06-07","statement_date_precision":"day","excerpt":"The core problem that follows from that has always been that these systems aren’t stable or trustworthy; a crumbling house follows from a rotten foundation.","locator":"paragraph 50","document":{"url":"https://garymarcus.substack.com/p/what-does-it-mean-when-an-ai-fails","title":"What does it mean when an AI fails? A Reply to SlateStarCodex’s riff on Gary Marcus","publisher":"Marcus on AI","published_at":"2022-06-07T17:28:09Z","date_uncertain":false,"date_provenance":"JSON-LD datePublished"},"observed_at":"2026-10-29T00:05:19Z","observed_via":"live","archive_url":null,"access_state":"full_public","snapshot_hash":"fa2a39c0800cc5475acd393215c920495373f319a6dd41cc57a65d2d3d491dbc","receipts":1}],"first_statement_date":"2022-06-07","related":[],"current":{"assessment_id":"marcus-what-does-it-mean-when-an-ai-fails-739569-a1","payload":{"claim_id":"marcus-what-does-it-mean-when-an-ai-fails-739569","subject_id":"gary-marcus","normalized_proposition":"Because LLMs lack causal world models, they are not stable or trustworthy.","kind":"causal","conditions":[],"original_horizon_text":null,"statement":{"excerpt":"The core problem that follows from that has always been that these systems aren’t stable or trustworthy; a crumbling house follows from a rotten foundation.","locator":"paragraph 50","url":"https://garymarcus.substack.com/p/what-does-it-mean-when-an-ai-fails","date":"2022-06-07","document_title":"What does it mean when an AI fails? A Reply to SlateStarCodex’s riff on Gary Marcus","observed_at":"2026-10-29T00:05:19Z"},"criteria":{"from_speaker":[],"our_interpretation":[],"support_condition":"Not checked against evidence: both screens found that public evidence could not settle this claim.","contradiction_condition":"Not checked against evidence: both screens found that public evidence could not settle this claim."},"axis":"not_applicable","outcome":"not_adjudicable","horizon_state":"not_applicable","evaluation_as_of":"2026-10-28","rationale":"Not checked against evidence. Two independent screens, one from each provider, both found this too vague to test: informed readers would test it in different ways and reach different answers.","limitations":[],"evidence":[],"what_would_resolve":null,"retrospective":true,"uncheckable":["no_plain_reading"]},"payload_hash":"0719bea0f9077f022d83c158ece6581ed82b2c61369fd4e9f400fc47922a8f44","policy_version":"2026-10-28.7","recorded_at":"2026-10-29T01:56:19Z","drafter":{"provider":"anthropic","model":"claude-opus-5-5"},"reviewers":[{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-what-does-it-mean-when-an-ai-fails-screen-anthropic-1"},{"provider":"openai","model":"gpt-6-astra","run_id":"marcus-what-does-it-mean-when-an-ai-fails-screen-openai-1"}],"final_text":null,"status":"current","notice":null,"superseded_by":null,"evidence_meta":[],"public_from":null,"public_until":null},"history":[],"review":{"review_due":false,"triggers":[],"last_completed_review_at":"2026-10-29T01:56:19Z","next_review_due_at":null,"horizon_now":"not_applicable"},"event_ids":["marcus-what-does-it-mean-when-an-ai-fails-739569-assessment-published-a1"],"last_substantive_change_at":"2026-10-29T17:00:18Z"}},{"url":"https://punditclaims.ai/claims/marcus-what-does-it-mean-when-an-ai-fails-7fce77","standing":"current","assessment_label":"Supported","versions":[{"assessment_id":"marcus-what-does-it-mean-when-an-ai-fails-7fce77-a1","status":"current","url":"https://punditclaims.ai/claims/marcus-what-does-it-mean-when-an-ai-fails-7fce77#assessment-marcus-what-does-it-mean-when-an-ai-fails-7fce77-a1"}],"claim":{"claim_id":"marcus-what-does-it-mean-when-an-ai-fails-7fce77","subject_id":"gary-marcus","revision":1,"content_hash":"2d6cf749369787329e4550c2cf4b9d13268ca0c7566b34bd1f4a3178aad9fa3d","normalized_proposition":"GPT-4 will very likely still have a fundamental lack of reliability, though it may take longer prompts to break.","kind":"forecast","target":"GPT-4","conditions":[],"original_horizon_text":null,"topic_tags":["reliability","scaling"],"concerns_providers":["openai"],"selection_stratum":"declared-lead","occurrences":[{"id":"marcus-what-does-it-mean-when-an-ai-fails-7fce77-o1","statement_date":"2022-06-07","statement_date_precision":"day","excerpt":"I would be very surprised if that fundamental lack of reliability did not also hold true for GPT-4. Maybe it will take three paragraphs in a prompt rather than one to break it, but a big Transformer alone does not a a theory of the world create.","locator":"paragraph 50","document":{"url":"https://garymarcus.substack.com/p/what-does-it-mean-when-an-ai-fails","title":"What does it mean when an AI fails? A Reply to SlateStarCodex’s riff on Gary Marcus","publisher":"Marcus on AI","published_at":"2022-06-07T17:28:09Z","date_uncertain":false,"date_provenance":"JSON-LD datePublished"},"observed_at":"2026-10-29T00:05:19Z","observed_via":"live","archive_url":null,"access_state":"full_public","snapshot_hash":"fa2a39c0800cc5475acd393215c920495373f319a6dd41cc57a65d2d3d491dbc","receipts":1}],"first_statement_date":"2022-06-07","related":[],"current":{"assessment_id":"marcus-what-does-it-mean-when-an-ai-fails-7fce77-a1","payload":{"claim_id":"marcus-what-does-it-mean-when-an-ai-fails-7fce77","subject_id":"gary-marcus","normalized_proposition":"GPT-4 will very likely still have a fundamental lack of reliability, though it may take longer prompts to break.","kind":"forecast","conditions":[],"original_horizon_text":null,"statement":{"excerpt":"I would be very surprised if that fundamental lack of reliability did not also hold true for GPT-4. Maybe it will take three paragraphs in a prompt rather than one to break it, but a big Transformer alone does not a a theory of the world create.","locator":"paragraph 50","url":"https://garymarcus.substack.com/p/what-does-it-mean-when-an-ai-fails","date":"2022-06-07","document_title":"What does it mean when an AI fails? A Reply to SlateStarCodex’s riff on Gary Marcus","observed_at":"2026-10-29T00:05:19Z"},"criteria":{"from_speaker":["I would be very surprised","fundamental lack of reliability","also hold true for GPT-4","Maybe it will take three paragraphs in a prompt rather than one"],"our_interpretation":["The ordinary reading is that GPT-4 would retain recurring factual and reasoning failures serious enough to prevent unqualified reliance on its answers.","The statement allows improvement over earlier models and does not predict a particular error rate.","The three-paragraph remark is illustrative, not a firm prompt-length threshold.","The target is the forthcoming GPT-4 generation. The prediction can be resolved by evidence about that model after release; it does not require waiting for every future revision.","This assessment addresses reliability, not the separate philosophical assertion about whether Transformers create theories of the world."],"support_condition":"Dated evidence about released GPT-4 documents recurring hallucinations and reasoning errors that materially limit reliable use.","contradiction_condition":"Broad, reproducible testing establishes robust factual and reasoning reliability across the relevant tasks, including adversarial variations, such that the predicted recurring limitations are absent."},"axis":"forecast_outcome","outcome":"supported","horizon_state":"conclusive_event","evaluation_as_of":"2026-10-28","rationale":"OpenAI's launch-era technical report explicitly states that GPT-4 still hallucinates facts and makes reasoning errors. These are the recurring reliability problems the forecast anticipated. Substantial improvements and impressive capabilities do not contradict this forecast, which expressly allowed fewer failures. The evidence does not settle whether GPT-4 possesses a causal world model.","limitations":["Support applies to recurring factual and reasoning unreliability, not a claim that GPT-4 has no useful capabilities.","No numerical reliability threshold or causal explanation is established.","The decisive disclosure is from the developer."],"evidence":[{"id":"e1","relation":"supports","excerpt":"makes reasoning errors","locator":"Section 5, Limitations","url":"https://arxiv.org/html/2303.08774v1","publisher":"OpenAI / arXiv","event_date":"2023-03-15","available_at":"2023-03-15","explanation":"OpenAI's launch-era report explicitly acknowledges both predicted failure classes and advises safeguards where reliability matters. It also reports substantial factuality improvement, which is compatible with Marcus's allowance for fewer errors."},{"id":"e2","relation":"context","excerpt":"a true paradigm shift","locator":"Section 1, Introduction","url":"https://arxiv.org/html/2303.12712v1","publisher":"Microsoft Research researchers / arXiv","event_date":"2023-03-22","available_at":"2023-03-22","explanation":"The researchers interpret their experiments as substantial progress, cautioning against equating unreliability with an absence of capability. Their study concerns an early development model and also acknowledges limitations, so it does not contradict recurring failures in released GPT-4."}],"what_would_resolve":null,"retrospective":true,"plain_reading":true},"payload_hash":"07438d10d5bcbca0310685fdde2455f5562d8f35be0421508c52bd5f2ae490e7","policy_version":"2026-10-28.7","recorded_at":"2026-10-29T12:47:24Z","drafter":{"provider":"openai","model":"gpt-6-astra"},"reviewers":[{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-what-does-it-mean-when-an-ai-fails-204104-4x-review-anthropic-1"},{"provider":"openai","model":"gpt-6-astra","run_id":"marcus-what-does-it-mean-when-an-ai-fails-204104-4x-review-openai-1"}],"final_text":{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-what-does-it-mean-when-an-ai-fails-204104-4x-final-anthropic-1"},"status":"current","notice":null,"superseded_by":null,"evidence_meta":[{"id":"e1","source_type":"company_disclosure","audited":null,"derived":null,"metric":null},{"id":"e2","source_type":"preprint","audited":null,"derived":null,"metric":null}],"public_from":null,"public_until":null},"history":[],"review":{"review_due":false,"triggers":[],"last_completed_review_at":"2026-10-29T12:47:24Z","next_review_due_at":null,"horizon_now":"conclusive_event"},"event_ids":["marcus-what-does-it-mean-when-an-ai-fails-7fce77-assessment-published-a1"],"last_substantive_change_at":"2026-10-29T17:00:18Z"}},{"url":"https://punditclaims.ai/claims/marcus-what-does-it-mean-when-an-ai-fails-9b99b3","standing":"current","assessment_label":"Can’t be checked","versions":[{"assessment_id":"marcus-what-does-it-mean-when-an-ai-fails-9b99b3-a1","status":"current","url":"https://punditclaims.ai/claims/marcus-what-does-it-mean-when-an-ai-fails-9b99b3#assessment-marcus-what-does-it-mean-when-an-ai-fails-9b99b3-a1"}],"claim":{"claim_id":"marcus-what-does-it-mean-when-an-ai-fails-9b99b3","subject_id":"gary-marcus","revision":2,"content_hash":"b6b584009c939bea3d0d67c8769777ce96415abc596faf406ee73595f173b6f2","normalized_proposition":"Marcus thinks it is scandalous that OpenAI never followed through on its promise of GPT-3 access to him and Davis and never replied to his request for DALL-E 2 access, and that Google Brain never replied to his request for Imagen access.","kind":"normative","target":"OpenAI and Google Brain","conditions":[{"text":"I think","type":"qualifier","from_source":true},{"text":"OpenAI's unkept GPT-3 access promise and unanswered DALL-E 2 request; Google Brain's unanswered Imagen request","type":"scope","from_source":true}],"original_horizon_text":null,"topic_tags":["other"],"concerns_providers":["openai"],"selection_stratum":"declared-lead","occurrences":[{"id":"marcus-what-does-it-mean-when-an-ai-fails-9b99b3-o1","statement_date":"2022-06-07","statement_date_precision":"day","excerpt":"I think all that’s scandalous.","locator":"paragraph 26","document":{"url":"https://garymarcus.substack.com/p/what-does-it-mean-when-an-ai-fails","title":"What does it mean when an AI fails? A Reply to SlateStarCodex’s riff on Gary Marcus","publisher":"Marcus on AI","published_at":"2022-06-07T17:28:09Z","date_uncertain":false,"date_provenance":"JSON-LD datePublished"},"observed_at":"2026-10-29T00:05:19Z","observed_via":"live","archive_url":null,"access_state":"full_public","snapshot_hash":"fa2a39c0800cc5475acd393215c920495373f319a6dd41cc57a65d2d3d491dbc","receipts":1}],"first_statement_date":"2022-06-07","related":[],"current":{"assessment_id":"marcus-what-does-it-mean-when-an-ai-fails-9b99b3-a1","payload":{"claim_id":"marcus-what-does-it-mean-when-an-ai-fails-9b99b3","subject_id":"gary-marcus","normalized_proposition":"Marcus thinks it is scandalous that OpenAI never followed through on its promise of GPT-3 access to him and Davis and never replied to his request for DALL-E 2 access, and that Google Brain never replied to his request for Imagen access.","kind":"normative","conditions":["I think","OpenAI's unkept GPT-3 access promise and unanswered DALL-E 2 request; Google Brain's unanswered Imagen request"],"original_horizon_text":null,"statement":{"excerpt":"I think all that’s scandalous.","locator":"paragraph 26","url":"https://garymarcus.substack.com/p/what-does-it-mean-when-an-ai-fails","date":"2022-06-07","document_title":"What does it mean when an AI fails? A Reply to SlateStarCodex’s riff on Gary Marcus","observed_at":"2026-10-29T00:05:19Z"},"criteria":{"from_speaker":["I think","OpenAI's unkept GPT-3 access promise and unanswered DALL-E 2 request; Google Brain's unanswered Imagen request"],"our_interpretation":[],"support_condition":"Not checked against evidence: both screens found that public evidence could not settle this claim.","contradiction_condition":"Not checked against evidence: both screens found that public evidence could not settle this claim."},"axis":"not_applicable","outcome":"not_adjudicable","horizon_state":"not_applicable","evaluation_as_of":"2026-10-28","rationale":"Not checked against evidence. Two independent screens, one from each provider, both found this a recommendation or a value judgment.","limitations":[],"evidence":[],"what_would_resolve":null,"retrospective":true,"uncheckable":["value_judgment"]},"payload_hash":"2518078a5b6bccae083d2bc9dcdaa1e4860a337adbf88813f439ba0fdc2cd004","policy_version":"2026-10-28.7","recorded_at":"2026-10-29T02:32:05Z","drafter":{"provider":"anthropic","model":"claude-opus-5-5"},"reviewers":[{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-what-does-it-mean-when-an-ai-fails-screen-anthropic-2"},{"provider":"openai","model":"gpt-6-astra","run_id":"marcus-what-does-it-mean-when-an-ai-fails-screen-openai-2"}],"final_text":null,"status":"current","notice":null,"superseded_by":null,"evidence_meta":[],"public_from":null,"public_until":null},"history":[],"review":{"review_due":false,"triggers":[],"last_completed_review_at":"2026-10-29T02:32:05Z","next_review_due_at":null,"horizon_now":"not_applicable"},"event_ids":["marcus-what-does-it-mean-when-an-ai-fails-9b99b3-assessment-published-a1"],"last_substantive_change_at":"2026-10-29T17:00:18Z"}},{"url":"https://punditclaims.ai/claims/marcus-what-does-it-mean-when-an-ai-fails-a78bb4","standing":"current","assessment_label":"Unresolved","versions":[{"assessment_id":"marcus-what-does-it-mean-when-an-ai-fails-a78bb4-a1","status":"current","url":"https://punditclaims.ai/claims/marcus-what-does-it-mean-when-an-ai-fails-a78bb4#assessment-marcus-what-does-it-mean-when-an-ai-fails-a78bb4-a1"}],"claim":{"claim_id":"marcus-what-does-it-mean-when-an-ai-fails-a78bb4","subject_id":"gary-marcus","revision":1,"content_hash":"90f2f56ee019a1c73fb9e655ff625c108de814c1b92c6c9be7c3a0bd6c4b6b39","normalized_proposition":"LLMs on their own, without symbolic hybridization, are inherently weak at understanding the physical and psychological world.","kind":"observation","target":"large language models","conditions":[{"text":"unless hybridized with symbolic systems","type":"qualifier","from_source":true}],"original_horizon_text":null,"topic_tags":["capabilities"],"concerns_providers":[],"selection_stratum":"declared-lead","occurrences":[{"id":"marcus-what-does-it-mean-when-an-ai-fails-a78bb4-o1","statement_date":"2022-06-07","statement_date_precision":"day","excerpt":"Large language models are (unless hybridized with symbolic systems, a la Jurassic X) are, on their own, inherently weak at understanding the physical and psychological world.","locator":"paragraph 51","document":{"url":"https://garymarcus.substack.com/p/what-does-it-mean-when-an-ai-fails","title":"What does it mean when an AI fails? A Reply to SlateStarCodex’s riff on Gary Marcus","publisher":"Marcus on AI","published_at":"2022-06-07T17:28:09Z","date_uncertain":false,"date_provenance":"JSON-LD datePublished"},"observed_at":"2026-10-29T00:05:19Z","observed_via":"live","archive_url":null,"access_state":"full_public","snapshot_hash":"fa2a39c0800cc5475acd393215c920495373f319a6dd41cc57a65d2d3d491dbc","receipts":1}],"first_statement_date":"2022-06-07","related":[],"current":{"assessment_id":"marcus-what-does-it-mean-when-an-ai-fails-a78bb4-a1","payload":{"claim_id":"marcus-what-does-it-mean-when-an-ai-fails-a78bb4","subject_id":"gary-marcus","normalized_proposition":"LLMs on their own, without symbolic hybridization, are inherently weak at understanding the physical and psychological world.","kind":"observation","conditions":["unless hybridized with symbolic systems"],"original_horizon_text":null,"statement":{"excerpt":"Large language models are (unless hybridized with symbolic systems, a la Jurassic X) are, on their own, inherently weak at understanding the physical and psychological world.","locator":"paragraph 51","url":"https://garymarcus.substack.com/p/what-does-it-mean-when-an-ai-fails","date":"2022-06-07","document_title":"What does it mean when an AI fails? A Reply to SlateStarCodex’s riff on Gary Marcus","observed_at":"2026-10-29T00:05:19Z"},"criteria":{"from_speaker":["Large language models","unless hybridized with symbolic systems","on their own","inherently weak","understanding the physical and psychological world"],"our_interpretation":["The ordinary behavioral reading is a systematic weakness in reasoning about physical situations and people's beliefs, intentions and reactions, rather than a claim that models answer every such question incorrectly.","Inherently attributes the weakness to the unaided model approach; isolated mistakes are insufficient to establish that stronger component.","Symbolic solvers and external reasoning systems are excluded. Natural-language prompting alone does not count as symbolic hybridization.","The observation concerns models available by the post. Its contextual reference to continuing general phenomena supplies the August 2020-to-June 2022 historical interval.","Later models cannot directly decide this historical observation."],"support_condition":"Varied evaluations of contemporary unaided LLMs show systematic weaknesses in both physical and psychological reasoning, with evidence that those weaknesses belong to the approach rather than a particular benchmark or configuration.","contradiction_condition":"Contemporary unaided LLMs demonstrate robust, generalizable competence across varied physical and psychological reasoning tasks, including novel cases and controlled variations."},"axis":"historical_observation","outcome":"unresolved","horizon_state":"not_applicable","evaluation_as_of":"2026-10-28","rationale":"The evidence gives reasons for concern about physical reasoning and reasons to recognize substantial commonsense competence. Neither establishes the full claim about inherent weakness across both physical and psychological understanding in contemporary unaided LLMs. Calling it supported would generalize beyond the tested models and domains; calling it contradicted would treat benchmark success as broad robustness.","limitations":["TRIP tests a narrower model population than the frontier generative systems discussed in the post.","PaLM's favorable results do not by themselves demonstrate robust understanding.","The word inherently makes the claim stronger than an observation of particular errors."],"evidence":[{"id":"e1","relation":"context","excerpt":"supporting evidence.","locator":"Abstract; experimental population checked in full-paper Section 4","url":"https://arxiv.org/abs/2109.04947","publisher":"Shane Storks, Qiaozi Gao, Yichi Zhang and Joyce Chai / arXiv","event_date":"2021","available_at":"2022-05-10","explanation":"TRIP documents difficulties connecting physical story judgments to supporting states and conflicts. Its BERT-family systems and physical-only coverage do not establish the complete claim about contemporary leading LLMs and both domains."},{"id":"e2","relation":"context","excerpt":"reasoning tasks.","locator":"Conclusion; reasoning methods in Section 6.3","url":"https://arxiv.org/html/2204.02311v3","publisher":"Google researchers / arXiv","event_date":"2022","available_at":"2022-04-19","explanation":"PaLM's commonsense results show meaningful competence using natural-language prompting, limiting a blanket reading of weakness. The combined statement also covers calculator-assisted arithmetic, and does not establish robust competence across both physical and psychological reasoning."}],"what_would_resolve":"Broad, controlled evaluations of the contemporaneous models across both domains, with novel task variations and documented absence of symbolic assistance, together with evidence distinguishing systematic model limitations from benchmark-specific failures.","retrospective":true,"plain_reading":true},"payload_hash":"19df496e443b745a55dfb43313c55a44dedc42dd8b6eb4193e70e9d0926891ad","policy_version":"2026-10-28.7","recorded_at":"2026-10-29T12:47:24Z","drafter":{"provider":"openai","model":"gpt-6-astra"},"reviewers":[{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-what-does-it-mean-when-an-ai-fails-204104-4x-review-anthropic-1"},{"provider":"openai","model":"gpt-6-astra","run_id":"marcus-what-does-it-mean-when-an-ai-fails-204104-4x-review-openai-1"}],"final_text":{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-what-does-it-mean-when-an-ai-fails-204104-4x-final-anthropic-1"},"status":"current","notice":null,"superseded_by":null,"evidence_meta":[{"id":"e1","source_type":"peer_reviewed","audited":null,"derived":null,"metric":null},{"id":"e2","source_type":"preprint","audited":null,"derived":null,"metric":null}],"public_from":null,"public_until":null},"history":[],"review":{"review_due":false,"triggers":[],"last_completed_review_at":"2026-10-29T12:47:24Z","next_review_due_at":null,"horizon_now":"not_applicable"},"event_ids":["marcus-what-does-it-mean-when-an-ai-fails-a78bb4-assessment-published-a1"],"last_substantive_change_at":"2026-10-29T17:00:18Z"}},{"url":"https://punditclaims.ai/claims/marcus-what-does-it-mean-when-an-ai-fails-ae0bd9","standing":"current","assessment_label":"Can’t be checked","versions":[{"assessment_id":"marcus-what-does-it-mean-when-an-ai-fails-ae0bd9-a1","status":"current","url":"https://punditclaims.ai/claims/marcus-what-does-it-mean-when-an-ai-fails-ae0bd9#assessment-marcus-what-does-it-mean-when-an-ai-fails-ae0bd9-a1"}],"claim":{"claim_id":"marcus-what-does-it-mean-when-an-ai-fails-ae0bd9","subject_id":"gary-marcus","revision":2,"content_hash":"08ae0365cccecb5fe9e3c7186d12e21e4d1efe7763778cef27c0a282399a1e25","normalized_proposition":"Marcus sees no reason whatsoever to think that the underlying problem of these models, a lack of cognitive models of the world, has been remedied.","kind":"observation","target":"large language models","conditions":[{"text":"I see no reason whatsoever to think","type":"qualifier","from_source":true}],"original_horizon_text":null,"topic_tags":["capabilities"],"concerns_providers":[],"selection_stratum":"declared-lead","occurrences":[{"id":"marcus-what-does-it-mean-when-an-ai-fails-ae0bd9-o1","statement_date":"2022-06-07","statement_date_precision":"day","excerpt":"But I see no reason whatsoever to think that the underlying problem — a lack of cognitive models of the world —have been remedied.","locator":"paragraph 38","document":{"url":"https://garymarcus.substack.com/p/what-does-it-mean-when-an-ai-fails","title":"What does it mean when an AI fails? A Reply to SlateStarCodex’s riff on Gary Marcus","publisher":"Marcus on AI","published_at":"2022-06-07T17:28:09Z","date_uncertain":false,"date_provenance":"JSON-LD datePublished"},"observed_at":"2026-10-29T00:05:19Z","observed_via":"live","archive_url":null,"access_state":"full_public","snapshot_hash":"fa2a39c0800cc5475acd393215c920495373f319a6dd41cc57a65d2d3d491dbc","receipts":1}],"first_statement_date":"2022-06-07","related":[],"current":{"assessment_id":"marcus-what-does-it-mean-when-an-ai-fails-ae0bd9-a1","payload":{"claim_id":"marcus-what-does-it-mean-when-an-ai-fails-ae0bd9","subject_id":"gary-marcus","normalized_proposition":"Marcus sees no reason whatsoever to think that the underlying problem of these models, a lack of cognitive models of the world, has been remedied.","kind":"observation","conditions":["I see no reason whatsoever to think"],"original_horizon_text":null,"statement":{"excerpt":"But I see no reason whatsoever to think that the underlying problem — a lack of cognitive models of the world —have been remedied.","locator":"paragraph 38","url":"https://garymarcus.substack.com/p/what-does-it-mean-when-an-ai-fails","date":"2022-06-07","document_title":"What does it mean when an AI fails? A Reply to SlateStarCodex’s riff on Gary Marcus","observed_at":"2026-10-29T00:05:19Z"},"criteria":{"from_speaker":["I see no reason whatsoever to think"],"our_interpretation":[],"support_condition":"Not checked against evidence: both screens found that public evidence could not settle this claim.","contradiction_condition":"Not checked against evidence: both screens found that public evidence could not settle this claim."},"axis":"not_applicable","outcome":"not_adjudicable","horizon_state":"not_applicable","evaluation_as_of":"2026-10-28","rationale":"Not checked against evidence. Two independent screens, one from each provider, both found this too vague to test: informed readers would test it in different ways and reach different answers.","limitations":[],"evidence":[],"what_would_resolve":null,"retrospective":true,"uncheckable":["no_plain_reading"]},"payload_hash":"b0c165e4c5de4d3a4b1f547fe9c1c3cb278756afeb07b3a3d680396f1c1b1283","policy_version":"2026-10-28.7","recorded_at":"2026-10-29T02:32:05Z","drafter":{"provider":"anthropic","model":"claude-opus-5-5"},"reviewers":[{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-what-does-it-mean-when-an-ai-fails-screen-anthropic-2"},{"provider":"openai","model":"gpt-6-astra","run_id":"marcus-what-does-it-mean-when-an-ai-fails-screen-openai-2"}],"final_text":null,"status":"current","notice":null,"superseded_by":null,"evidence_meta":[],"public_from":null,"public_until":null},"history":[],"review":{"review_due":false,"triggers":[],"last_completed_review_at":"2026-10-29T02:32:05Z","next_review_due_at":null,"horizon_now":"not_applicable"},"event_ids":["marcus-what-does-it-mean-when-an-ai-fails-ae0bd9-assessment-published-a1"],"last_substantive_change_at":"2026-10-29T17:00:18Z"}},{"url":"https://punditclaims.ai/claims/marcus-what-does-it-mean-when-an-ai-fails-b49d13","standing":"current","assessment_label":"Can’t be checked","versions":[{"assessment_id":"marcus-what-does-it-mean-when-an-ai-fails-b49d13-a1","status":"current","url":"https://punditclaims.ai/claims/marcus-what-does-it-mean-when-an-ai-fails-b49d13#assessment-marcus-what-does-it-mean-when-an-ai-fails-b49d13-a1"}],"claim":{"claim_id":"marcus-what-does-it-mean-when-an-ai-fails-b49d13","subject_id":"gary-marcus","revision":1,"content_hash":"995cc2aca48e697a4206dc61226bc61a0f418b60eabfc9cf148caa7a23dfde6c","normalized_proposition":"Current LLMs have no real physical, social, or psychological understanding; they are richer parroting of larger corpora.","kind":"observation","target":"large language models","conditions":[],"original_horizon_text":null,"topic_tags":["capabilities"],"concerns_providers":["openai"],"selection_stratum":"declared-lead","occurrences":[{"id":"marcus-what-does-it-mean-when-an-ai-fails-b49d13-o1","statement_date":"2022-06-07","statement_date_precision":"day","excerpt":"Either way, though, there is still no real physical, social, or psychological understanding, just richer parroting of larger corpora (and some reinforcement learning to keep the system more on check).","locator":"paragraph 47","document":{"url":"https://garymarcus.substack.com/p/what-does-it-mean-when-an-ai-fails","title":"What does it mean when an AI fails? A Reply to SlateStarCodex’s riff on Gary Marcus","publisher":"Marcus on AI","published_at":"2022-06-07T17:28:09Z","date_uncertain":false,"date_provenance":"JSON-LD datePublished"},"observed_at":"2026-10-29T00:05:19Z","observed_via":"live","archive_url":null,"access_state":"full_public","snapshot_hash":"fa2a39c0800cc5475acd393215c920495373f319a6dd41cc57a65d2d3d491dbc","receipts":1}],"first_statement_date":"2022-06-07","related":[],"current":{"assessment_id":"marcus-what-does-it-mean-when-an-ai-fails-b49d13-a1","payload":{"claim_id":"marcus-what-does-it-mean-when-an-ai-fails-b49d13","subject_id":"gary-marcus","normalized_proposition":"Current LLMs have no real physical, social, or psychological understanding; they are richer parroting of larger corpora.","kind":"observation","conditions":[],"original_horizon_text":null,"statement":{"excerpt":"Either way, though, there is still no real physical, social, or psychological understanding, just richer parroting of larger corpora (and some reinforcement learning to keep the system more on check).","locator":"paragraph 47","url":"https://garymarcus.substack.com/p/what-does-it-mean-when-an-ai-fails","date":"2022-06-07","document_title":"What does it mean when an AI fails? A Reply to SlateStarCodex’s riff on Gary Marcus","observed_at":"2026-10-29T00:05:19Z"},"criteria":{"from_speaker":[],"our_interpretation":[],"support_condition":"Not checked against evidence: both screens found that public evidence could not settle this claim.","contradiction_condition":"Not checked against evidence: both screens found that public evidence could not settle this claim."},"axis":"not_applicable","outcome":"not_adjudicable","horizon_state":"not_applicable","evaluation_as_of":"2026-10-28","rationale":"Not checked against evidence. Two independent screens, one from each provider, both found this too vague to test: informed readers would test it in different ways and reach different answers.","limitations":[],"evidence":[],"what_would_resolve":null,"retrospective":true,"uncheckable":["no_plain_reading"]},"payload_hash":"4391e07f5e01eb6965bf0d8c247ea8781c5e8d72f107ede8cb43bfc875e90b65","policy_version":"2026-10-28.7","recorded_at":"2026-10-29T01:56:19Z","drafter":{"provider":"anthropic","model":"claude-opus-5-5"},"reviewers":[{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-what-does-it-mean-when-an-ai-fails-screen-anthropic-1"},{"provider":"openai","model":"gpt-6-astra","run_id":"marcus-what-does-it-mean-when-an-ai-fails-screen-openai-1"}],"final_text":null,"status":"current","notice":null,"superseded_by":null,"evidence_meta":[],"public_from":null,"public_until":null},"history":[],"review":{"review_due":false,"triggers":[],"last_completed_review_at":"2026-10-29T01:56:19Z","next_review_due_at":null,"horizon_now":"not_applicable"},"event_ids":["marcus-what-does-it-mean-when-an-ai-fails-b49d13-assessment-published-a1"],"last_substantive_change_at":"2026-10-29T17:00:18Z"}},{"url":"https://punditclaims.ai/claims/marcus-what-does-it-mean-when-an-ai-fails-c64e20","standing":"current","assessment_label":"Can’t be checked","versions":[{"assessment_id":"marcus-what-does-it-mean-when-an-ai-fails-c64e20-a1","status":"current","url":"https://punditclaims.ai/claims/marcus-what-does-it-mean-when-an-ai-fails-c64e20#assessment-marcus-what-does-it-mean-when-an-ai-fails-c64e20-a1"}],"claim":{"claim_id":"marcus-what-does-it-mean-when-an-ai-fails-c64e20","subject_id":"gary-marcus","revision":2,"content_hash":"3fe950db3ffe5baa96c5bbab8a01f626c43297d9e12eae4c1226f10d432eecb3","normalized_proposition":"If GPT-4 is basically the same architecture as GPT-3 but with more data, it will not be proficient enough to have developed a genuine causal model of the world, though Marcus and Davis may find fewer absolute howlers per attempt than with GPT-3.","kind":"conditional_forecast","target":"GPT-4","conditions":[{"text":"assuming GPT-4 is basically the same architecture but with more data","type":"antecedent","from_source":true},{"text":"maybe fewer absolute howlers per attempt","type":"qualifier","from_source":true}],"original_horizon_text":null,"topic_tags":["capabilities","scaling"],"concerns_providers":["openai"],"selection_stratum":"declared-lead","occurrences":[{"id":"marcus-what-does-it-mean-when-an-ai-fails-c64e20-o1","statement_date":"2022-06-07","statement_date_precision":"day","excerpt":"Maybe we will find fewer absolute howlers per attempt than we did with GPT-3, but it still won’t be so proficient as to have developed a genuine causal model of the world.","locator":"paragraph 49","document":{"url":"https://garymarcus.substack.com/p/what-does-it-mean-when-an-ai-fails","title":"What does it mean when an AI fails? A Reply to SlateStarCodex’s riff on Gary Marcus","publisher":"Marcus on AI","published_at":"2022-06-07T17:28:09Z","date_uncertain":false,"date_provenance":"JSON-LD datePublished"},"observed_at":"2026-10-29T00:05:19Z","observed_via":"live","archive_url":null,"access_state":"full_public","snapshot_hash":"fa2a39c0800cc5475acd393215c920495373f319a6dd41cc57a65d2d3d491dbc","receipts":1}],"first_statement_date":"2022-06-07","related":[],"current":{"assessment_id":"marcus-what-does-it-mean-when-an-ai-fails-c64e20-a1","payload":{"claim_id":"marcus-what-does-it-mean-when-an-ai-fails-c64e20","subject_id":"gary-marcus","normalized_proposition":"If GPT-4 is basically the same architecture as GPT-3 but with more data, it will not be proficient enough to have developed a genuine causal model of the world, though Marcus and Davis may find fewer absolute howlers per attempt than with GPT-3.","kind":"conditional_forecast","conditions":["assuming GPT-4 is basically the same architecture but with more data","maybe fewer absolute howlers per attempt"],"original_horizon_text":null,"statement":{"excerpt":"Maybe we will find fewer absolute howlers per attempt than we did with GPT-3, but it still won’t be so proficient as to have developed a genuine causal model of the world.","locator":"paragraph 49","url":"https://garymarcus.substack.com/p/what-does-it-mean-when-an-ai-fails","date":"2022-06-07","document_title":"What does it mean when an AI fails? A Reply to SlateStarCodex’s riff on Gary Marcus","observed_at":"2026-10-29T00:05:19Z"},"criteria":{"from_speaker":["assuming GPT-4 is basically the same architecture but with more data","maybe fewer absolute howlers per attempt"],"our_interpretation":[],"support_condition":"Not checked against evidence: both screens found that public evidence could not settle this claim.","contradiction_condition":"Not checked against evidence: both screens found that public evidence could not settle this claim."},"axis":"not_applicable","outcome":"not_adjudicable","horizon_state":"not_applicable","evaluation_as_of":"2026-10-28","rationale":"Not checked against evidence. Two independent screens, one from each provider, both found this too vague to test: informed readers would test it in different ways and reach different answers.","limitations":[],"evidence":[],"what_would_resolve":null,"retrospective":true,"uncheckable":["no_plain_reading"]},"payload_hash":"c68c66b0f61dc647fd31e17ab5efe00165bd7bc2a896426107c1ae5eb2abae7f","policy_version":"2026-10-28.7","recorded_at":"2026-10-29T02:32:05Z","drafter":{"provider":"anthropic","model":"claude-opus-5-5"},"reviewers":[{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-what-does-it-mean-when-an-ai-fails-screen-anthropic-2"},{"provider":"openai","model":"gpt-6-astra","run_id":"marcus-what-does-it-mean-when-an-ai-fails-screen-openai-2"}],"final_text":null,"status":"current","notice":null,"superseded_by":null,"evidence_meta":[],"public_from":null,"public_until":null},"history":[],"review":{"review_due":false,"triggers":[],"last_completed_review_at":"2026-10-29T02:32:05Z","next_review_due_at":null,"horizon_now":"not_applicable"},"event_ids":["marcus-what-does-it-mean-when-an-ai-fails-c64e20-assessment-published-a1"],"last_substantive_change_at":"2026-10-29T17:00:18Z"}},{"url":"https://punditclaims.ai/claims/marcus-what-does-it-mean-when-an-ai-fails-cdba9a","standing":"current","assessment_label":"Unresolved","versions":[{"assessment_id":"marcus-what-does-it-mean-when-an-ai-fails-cdba9a-a1","status":"current","url":"https://punditclaims.ai/claims/marcus-what-does-it-mean-when-an-ai-fails-cdba9a#assessment-marcus-what-does-it-mean-when-an-ai-fails-cdba9a-a1"}],"claim":{"claim_id":"marcus-what-does-it-mean-when-an-ai-fails-cdba9a","subject_id":"gary-marcus","revision":2,"content_hash":"fa0fa6313322f49096628f8c70ecf45bd6a00902a5a1f654504951079621bb0d","normalized_proposition":"As Davis has found, across the full set of their Technology Review questions, the difference between GPT-3's earlier version and InstructGPT is far more modest than one might have expected: fewer outright wacky answers and some old answers now correct, but some backsliding too.","kind":"observation","target":"InstructGPT","conditions":[{"text":"across the full set of our Technology Review questions","type":"scope","from_source":true},{"text":"far more modest than one might have expected","type":"qualifier","from_source":true}],"original_horizon_text":null,"topic_tags":["capabilities","reliability"],"concerns_providers":["openai"],"selection_stratum":"declared-lead","occurrences":[{"id":"marcus-what-does-it-mean-when-an-ai-fails-cdba9a-o1","statement_date":"2022-06-07","statement_date_precision":"day","excerpt":"Interestingly, as Ernie has just discovered, across the full set of our Technology Review questions the difference from GPT-3’s earlier version to the current version (InstructGPT) is far more modest than one might have expected: fewer outright wacky answers, some old answers now correct, but some backsliding, too.","locator":"paragraph 57","document":{"url":"https://garymarcus.substack.com/p/what-does-it-mean-when-an-ai-fails","title":"What does it mean when an AI fails? A Reply to SlateStarCodex’s riff on Gary Marcus","publisher":"Marcus on AI","published_at":"2022-06-07T17:28:09Z","date_uncertain":false,"date_provenance":"JSON-LD datePublished"},"observed_at":"2026-10-29T00:05:19Z","observed_via":"live","archive_url":null,"access_state":"full_public","snapshot_hash":"fa2a39c0800cc5475acd393215c920495373f319a6dd41cc57a65d2d3d491dbc","receipts":1}],"first_statement_date":"2022-06-07","related":[],"current":{"assessment_id":"marcus-what-does-it-mean-when-an-ai-fails-cdba9a-a1","payload":{"claim_id":"marcus-what-does-it-mean-when-an-ai-fails-cdba9a","subject_id":"gary-marcus","normalized_proposition":"As Davis has found, across the full set of their Technology Review questions, the difference between GPT-3's earlier version and InstructGPT is far more modest than one might have expected: fewer outright wacky answers and some old answers now correct, but some backsliding too.","kind":"observation","conditions":["across the full set of our Technology Review questions","far more modest than one might have expected"],"original_horizon_text":null,"statement":{"excerpt":"Interestingly, as Ernie has just discovered, across the full set of our Technology Review questions the difference from GPT-3’s earlier version to the current version (InstructGPT) is far more modest than one might have expected: fewer outright wacky answers, some old answers now correct, but some backsliding, too.","locator":"paragraph 57","url":"https://garymarcus.substack.com/p/what-does-it-mean-when-an-ai-fails","date":"2022-06-07","document_title":"What does it mean when an AI fails? A Reply to SlateStarCodex’s riff on Gary Marcus","observed_at":"2026-10-29T00:05:19Z"},"criteria":{"from_speaker":["across the full set of our Technology Review questions","GPT-3’s earlier version to the current version (InstructGPT)","far more modest than one might have expected","fewer outright wacky answers","some old answers now correct","some backsliding, too"],"our_interpretation":["The plain reading is a limited overall improvement on the complete question set, with fewer plainly nonsensical responses, some corrected failures, and some newly incorrect answers.","The surrounding discussion supplies Alexander’s six-question comparison as the reference for expectations. It does not supply a numerical threshold for modest improvement.","The target is the comparison newly reported around the post date. I use June 1–7, 2022 for this early-June observation; the earlier GPT-3 answers serve as its historical baseline.","Backsliding means an earlier correct response becoming incorrect on the same question. It does not by itself establish a decline in the model’s average capability."],"support_condition":"An admissible comparison covering the complete question set shows a limited aggregate improvement, fewer plainly nonsensical answers, at least one corrected earlier failure, and at least one earlier success becoming incorrect.","contradiction_condition":"An admissible complete-set comparison shows a clearly large aggregate improvement under consistent scoring, or establishes that one of the claimed directional changes did not occur."},"axis":"historical_observation","outcome":"unresolved","horizon_state":"not_applicable","evaluation_as_of":"2026-10-28","rationale":"The search located the directly relevant full-set report, but Marcus coauthored it. The supplied rules prohibit using his own writing to establish that he was right. Alexander’s independent test reports four corrected answers among six earlier failures, but that selection cannot reveal backsliding or establish the overall change across the complete set. The admissible evidence therefore does not settle the claim. This is an evidence limitation, not evidence that Marcus was wrong.","limitations":["The source restriction excludes the report most directly associated with the claim.","A small set selected from earlier failures cannot determine complete-set performance.","Modest improvement and wacky answers lack numerical definitions."],"evidence":[{"id":"e1","relation":"context","excerpt":"Of these six prompts that GPT-3 original failed, GPT-3 advanced gets four unambiguously right.","locator":"Section Marcus vs. GPT, Round 2, paragraph summarizing the six retested prompts","url":"https://www.astralcodexten.com/p/my-bet-ai-size-solves-flubs","publisher":"Scott Alexander, Astral Codex Ten","event_date":"2022-06","available_at":"2022-06-07","explanation":"Alexander reports corrections on a subset of earlier failures. This cannot establish the complete-set improvement or reveal regressions on earlier successes, so it neither settles nor contradicts Marcus’s full claim."}],"what_would_resolve":"An independent audit of authenticated outputs from the original and early-June 2022 tests, covering the complete question set and applying consistent scoring, with matched counts of corrections, regressions, unchanged failures, and nonsensical responses.","retrospective":true,"plain_reading":true},"payload_hash":"ab9a71cd06798f2cbe99b054a6ab2e7175aeb1a097da0cf7b7d431823399ebb8","policy_version":"2026-10-28.7","recorded_at":"2026-10-29T12:47:24Z","drafter":{"provider":"openai","model":"gpt-6-astra"},"reviewers":[{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-what-does-it-mean-when-an-ai-fails-204104-4x-review-anthropic-1"},{"provider":"openai","model":"gpt-6-astra","run_id":"marcus-what-does-it-mean-when-an-ai-fails-204104-4x-review-openai-1"}],"final_text":{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-what-does-it-mean-when-an-ai-fails-204104-4x-final-anthropic-1"},"status":"current","notice":null,"superseded_by":null,"evidence_meta":[{"id":"e1","source_type":"independent_measurement","audited":null,"derived":null,"metric":null}],"public_from":null,"public_until":null},"history":[],"review":{"review_due":false,"triggers":[],"last_completed_review_at":"2026-10-29T12:47:24Z","next_review_due_at":null,"horizon_now":"not_applicable"},"event_ids":["marcus-what-does-it-mean-when-an-ai-fails-cdba9a-assessment-published-a1"],"last_substantive_change_at":"2026-10-29T17:00:18Z"}},{"url":"https://punditclaims.ai/claims/marcus-what-does-it-mean-when-an-ai-fails-d02c78","standing":"current","assessment_label":"Unresolved","versions":[{"assessment_id":"marcus-what-does-it-mean-when-an-ai-fails-d02c78-a1","status":"current","url":"https://punditclaims.ai/claims/marcus-what-does-it-mean-when-an-ai-fails-d02c78#assessment-marcus-what-does-it-mean-when-an-ai-fails-d02c78-a1"}],"claim":{"claim_id":"marcus-what-does-it-mean-when-an-ai-fails-d02c78","subject_id":"gary-marcus","revision":2,"content_hash":"8f45544e6875f237e7a66ed32a4eeaeed71c751cbca15bb45804bc3d8e0c38c2","normalized_proposition":"So far as Marcus can tell, all the core claims he and Davis made in Technology Review still hold true nearly two years and many even larger models later; some of their specific examples have been remedied, but the general problems have not.","kind":"observation","target":"large language models","conditions":[{"text":"so far as I can tell","type":"qualifier","from_source":true},{"text":"nearly two years and many even larger models later","type":"scope","from_source":true}],"original_horizon_text":null,"topic_tags":["reliability","capabilities","scaling"],"concerns_providers":["openai"],"selection_stratum":"declared-lead","occurrences":[{"id":"marcus-what-does-it-mean-when-an-ai-fails-d02c78-o1","statement_date":"2022-06-07","statement_date_precision":"day","excerpt":"The core claims that Ernie and I made in Technology Review still all, so far as I can tell, hold true, nearly two years and many even larger models later. Yes, some of the specific examples we pointed out have been remedied, but the general problems have not.","locator":"paragraph 29","document":{"url":"https://garymarcus.substack.com/p/what-does-it-mean-when-an-ai-fails","title":"What does it mean when an AI fails? A Reply to SlateStarCodex’s riff on Gary Marcus","publisher":"Marcus on AI","published_at":"2022-06-07T17:28:09Z","date_uncertain":false,"date_provenance":"JSON-LD datePublished"},"observed_at":"2026-10-29T00:05:19Z","observed_via":"live","archive_url":null,"access_state":"full_public","snapshot_hash":"fa2a39c0800cc5475acd393215c920495373f319a6dd41cc57a65d2d3d491dbc","receipts":1}],"first_statement_date":"2022-06-07","related":[],"current":{"assessment_id":"marcus-what-does-it-mean-when-an-ai-fails-d02c78-a1","payload":{"claim_id":"marcus-what-does-it-mean-when-an-ai-fails-d02c78","subject_id":"gary-marcus","normalized_proposition":"So far as Marcus can tell, all the core claims he and Davis made in Technology Review still hold true nearly two years and many even larger models later; some of their specific examples have been remedied, but the general problems have not.","kind":"observation","conditions":["so far as I can tell","nearly two years and many even larger models later"],"original_horizon_text":null,"statement":{"excerpt":"The core claims that Ernie and I made in Technology Review still all, so far as I can tell, hold true, nearly two years and many even larger models later. Yes, some of the specific examples we pointed out have been remedied, but the general problems have not.","locator":"paragraph 29","url":"https://garymarcus.substack.com/p/what-does-it-mean-when-an-ai-fails","date":"2022-06-07","document_title":"What does it mean when an AI fails? A Reply to SlateStarCodex’s riff on Gary Marcus","observed_at":"2026-10-29T00:05:19Z"},"criteria":{"from_speaker":["core claims","still all","so far as I can tell","nearly two years","some of the specific examples ... have been remedied","the general problems have not"],"our_interpretation":["The context identifies six general problems: biological, physical, social and practical reasoning, object and individual tracking, and non sequiturs.","The ordinary reading is that these remained substantive weaknesses, despite improvements on particular examples. It does not require the models to fail every test.","Because this describes the current model generation without a reporting period, I use 2022 through the post date as the latest public evaluation period.","The qualification about Marcus's knowledge does not remove the factual assertion. However, evidence about only some categories cannot establish his assertion about all six."],"support_condition":"Public evaluations of the relevant model generation establish continuing substantive failures across all six categories, while allowing improvements on individual examples.","contradiction_condition":"Robust evaluations establish that at least one of the six general problems had been resolved across varied tests by the post date. Better average scores or corrected examples alone do not suffice."},"axis":"historical_observation","outcome":"unresolved","horizon_state":"not_applicable","evaluation_as_of":"2026-10-28","rationale":"The records support continuing mistakes and also show substantial progress. They do not establish Marcus's stronger assertion that all six general problems remained unresolved. Nor do the improved benchmark results establish that any whole category had been solved. The breadth of the claim remains unverified.","limitations":["This is a gap in the retrieved evidence, not evidence that the categories had been solved.","Benchmark averages do not directly measure the six categories as Marcus defined them.","The dated disclosures identify public capability snapshots, not the exact dates each experiment ran."],"evidence":[{"id":"e1","relation":"supports","excerpt":"InstructGPT still makes simple mistakes.","locator":"Introduction, summary of limitations; section 4, qualitative results","url":"https://arxiv.org/html/2203.02155v1","publisher":"OpenAI researchers / arXiv","event_date":"2022-03-04","available_at":"2022-03-04","explanation":"The reported model still accepted false premises, invented facts and struggled with multiple constraints. This supports persistent reliability problems, but does not establish failures across all six categories."},{"id":"e2","relation":"context","excerpt":"average performance","locator":"Language Understanding and Generation, BIG-bench comparison","url":"https://research.google/blog/pathways-language-model-palm-scaling-to-540-billion-parameters-for-breakthrough-performance","publisher":"Google Research","event_date":"2022-04-04","available_at":"2022-04-04","explanation":"Google reported strong average performance on a subset of 58 BIG-bench tasks. This is material evidence of progress, but the aggregate does not establish that any entire Marcus category had been resolved."}],"what_would_resolve":"Independent tests of the models available by June 7, 2022, covering all six categories with varied examples, or robust evidence that one complete category had been resolved.","retrospective":true,"plain_reading":true},"payload_hash":"7daa73a907a99f3714a401fc9a88a175c00313d753d0b5fac985b1e86c9ed799","policy_version":"2026-10-28.7","recorded_at":"2026-10-29T12:47:03Z","drafter":{"provider":"openai","model":"gpt-6-astra"},"reviewers":[{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-what-does-it-mean-when-an-ai-fails-d02c78-4x-review-anthropic-1"},{"provider":"openai","model":"gpt-6-astra","run_id":"marcus-what-does-it-mean-when-an-ai-fails-d02c78-4x-review-openai-1"}],"final_text":{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-what-does-it-mean-when-an-ai-fails-d02c78-4x-final-anthropic-1"},"status":"current","notice":null,"superseded_by":null,"evidence_meta":[{"id":"e1","source_type":"preprint","audited":null,"derived":null,"metric":null},{"id":"e2","source_type":"company_disclosure","audited":null,"derived":null,"metric":null}],"public_from":null,"public_until":null},"history":[],"review":{"review_due":false,"triggers":[],"last_completed_review_at":"2026-10-29T12:47:03Z","next_review_due_at":null,"horizon_now":"not_applicable"},"event_ids":["marcus-what-does-it-mean-when-an-ai-fails-d02c78-assessment-published-a1"],"last_substantive_change_at":"2026-10-29T17:00:18Z"}},{"url":"https://punditclaims.ai/claims/marcus-what-does-it-mean-when-an-ai-fails-d5182b","standing":"current","assessment_label":"Unresolved","versions":[{"assessment_id":"marcus-what-does-it-mean-when-an-ai-fails-d5182b-a2","status":"current","url":"https://punditclaims.ai/claims/marcus-what-does-it-mean-when-an-ai-fails-d5182b#assessment-marcus-what-does-it-mean-when-an-ai-fails-d5182b-a2"}],"claim":{"claim_id":"marcus-what-does-it-mean-when-an-ai-fails-d5182b","subject_id":"gary-marcus","revision":1,"content_hash":"b9dfdced78e090cdd40a5e8e7de0ef73a053f60e58e3ef6dee26ad333124f65c","normalized_proposition":"Newer LLMs still lack genuine comprehension and are therefore still very easy for anyone to break.","kind":"causal","target":"large language models","conditions":[],"original_horizon_text":null,"topic_tags":["reliability","capabilities"],"concerns_providers":[],"selection_stratum":"declared-lead","occurrences":[{"id":"marcus-what-does-it-mean-when-an-ai-fails-d5182b-o1","statement_date":"2022-06-07","statement_date_precision":"day","excerpt":"But they still don’t convey genuine comprehension, and so they are still very easy for Ernie and me (or anyone else who cares to try) to break.","locator":"paragraph 38","document":{"url":"https://garymarcus.substack.com/p/what-does-it-mean-when-an-ai-fails","title":"What does it mean when an AI fails? A Reply to SlateStarCodex’s riff on Gary Marcus","publisher":"Marcus on AI","published_at":"2022-06-07T17:28:09Z","date_uncertain":false,"date_provenance":"JSON-LD datePublished"},"observed_at":"2026-10-29T00:05:19Z","observed_via":"live","archive_url":null,"access_state":"full_public","snapshot_hash":"fa2a39c0800cc5475acd393215c920495373f319a6dd41cc57a65d2d3d491dbc","receipts":1}],"first_statement_date":"2022-06-07","related":[],"current":{"assessment_id":"marcus-what-does-it-mean-when-an-ai-fails-d5182b-a2","payload":{"claim_id":"marcus-what-does-it-mean-when-an-ai-fails-d5182b","subject_id":"gary-marcus","normalized_proposition":"Newer LLMs still lack genuine comprehension and are therefore still very easy for anyone to break.","kind":"causal","conditions":[],"original_horizon_text":null,"statement":{"excerpt":"But they still don’t convey genuine comprehension, and so they are still very easy for Ernie and me (or anyone else who cares to try) to break.","locator":"paragraph 38","url":"https://garymarcus.substack.com/p/what-does-it-mean-when-an-ai-fails","date":"2022-06-07","document_title":"What does it mean when an AI fails? A Reply to SlateStarCodex’s riff on Gary Marcus","observed_at":"2026-10-29T00:05:19Z"},"criteria":{"from_speaker":["still don’t convey genuine comprehension","and so","very easy","anyone else who cares to try","to break"],"our_interpretation":["The ordinary behavioral reading is that interested users could readily elicit clear factual or commonsense errors from the contemporary models.","The words 'and so' additionally attribute those failures to missing comprehension. Demonstrating errors alone does not establish that explanation.","The historical scope is the continuation of the problems identified on August 22, 2020 through the post date, including the successors discussed in the post.","No numerical failure rate, testing budget, or independent measure of comprehension is supplied."],"support_condition":"Independent tests show readily reproducible failures in contemporary models, and evidence distinguishes absent comprehension as their cause from alternative explanations.","contradiction_condition":"Representative tests show robust performance against ordinary attempts to elicit these failures, or controlled evidence undermines the proposed explanation that missing comprehension causes them."},"axis":"causal_assessment","outcome":"unresolved","horizon_state":"not_applicable","evaluation_as_of":"2026-10-28","rationale":"TruthfulQA's 2021 results document factual errors in GPT-3-era models; the best tested model was truthful on 58% of its misconception-focused questions [e1]. The report published March 4, 2022 documents InstructGPT fabricating facts, failing instructions and accepting false premises, alongside improved instruction following [e2]. This supports continued fallibility, but only partially addresses the behavioral claim: neither report measures how readily ordinary users could find failures across the relevant models [e1, e2]. Neither independently measures comprehension or establishes its absence as the cause of those failures, leaving the causal claim unresolved [e1, e2].","limitations":["Benchmark scores and qualitative examples do not establish that anyone could easily elicit failures across all relevant models [e1, e2].","The evidence does not distinguish missing comprehension from competing explanations for the errors [e1, e2]."],"evidence":[{"id":"e1","relation":"supports","excerpt":"truthful on 58% of questions","locator":"Abstract","url":"https://arxiv.org/abs/2109.07958","publisher":"Stephanie Lin, Jacob Hilton and Owain Evans / arXiv","event_date":"2021","available_at":"2022-05-08","explanation":"The benchmark documents substantial factual fallibility in the tested GPT-3-era models. It supports the behavioral component, but measures neither users' effort to discover failures nor their proposed cognitive cause."},{"id":"e2","relation":"supports","excerpt":"InstructGPT still makes simple mistakes.","locator":"Section 4.3, Qualitative results","url":"https://arxiv.org/html/2203.02155v1","publisher":"OpenAI researchers / arXiv","event_date":"2022","available_at":"2022-03-04","explanation":"The report describes false-premise acceptance and failures under multiple constraints, extending the behavioral evidence to InstructGPT. It also reports improved instruction following and generalization, without establishing whether comprehension is absent."}],"what_would_resolve":"Reproducible tests of the contemporary model versions measuring ordinary users' attempts and time needed to find failures, plus controlled studies using an independent measure of comprehension to test whether its absence explains those failures rather than competing causes.","retrospective":true,"plain_reading":true},"payload_hash":"3d9dd00d6dd3781f3ecccb16d59c73add9bb3d599a0bf630a1b62e0d0402191a","policy_version":"2026-10-28.7","recorded_at":"2026-10-29T12:56:44Z","drafter":{"provider":"openai","model":"gpt-6-astra"},"reviewers":[{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-what-does-it-mean-when-an-ai-fails-d02c78-4x-review-anthropic-1"},{"provider":"openai","model":"gpt-6-astra","run_id":"marcus-what-does-it-mean-when-an-ai-fails-d02c78-4x-review-openai-1"}],"final_text":{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-what-does-it-mean-when-an-ai-fails-204104-3x-final-anthropic-1"},"status":"current","notice":null,"superseded_by":null,"evidence_meta":[{"id":"e1","source_type":"preprint","audited":null,"derived":null,"metric":null},{"id":"e2","source_type":"preprint","audited":null,"derived":null,"metric":null}],"public_from":null,"public_until":null},"history":[],"review":{"review_due":false,"triggers":[],"last_completed_review_at":"2026-10-29T12:56:44Z","next_review_due_at":null,"horizon_now":"not_applicable"},"event_ids":["marcus-what-does-it-mean-when-an-ai-fails-d5182b-assessment-published-a2"],"last_substantive_change_at":"2026-10-29T17:00:18Z"}},{"url":"https://punditclaims.ai/claims/marcus-what-does-it-mean-when-an-ai-fails-e02fad","standing":"current","assessment_label":"Unresolved","versions":[{"assessment_id":"marcus-what-does-it-mean-when-an-ai-fails-e02fad-a2","status":"current","url":"https://punditclaims.ai/claims/marcus-what-does-it-mean-when-an-ai-fails-e02fad#assessment-marcus-what-does-it-mean-when-an-ai-fails-e02fad-a2"}],"claim":{"claim_id":"marcus-what-does-it-mean-when-an-ai-fails-e02fad","subject_id":"gary-marcus","revision":1,"content_hash":"fbbe18141921fe74fd21f39088eef3339ffba2eb4cbbd0dd76b137f1232daad8","normalized_proposition":"If given unrestricted access to GPT-4 and it is basically the same architecture with more data, Marcus and Davis will find many failures in physical, temporal, and causal reasoning within a day.","kind":"conditional_forecast","target":"GPT-4","conditions":[{"text":"OpenAI gives unrestricted access to GPT-4","type":"antecedent","from_source":true},{"text":"GPT-4 is basically the same architecture with more data","type":"antecedent","from_source":true}],"original_horizon_text":"within a day of playing around with it","topic_tags":["reliability","capabilities","scaling"],"concerns_providers":["openai"],"selection_stratum":"declared-lead","occurrences":[{"id":"marcus-what-does-it-mean-when-an-ai-fails-e02fad-o1","statement_date":"2022-06-07","statement_date_precision":"day","excerpt":"I am willing to bet SlateStarCodex now (terms to be negotiated) that if OpenAI gives us unrestricted access to GPT-4, whenever that is released, and assuming that is basically the same architecture but with more data, that within a day of playing around with it, Ernie and I will still be able find lots of examples of failures in physical reasoning, temporal reasoning, causal reasoning, and so forth.","locator":"paragraph 48","document":{"url":"https://garymarcus.substack.com/p/what-does-it-mean-when-an-ai-fails","title":"What does it mean when an AI fails? A Reply to SlateStarCodex’s riff on Gary Marcus","publisher":"Marcus on AI","published_at":"2022-06-07T17:28:09Z","date_uncertain":false,"date_provenance":"JSON-LD datePublished"},"observed_at":"2026-10-29T00:05:19Z","observed_via":"live","archive_url":null,"access_state":"full_public","snapshot_hash":"fa2a39c0800cc5475acd393215c920495373f319a6dd41cc57a65d2d3d491dbc","receipts":1}],"first_statement_date":"2022-06-07","related":[],"current":{"assessment_id":"marcus-what-does-it-mean-when-an-ai-fails-e02fad-a2","payload":{"claim_id":"marcus-what-does-it-mean-when-an-ai-fails-e02fad","subject_id":"gary-marcus","normalized_proposition":"If given unrestricted access to GPT-4 and it is basically the same architecture with more data, Marcus and Davis will find many failures in physical, temporal, and causal reasoning within a day.","kind":"conditional_forecast","conditions":["OpenAI gives unrestricted access to GPT-4","GPT-4 is basically the same architecture with more data"],"original_horizon_text":"within a day of playing around with it","statement":{"excerpt":"I am willing to bet SlateStarCodex now (terms to be negotiated) that if OpenAI gives us unrestricted access to GPT-4, whenever that is released, and assuming that is basically the same architecture but with more data, that within a day of playing around with it, Ernie and I will still be able find lots of examples of failures in physical reasoning, temporal reasoning, causal reasoning, and so forth.","locator":"paragraph 48","url":"https://garymarcus.substack.com/p/what-does-it-mean-when-an-ai-fails","date":"2022-06-07","document_title":"What does it mean when an AI fails? A Reply to SlateStarCodex’s riff on Gary Marcus","observed_at":"2026-10-29T00:05:19Z"},"criteria":{"from_speaker":["if OpenAI gives us unrestricted access to GPT-4","whenever that is released","assuming that is basically the same architecture but with more data","within a day of playing around with it","Ernie and I","lots of examples","physical reasoning, temporal reasoning, causal reasoning"],"our_interpretation":["Both access and architecture conditions must be established before scoring the forecast.","The named testers are Marcus and Ernest Davis. Other researchers' results cannot substitute for their first day of testing.","The deadline is relative to qualifying access and the start of testing; the post supplies no calendar deadline.","Lots means numerous distinct failures across the stated kinds of reasoning, but no exact count was negotiated.","Unrestricted access is not assumed to mean ordinary access to a rate-limited public service."],"support_condition":"Evidence establishes both antecedents and a timestamped first-day testing record shows Marcus and Davis finding numerous distinct failures across the specified reasoning areas.","contradiction_condition":"Evidence establishes both antecedents and a sufficiently complete record of their first day shows that they did not find numerous qualifying failures."},"axis":"forecast_outcome","outcome":"unresolved","horizon_state":"open_horizon","evaluation_as_of":"2026-10-28","rationale":"OpenAI's March 14, 2023 launch disclosure describes capped ChatGPT Plus access and a staged API rollout with default rate limits [e1]. It does not establish unrestricted access for Marcus and Davis, verify the same-architecture-with-more-data condition, or document their first day of testing [e1]. The supplied evidence therefore establishes neither the conditions needed to start the one-day window nor a qualifying testing result; the forecast remains unresolved with an open horizon [e1].","limitations":["The public rollout disclosure does not establish whether Marcus and Davis separately received qualifying access [e1].","The supplied record does not verify the architecture and additional-data condition or provide timestamped first-day testing logs.","The proposed bet leaves 'lots,' 'unrestricted access' and the precise meaning of a day undefined."],"evidence":[{"id":"e1","relation":"context","excerpt":"start inviting some developers today","locator":"API section","url":"https://openai.com/index/gpt-4-research","publisher":"OpenAI","event_date":"2023-03-14","available_at":"2023-03-14","explanation":"The launch describes a staged public rollout. It supplies no confirmation that Marcus and Davis received access satisfying the forecast's unrestricted-access condition."}],"what_would_resolve":"Documentation establishing both antecedents, followed by independently checkable logs of Marcus and Davis's first qualifying day, including start time, model version, prompts, outputs and distinct failures across the specified reasoning areas. Applying the result also requires clear interpretations of 'lots,' 'unrestricted access' and the one-day window.","retrospective":true,"plain_reading":true},"payload_hash":"44ffa27bb07be956873ddaeafe6815a7bb598bf747baf715585a7c358097533e","policy_version":"2026-10-28.7","recorded_at":"2026-10-29T12:56:44Z","drafter":{"provider":"openai","model":"gpt-6-astra"},"reviewers":[{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-what-does-it-mean-when-an-ai-fails-d02c78-4x-review-anthropic-1"},{"provider":"openai","model":"gpt-6-astra","run_id":"marcus-what-does-it-mean-when-an-ai-fails-d02c78-4x-review-openai-1"}],"final_text":{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-what-does-it-mean-when-an-ai-fails-204104-3x-final-anthropic-1"},"status":"current","notice":null,"superseded_by":null,"evidence_meta":[{"id":"e1","source_type":"company_disclosure","audited":null,"derived":null,"metric":null}],"public_from":null,"public_until":null},"history":[],"review":{"review_due":false,"triggers":[],"last_completed_review_at":"2026-10-29T12:56:44Z","next_review_due_at":null,"horizon_now":"open_horizon"},"event_ids":["marcus-what-does-it-mean-when-an-ai-fails-e02fad-assessment-published-a2"],"last_substantive_change_at":"2026-10-29T17:00:18Z"}},{"url":"https://punditclaims.ai/claims/marcus-what-does-it-mean-when-an-ai-fails-e6235f","standing":"current","assessment_label":"Unresolved","versions":[{"assessment_id":"marcus-what-does-it-mean-when-an-ai-fails-e6235f-a1","status":"current","url":"https://punditclaims.ai/claims/marcus-what-does-it-mean-when-an-ai-fails-e6235f#assessment-marcus-what-does-it-mean-when-an-ai-fails-e6235f-a1"}],"claim":{"claim_id":"marcus-what-does-it-mean-when-an-ai-fails-e6235f","subject_id":"gary-marcus","revision":2,"content_hash":"7790982245bf1d6e94b257e4f79fd67a996fdc84d25a5cd4ccc0d519cc063251","normalized_proposition":"The newer models' improvements, such as they are, come primarily from their larger and larger sets of data about how humans use word sequences, which are certainly helpful for pattern-matching machines.","kind":"causal","target":"large language models","conditions":[{"text":"such as they are","type":"qualifier","from_source":true},{"text":"primarily","type":"qualifier","from_source":true}],"original_horizon_text":null,"topic_tags":["scaling","capabilities"],"concerns_providers":[],"selection_stratum":"declared-lead","occurrences":[{"id":"marcus-what-does-it-mean-when-an-ai-fails-e6235f-o1","statement_date":"2022-06-07","statement_date_precision":"day","excerpt":"The improvements, such as they are, come, primarily because the newer models have larger and larger sets of data about how human beings use word sequences, and bigger word sequences are certainly helpful for pattern matching machines.","locator":"paragraph 38","document":{"url":"https://garymarcus.substack.com/p/what-does-it-mean-when-an-ai-fails","title":"What does it mean when an AI fails? A Reply to SlateStarCodex’s riff on Gary Marcus","publisher":"Marcus on AI","published_at":"2022-06-07T17:28:09Z","date_uncertain":false,"date_provenance":"JSON-LD datePublished"},"observed_at":"2026-10-29T00:05:19Z","observed_via":"live","archive_url":null,"access_state":"full_public","snapshot_hash":"fa2a39c0800cc5475acd393215c920495373f319a6dd41cc57a65d2d3d491dbc","receipts":1}],"first_statement_date":"2022-06-07","related":[],"current":{"assessment_id":"marcus-what-does-it-mean-when-an-ai-fails-e6235f-a1","payload":{"claim_id":"marcus-what-does-it-mean-when-an-ai-fails-e6235f","subject_id":"gary-marcus","normalized_proposition":"The newer models' improvements, such as they are, come primarily from their larger and larger sets of data about how humans use word sequences, which are certainly helpful for pattern-matching machines.","kind":"causal","conditions":["such as they are","primarily"],"original_horizon_text":null,"statement":{"excerpt":"The improvements, such as they are, come, primarily because the newer models have larger and larger sets of data about how human beings use word sequences, and bigger word sequences are certainly helpful for pattern matching machines.","locator":"paragraph 38","url":"https://garymarcus.substack.com/p/what-does-it-mean-when-an-ai-fails","date":"2022-06-07","document_title":"What does it mean when an AI fails? A Reply to SlateStarCodex’s riff on Gary Marcus","observed_at":"2026-10-29T00:05:19Z"},"criteria":{"from_speaker":["The improvements, such as they are","come, primarily because","larger and larger sets of data","how human beings use word sequences"],"our_interpretation":["The claim concerns the main cause of improvements from GPT-2 through GPT-3 and the successors discussed by June 2022.","Primarily means that increasing the quantity of training text was the leading explanation, rather than merely one useful contributor.","The ordinary reading concerns training-data quantity. It does not automatically include model parameter count, different training objectives, human feedback or prompting changes.","The sentence about pattern matching is not treated as a separately established theory of the models' internal representations."],"support_condition":"Controlled comparisons and a cross-generation accounting show that larger text datasets explain the dominant share of the relevant gains after accounting for model size, compute, data quality, prompting and training methods.","contradiction_condition":"Controlled comparisons and a cross-generation accounting establish that another factor, or factors other than data quantity together, predominantly explain those gains."},"axis":"causal_assessment","outcome":"unresolved","horizon_state":"not_applicable","evaluation_as_of":"2026-10-28","rationale":"More training data clearly contributed to progress. The scaling studies also identify parameter count and compute as important, while InstructGPT demonstrates gains from changing the training procedure. These findings pull in different directions for particular improvements. They do not establish which factor primarily explains the whole progression Marcus describes, so they do not settle his causal claim.","limitations":["An optimal scaling prescription is not a decomposition of historical progress.","The Chinchilla comparison changes more than data quantity.","Instruction-following preference is not interchangeable with factual accuracy.","Evidence for and against individual components does not settle the dominant-cause assertion."],"evidence":[{"id":"e9","relation":"supports","excerpt":"equal proportions.","locator":"Section 1, Introduction; sections 3 and 4","url":"https://arxiv.org/html/2203.15556v1","publisher":"DeepMind researchers / arXiv","event_date":"2022-03-29","available_at":"2022-03-29","explanation":"The experiments support a substantial role for more training data, including a smaller model trained on more tokens outperforming larger predecessors. They do not establish that data quantity predominantly caused the entire set of gains Marcus discusses; the paper also reports changes in the final model's training setup."},{"id":"e10","relation":"context","excerpt":"capacity-limited.","locator":"Section 4.1; section 1.1 summary of scaling results","url":"https://arxiv.org/html/2001.08361v1","publisher":"OpenAI researchers / arXiv","event_date":"2020-01-23","available_at":"2020-01-23","explanation":"The study identifies model size, dataset size and compute as distinct contributors to language-model loss. It cautions against a data-only explanation, but does not apportion the downstream gains of the full model sequence in Marcus's claim."},{"id":"e11","relation":"contradicts","excerpt":"better at following instructions","locator":"Opening explanation of reinforcement learning from human feedback; Methods","url":"https://openai.com/index/instruction-following","publisher":"OpenAI","event_date":"2022-01-27","available_at":"2022-01-27","explanation":"OpenAI attributes these gains to supervised demonstrations and preference-based reinforcement learning applied to GPT-3, using little additional data relative to pretraining. This challenges the proposed explanation for this particular successor, but cannot determine the primary cause of all the improvements under discussion."}],"what_would_resolve":"Matched ablations and a clearly scoped comparison of the relevant model generations that quantify the contributions of data quantity, model size, compute and training methods to the same performance measures.","retrospective":true,"plain_reading":true},"payload_hash":"979522e47f6216084175fa45daf1c416304ab4680e6af3c1e94b5d4c2657c306","policy_version":"2026-10-28.7","recorded_at":"2026-10-29T12:47:03Z","drafter":{"provider":"openai","model":"gpt-6-astra"},"reviewers":[{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-what-does-it-mean-when-an-ai-fails-d02c78-4x-review-anthropic-1"},{"provider":"openai","model":"gpt-6-astra","run_id":"marcus-what-does-it-mean-when-an-ai-fails-d02c78-4x-review-openai-1"}],"final_text":{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-what-does-it-mean-when-an-ai-fails-d02c78-4x-final-anthropic-1"},"status":"current","notice":null,"superseded_by":null,"evidence_meta":[{"id":"e9","source_type":"preprint","audited":null,"derived":null,"metric":null},{"id":"e10","source_type":"preprint","audited":null,"derived":null,"metric":null},{"id":"e11","source_type":"company_disclosure","audited":null,"derived":null,"metric":null}],"public_from":null,"public_until":null},"history":[],"review":{"review_due":false,"triggers":[],"last_completed_review_at":"2026-10-29T12:47:03Z","next_review_due_at":null,"horizon_now":"not_applicable"},"event_ids":["marcus-what-does-it-mean-when-an-ai-fails-e6235f-assessment-published-a1"],"last_substantive_change_at":"2026-10-29T17:00:18Z"}},{"url":"https://punditclaims.ai/claims/marcus-what-does-it-mean-when-an-ai-fails-f2ac96","standing":"current","assessment_label":"Can’t be checked","versions":[{"assessment_id":"marcus-what-does-it-mean-when-an-ai-fails-f2ac96-a1","status":"current","url":"https://punditclaims.ai/claims/marcus-what-does-it-mean-when-an-ai-fails-f2ac96#assessment-marcus-what-does-it-mean-when-an-ai-fails-f2ac96-a1"}],"claim":{"claim_id":"marcus-what-does-it-mean-when-an-ai-fails-f2ac96","subject_id":"gary-marcus","revision":1,"content_hash":"09dac4a855ae1c079e7ef9f070e775c5eafb94b63e4e3e157bd7b5f10adf017b","normalized_proposition":"Improvement from GPT-2 to GPT-3 has been quantitative only, with zero qualitative change.","kind":"observation","target":"GPT models","conditions":[],"original_horizon_text":null,"topic_tags":["capabilities","scaling"],"concerns_providers":["openai"],"selection_stratum":"declared-lead","occurrences":[{"id":"marcus-what-does-it-mean-when-an-ai-fails-f2ac96-o1","statement_date":"2022-06-07","statement_date_precision":"day","excerpt":"And yes, there has been improvement eg from GPT-2 to GPT-3—but zero qualitative change.","locator":"paragraph 46","document":{"url":"https://garymarcus.substack.com/p/what-does-it-mean-when-an-ai-fails","title":"What does it mean when an AI fails? A Reply to SlateStarCodex’s riff on Gary Marcus","publisher":"Marcus on AI","published_at":"2022-06-07T17:28:09Z","date_uncertain":false,"date_provenance":"JSON-LD datePublished"},"observed_at":"2026-10-29T00:05:19Z","observed_via":"live","archive_url":null,"access_state":"full_public","snapshot_hash":"fa2a39c0800cc5475acd393215c920495373f319a6dd41cc57a65d2d3d491dbc","receipts":1}],"first_statement_date":"2022-06-07","related":[],"current":{"assessment_id":"marcus-what-does-it-mean-when-an-ai-fails-f2ac96-a1","payload":{"claim_id":"marcus-what-does-it-mean-when-an-ai-fails-f2ac96","subject_id":"gary-marcus","normalized_proposition":"Improvement from GPT-2 to GPT-3 has been quantitative only, with zero qualitative change.","kind":"observation","conditions":[],"original_horizon_text":null,"statement":{"excerpt":"And yes, there has been improvement eg from GPT-2 to GPT-3—but zero qualitative change.","locator":"paragraph 46","url":"https://garymarcus.substack.com/p/what-does-it-mean-when-an-ai-fails","date":"2022-06-07","document_title":"What does it mean when an AI fails? A Reply to SlateStarCodex’s riff on Gary Marcus","observed_at":"2026-10-29T00:05:19Z"},"criteria":{"from_speaker":[],"our_interpretation":[],"support_condition":"Not checked against evidence: both screens found that public evidence could not settle this claim.","contradiction_condition":"Not checked against evidence: both screens found that public evidence could not settle this claim."},"axis":"not_applicable","outcome":"not_adjudicable","horizon_state":"not_applicable","evaluation_as_of":"2026-10-28","rationale":"Not checked against evidence. Two independent screens, one from each provider, both found this too vague to test: informed readers would test it in different ways and reach different answers.","limitations":[],"evidence":[],"what_would_resolve":null,"retrospective":true,"uncheckable":["no_plain_reading"]},"payload_hash":"fb859ef214eaa645aadfb437f07f26ace0e0820c92b2dac7573d1579ddc91eab","policy_version":"2026-10-28.7","recorded_at":"2026-10-29T01:56:19Z","drafter":{"provider":"anthropic","model":"claude-opus-5-5"},"reviewers":[{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-what-does-it-mean-when-an-ai-fails-screen-anthropic-1"},{"provider":"openai","model":"gpt-6-astra","run_id":"marcus-what-does-it-mean-when-an-ai-fails-screen-openai-1"}],"final_text":null,"status":"current","notice":null,"superseded_by":null,"evidence_meta":[],"public_from":null,"public_until":null},"history":[],"review":{"review_due":false,"triggers":[],"last_completed_review_at":"2026-10-29T01:56:19Z","next_review_due_at":null,"horizon_now":"not_applicable"},"event_ids":["marcus-what-does-it-mean-when-an-ai-fails-f2ac96-assessment-published-a1"],"last_substantive_change_at":"2026-10-29T17:00:18Z"}},{"url":"https://punditclaims.ai/claims/marcus-dear-elon-musk-here-are-five-things-056704","standing":"current","assessment_label":"Unresolved","versions":[{"assessment_id":"marcus-dear-elon-musk-here-are-five-things-056704-a1","status":"current","url":"https://punditclaims.ai/claims/marcus-dear-elon-musk-here-are-five-things-056704#assessment-marcus-dear-elon-musk-here-are-five-things-056704-a1"}],"claim":{"claim_id":"marcus-dear-elon-musk-here-are-five-things-056704","subject_id":"gary-marcus","revision":1,"content_hash":"45c50e5adf326e44ad090d14e42bcacd0fa4a0fe5b58326ad65dfec79df4e6df","normalized_proposition":"Humanity lacks an adequate methodology for building complex cognitive systems.","kind":"observation","target":"AI engineering","conditions":[],"original_horizon_text":null,"topic_tags":["capabilities"],"concerns_providers":[],"selection_stratum":"declared-lead","occurrences":[{"id":"marcus-dear-elon-musk-here-are-five-things-056704-o1","statement_date":"2022-05-31","statement_date_precision":"day","excerpt":"The fourth thing to realize is that we, and by this I mean humanity as a whole, still don’t really have an adequate methodology for building complex cognitive systems.","locator":"paragraph 27","document":{"url":"https://garymarcus.substack.com/p/dear-elon-musk-here-are-five-things","title":"Dear Elon Musk, here are five things you might want to consider about AGI","publisher":"Marcus on AI","published_at":"2022-05-31T18:32:41Z","date_uncertain":false,"date_provenance":"JSON-LD datePublished"},"observed_at":"2026-10-28T22:49:04Z","observed_via":"live","archive_url":null,"access_state":"full_public","snapshot_hash":"d64c073bb595b86ed0d5a52382668c601d5a8e7009353d00f0e54061bdd4f0f7","receipts":1}],"first_statement_date":"2022-05-31","related":[],"current":{"assessment_id":"marcus-dear-elon-musk-here-are-five-things-056704-a1","payload":{"claim_id":"marcus-dear-elon-musk-here-are-five-things-056704","subject_id":"gary-marcus","normalized_proposition":"Humanity lacks an adequate methodology for building complex cognitive systems.","kind":"observation","conditions":[],"original_horizon_text":null,"statement":{"excerpt":"The fourth thing to realize is that we, and by this I mean humanity as a whole, still don’t really have an adequate methodology for building complex cognitive systems.","locator":"paragraph 27","url":"https://garymarcus.substack.com/p/dear-elon-musk-here-are-five-things","date":"2022-05-31","document_title":"Dear Elon Musk, here are five things you might want to consider about AGI","observed_at":"2026-10-28T22:49:04Z"},"criteria":{"from_speaker":["humanity as a whole","still don’t really have","an adequate methodology","building complex cognitive systems"],"our_interpretation":["An adequate methodology means a repeatable engineering approach for building and modifying integrated cognitive systems with dependable behavior and manageable regressions.","Adequacy does not require eliminating every bug. Conversely, a successful isolated model does not establish a generally adequate integration methodology.","The context concerns perception, reasoning, planning and action working together, including autonomous vehicles.","The current observation window is calendar 2022 through May 31 because the research field has no regular reporting cycle; earlier methods are context."],"support_condition":"Broad contemporary evidence shows that available engineering approaches cannot reliably manage integration and modification of complex cognitive systems across relevant applications.","contradiction_condition":"A publicly documented, repeatable methodology had demonstrated dependable construction and modification of such systems across relevant applications by May 2022."},"axis":"historical_observation","outcome":"unresolved","horizon_state":"not_applicable","evaluation_as_of":"2026-10-28","rationale":"The industry study corroborates the specific integration and regression problems Marcus describes. However, concrete testing methods and integrated systems also existed. Persistent failures do not establish that humanity lacked an adequate methodology, just as a successful demonstration does not establish a generally dependable one. The retrieved evidence cannot settle the field-wide claim at the stated level of adequacy.","limitations":["The empirical study covers a sample of practitioners and systems.","The evidence mixes production engineering practices with research demonstrations.","Publication dates do not identify the exact dates of all underlying experiments or interviews."],"evidence":[{"id":"e1","relation":"supports","excerpt":"regression on model performance.","locator":"Abstract, finding 2","url":"https://www.microsoft.com/en-us/research/publication/testing-machine-learning-systems-in-industry-an-empirical-study","publisher":"Microsoft Research; ICSE 2022","event_date":"2022-05","available_at":"2022-05-31","explanation":"The study reports industry integration and regression problems matching Marcus's concern, while also identifying useful practices. The page gives publication month only; available_at uses that month's final day as a conservative availability bound."},{"id":"e2","relation":"context","excerpt":"multi-embodiment generalist policy.","locator":"Abstract","url":"https://arxiv.org/abs/2205.06175v1","publisher":"arXiv; Scott Reed and colleagues","event_date":"2022-05","available_at":"2022-05-12","explanation":"Gato demonstrates integration across several kinds of tasks and embodiments before the post. That is relevant counterevidence to an inability-to-build reading, but it does not demonstrate dependable maintenance or regression control under the contract."},{"id":"e3","relation":"context","excerpt":"technical debt","locator":"Abstract","url":"https://research.google/pubs/the-ml-test-score-a-rubric-for-ml-production-readiness-and-technical-debt-reduction","publisher":"Google Research; Proceedings of IEEE Big Data","event_date":"2017","available_at":"2017-12-31","explanation":"The paper proposes 28 tests and monitoring needs based on production experience, establishing that concrete engineering methods existed. It predates the target period, and the page supplies only a publication year; available_at is a conservative year-end bound."}],"what_would_resolve":"A defined reliability standard and comparative evidence showing whether contemporary methods could repeatedly build, modify and maintain representative integrated cognitive systems to that standard.","retrospective":true,"plain_reading":true},"payload_hash":"cdb08833d1f1f625a1dbdd13fcd73c4a75b21e7b3193ec0d93c0cb24272e1e7d","policy_version":"2026-10-28.7","recorded_at":"2026-10-29T06:55:52Z","drafter":{"provider":"openai","model":"gpt-6-astra"},"reviewers":[{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-dear-elon-musk-here-are-five-things-056704-4x-review-anthropic-1"},{"provider":"openai","model":"gpt-6-astra","run_id":"marcus-dear-elon-musk-here-are-five-things-056704-4x-review-openai-1"}],"final_text":{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-dear-elon-musk-here-are-five-things-056704-4x-final-anthropic-1"},"status":"current","notice":null,"superseded_by":null,"evidence_meta":[{"id":"e1","source_type":"peer_reviewed","audited":null,"derived":null,"metric":null},{"id":"e2","source_type":"preprint","audited":null,"derived":null,"metric":null},{"id":"e3","source_type":"peer_reviewed","audited":null,"derived":null,"metric":null}],"public_from":null,"public_until":null},"history":[],"review":{"review_due":false,"triggers":[],"last_completed_review_at":"2026-10-29T06:55:52Z","next_review_due_at":null,"horizon_now":"not_applicable"},"event_ids":["marcus-dear-elon-musk-here-are-five-things-056704-assessment-published-a1"],"last_substantive_change_at":"2026-10-29T17:00:18Z"}},{"url":"https://punditclaims.ai/claims/marcus-dear-elon-musk-here-are-five-things-082354","standing":"current","assessment_label":"Unresolved","versions":[{"assessment_id":"marcus-dear-elon-musk-here-are-five-things-082354-a1","status":"current","url":"https://punditclaims.ai/claims/marcus-dear-elon-musk-here-are-five-things-082354#assessment-marcus-dear-elon-musk-here-are-five-things-082354-a1"}],"claim":{"claim_id":"marcus-dear-elon-musk-here-are-five-things-082354","subject_id":"gary-marcus","revision":1,"content_hash":"70a967ae0f7aedf0b5199c445ccf34d08b87594523c60561772d499166b73a59","normalized_proposition":"In 2029, AI will not be able to reliably build bug-free code over 10,000 lines from a natural-language spec or from talking with a non-expert user, not counting glued-together library code.","kind":"forecast","target":"AI","conditions":[{"text":"from natural language spec or non-expert interaction","type":"qualifier","from_source":true},{"text":"gluing existing library code doesn't count","type":"qualifier","from_source":true}],"original_horizon_text":"In 2029","topic_tags":["capabilities","agi-timelines"],"concerns_providers":[],"selection_stratum":"declared-lead","occurrences":[{"id":"marcus-dear-elon-musk-here-are-five-things-082354-o1","statement_date":"2022-05-31","statement_date_precision":"day","excerpt":"In 2029, AI will not be able to reliably construct bug-free code of more than 10,000 lines from natural language specification or by interactions with a non-expert user.","locator":"prediction 4 of 5","document":{"url":"https://garymarcus.substack.com/p/dear-elon-musk-here-are-five-things","title":"Dear Elon Musk, here are five things you might want to consider about AGI","publisher":"Marcus on AI","published_at":"2022-05-31T18:32:41Z","date_uncertain":false,"date_provenance":"JSON-LD datePublished"},"observed_at":"2026-10-28T22:49:04Z","observed_via":"live","archive_url":null,"access_state":"full_public","snapshot_hash":"d64c073bb595b86ed0d5a52382668c601d5a8e7009353d00f0e54061bdd4f0f7","receipts":1}],"first_statement_date":"2022-05-31","related":[],"current":{"assessment_id":"marcus-dear-elon-musk-here-are-five-things-082354-a1","payload":{"claim_id":"marcus-dear-elon-musk-here-are-five-things-082354","subject_id":"gary-marcus","normalized_proposition":"In 2029, AI will not be able to reliably build bug-free code over 10,000 lines from a natural-language spec or from talking with a non-expert user, not counting glued-together library code.","kind":"forecast","conditions":["from natural language spec or non-expert interaction","gluing existing library code doesn't count"],"original_horizon_text":"In 2029","statement":{"excerpt":"In 2029, AI will not be able to reliably construct bug-free code of more than 10,000 lines from natural language specification or by interactions with a non-expert user.","locator":"prediction 4 of 5","url":"https://garymarcus.substack.com/p/dear-elon-musk-here-are-five-things","date":"2022-05-31","document_title":"Dear Elon Musk, here are five things you might want to consider about AGI","observed_at":"2026-10-28T22:49:04Z"},"criteria":{"from_speaker":["In 2029","AI will not be able to reliably construct bug-free code","more than 10,000 lines","from natural language specification or by interactions with a non-expert user","Gluing together code from existing libraries doesn’t count."],"our_interpretation":["The deadline is December 31, 2029, allowing the whole named year.","Reliably means repeatable success across varied substantive programming tasks, rather than one selected demonstration. Marcus supplies no numerical success rate.","Bug-free means correctly implementing the requirements without functional defects. Passing a finite test suite is evidence of correctness, but does not by itself establish this.","Count substantive newly generated source code, excluding padding and pre-existing library code. Marcus does not specify a line-count convention.","Either input route qualifies. An expert may supply the natural-language specification under the first route; expert implementation or repair would not establish the AI's capability unaided.","The input restrictions define the task, rather than a conditional forecast requiring a separate antecedent.","This assessment concerns the individual programming prediction, not winning the broader three-of-five AGI bet."],"support_condition":"Evaluations during 2029, covering leading available systems and varied qualifying tasks, consistently find that systems cannot reliably produce correct new programs exceeding 10,000 substantive source lines under either permitted input route. Mere absence of a public demonstration is insufficient.","contradiction_condition":"A system repeatedly produces correct new programs exceeding 10,000 substantive source lines from natural-language specifications or non-expert interaction, with independently assessed correctness and without counting library assembly as the implementation."},"axis":"forecast_outcome","outcome":"unresolved","horizon_state":"not_due","evaluation_as_of":"2026-10-28","rationale":"The deadline is December 31, 2029. The compiler demonstration shows substantial progress beyond the size threshold, but its disclosed defects prevent it from establishing reliable bug-free construction. The verification work proves selected properties of existing software under stated assumptions. Neither meets the early-resolution rule. Present limitations also cannot establish what AI will be unable to do in 2029.","limitations":["The cited demonstrations are developer reports, not independent evaluations satisfying the whole contract.","The search cannot establish that no qualifying system exists.","No numerical reliability threshold was supplied; none is invented here."],"evidence":[{"id":"e1","relation":"context","excerpt":"New features and bugfixes frequently broke existing functionality.","locator":"Stress testing the limits of agent teams, Evaluation","url":"https://www.anthropic.com/engineering/building-c-compiler","publisher":"Anthropic","event_date":"2026-02-05","available_at":"2026-02-05","explanation":"Anthropic reported a newly built 100,000-line compiler, exceeding the forecast's size threshold. The same report describes defects, regressions and substantial expert work on testing, so it does not establish reliable bug-free construction under the complete criteria."},{"id":"e2","relation":"context","excerpt":"“Formally verified” does not mean “bug-free.”","locator":"Limitations","url":"https://www.anthropic.com/research/formal-verification","publisher":"Anthropic","event_date":"2026-10-22","available_at":"2026-10-22","explanation":"The report describes proving security properties for existing software under explicit machine and input assumptions. This is relevant progress, but it does not demonstrate repeated generation of qualifying new programs with complete correctness against natural-language requirements."}],"what_would_resolve":"A reproducible independent evaluation showing repeated qualifying generation could contradict the forecast early. Otherwise, broad evaluations during 2029 must assess the full task, with disclosed success rates, substantive line counts, intervention logs and correctness evidence.","retrospective":false,"plain_reading":true},"payload_hash":"a200f052c8eca1b457d741ce5204270dcfb817e5df036b0c28de8344ed571146","policy_version":"2026-10-28.7","recorded_at":"2026-10-29T06:55:52Z","drafter":{"provider":"openai","model":"gpt-6-astra"},"reviewers":[{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-dear-elon-musk-here-are-five-things-056704-4x-review-anthropic-1"},{"provider":"openai","model":"gpt-6-astra","run_id":"marcus-dear-elon-musk-here-are-five-things-056704-4x-review-openai-1"}],"final_text":{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-dear-elon-musk-here-are-five-things-056704-4x-final-anthropic-1"},"status":"current","notice":null,"superseded_by":null,"evidence_meta":[{"id":"e1","source_type":"company_disclosure","audited":null,"derived":null,"metric":null},{"id":"e2","source_type":"company_disclosure","audited":null,"derived":null,"metric":null}],"public_from":null,"public_until":null},"history":[],"review":{"review_due":false,"triggers":[],"last_completed_review_at":"2026-10-29T06:55:52Z","next_review_due_at":null,"horizon_now":"not_due"},"event_ids":["marcus-dear-elon-musk-here-are-five-things-082354-assessment-published-a1"],"last_substantive_change_at":"2026-10-29T17:00:18Z"}},{"url":"https://punditclaims.ai/claims/marcus-dear-elon-musk-here-are-five-things-12f5f4","standing":"current","assessment_label":"Can’t be checked","versions":[{"assessment_id":"marcus-dear-elon-musk-here-are-five-things-12f5f4-a1","status":"current","url":"https://punditclaims.ai/claims/marcus-dear-elon-musk-here-are-five-things-12f5f4#assessment-marcus-dear-elon-musk-here-are-five-things-12f5f4-a1"}],"claim":{"claim_id":"marcus-dear-elon-musk-here-are-five-things-12f5f4","subject_id":"gary-marcus","revision":2,"content_hash":"a335b6bb94ca93e3a5d2aeb265f79c4b0c2a304353a22a2b14d8bbe3e965602d","normalized_proposition":"Many people continue to hope deep learning will be a panacea, which seems pretty unrealistic to Marcus, and he still thinks a whole lot more techniques are needed.","kind":"normative","target":"deep learning","conditions":[{"text":"seems pretty unrealistic to me","type":"qualifier","from_source":true},{"text":"I still think","type":"qualifier","from_source":true}],"original_horizon_text":null,"topic_tags":["capabilities","agi-timelines"],"concerns_providers":[],"selection_stratum":"declared-lead","occurrences":[{"id":"marcus-dear-elon-musk-here-are-five-things-12f5f4-o1","statement_date":"2022-05-31","statement_date_precision":"day","excerpt":"Four years later, many people continue to hope that deep learning will be a panacea; that still seems pretty unrealistic to me; I still think we need a whole lot more techniques.","locator":"paragraph 25","document":{"url":"https://garymarcus.substack.com/p/dear-elon-musk-here-are-five-things","title":"Dear Elon Musk, here are five things you might want to consider about AGI","publisher":"Marcus on AI","published_at":"2022-05-31T18:32:41Z","date_uncertain":false,"date_provenance":"JSON-LD datePublished"},"observed_at":"2026-10-28T22:49:04Z","observed_via":"live","archive_url":null,"access_state":"full_public","snapshot_hash":"d64c073bb595b86ed0d5a52382668c601d5a8e7009353d00f0e54061bdd4f0f7","receipts":1}],"first_statement_date":"2022-05-31","related":[],"current":{"assessment_id":"marcus-dear-elon-musk-here-are-five-things-12f5f4-a1","payload":{"claim_id":"marcus-dear-elon-musk-here-are-five-things-12f5f4","subject_id":"gary-marcus","normalized_proposition":"Many people continue to hope deep learning will be a panacea, which seems pretty unrealistic to Marcus, and he still thinks a whole lot more techniques are needed.","kind":"normative","conditions":["seems pretty unrealistic to me","I still think"],"original_horizon_text":null,"statement":{"excerpt":"Four years later, many people continue to hope that deep learning will be a panacea; that still seems pretty unrealistic to me; I still think we need a whole lot more techniques.","locator":"paragraph 25","url":"https://garymarcus.substack.com/p/dear-elon-musk-here-are-five-things","date":"2022-05-31","document_title":"Dear Elon Musk, here are five things you might want to consider about AGI","observed_at":"2026-10-28T22:49:04Z"},"criteria":{"from_speaker":["seems pretty unrealistic to me","I still think"],"our_interpretation":[],"support_condition":"Not checked against evidence: both screens found that public evidence could not settle this claim.","contradiction_condition":"Not checked against evidence: both screens found that public evidence could not settle this claim."},"axis":"not_applicable","outcome":"not_adjudicable","horizon_state":"not_applicable","evaluation_as_of":"2026-10-28","rationale":"Not checked against evidence. Two independent screens, one from each provider, both found that evidence could not settle it: one found it too vague to test: informed readers would test it in different ways and reach different answers; the other found it a recommendation or a value judgment.","limitations":[],"evidence":[],"what_would_resolve":null,"retrospective":true,"uncheckable":["no_plain_reading","value_judgment"]},"payload_hash":"a0e4fb1f30e96f7221cf6c2905be357ec254040374740a8c9a13fff33e646544","policy_version":"2026-10-28.7","recorded_at":"2026-10-29T02:32:05Z","drafter":{"provider":"anthropic","model":"claude-opus-5-5"},"reviewers":[{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-dear-elon-musk-here-are-five-things-screen-anthropic-2"},{"provider":"openai","model":"gpt-6-astra","run_id":"marcus-dear-elon-musk-here-are-five-things-screen-openai-2"}],"final_text":null,"status":"current","notice":null,"superseded_by":null,"evidence_meta":[],"public_from":null,"public_until":null},"history":[],"review":{"review_due":false,"triggers":[],"last_completed_review_at":"2026-10-29T02:32:05Z","next_review_due_at":null,"horizon_now":"not_applicable"},"event_ids":["marcus-dear-elon-musk-here-are-five-things-12f5f4-assessment-published-a1"],"last_substantive_change_at":"2026-10-29T17:00:18Z"}},{"url":"https://punditclaims.ai/claims/marcus-dear-elon-musk-here-are-five-things-1663f2","standing":"current","assessment_label":"Supported","versions":[{"assessment_id":"marcus-dear-elon-musk-here-are-five-things-1663f2-a1","status":"current","url":"https://punditclaims.ai/claims/marcus-dear-elon-musk-here-are-five-things-1663f2#assessment-marcus-dear-elon-musk-here-are-five-things-1663f2-a1"}],"claim":{"claim_id":"marcus-dear-elon-musk-here-are-five-things-1663f2","subject_id":"gary-marcus","revision":2,"content_hash":"aa6474244fcf5affa8770dd63f6bd6c5be7975b9f7a8653a106610cea2ecceff","normalized_proposition":"Musk said in 2015 that truly self-driving cars were two years away and has pretty much said the same thing every year since, and truly self-driving cars still had not arrived.","kind":"observation","target":"truly self-driving cars","conditions":[{"text":"pretty much every year since","type":"qualifier","from_source":true}],"original_horizon_text":null,"topic_tags":["capabilities","agi-timelines"],"concerns_providers":[],"selection_stratum":"declared-lead","occurrences":[{"id":"marcus-dear-elon-musk-here-are-five-things-1663f2-o1","statement_date":"2022-05-31","statement_date_precision":"day","excerpt":"You said, for instance in 2015, that (truly) self-driving cars were two years away; you’ve pretty much said the same thing every year since. It still hasn’t happened.","locator":"paragraph 9","document":{"url":"https://garymarcus.substack.com/p/dear-elon-musk-here-are-five-things","title":"Dear Elon Musk, here are five things you might want to consider about AGI","publisher":"Marcus on AI","published_at":"2022-05-31T18:32:41Z","date_uncertain":false,"date_provenance":"JSON-LD datePublished"},"observed_at":"2026-10-28T22:49:04Z","observed_via":"live","archive_url":null,"access_state":"full_public","snapshot_hash":"d64c073bb595b86ed0d5a52382668c601d5a8e7009353d00f0e54061bdd4f0f7","receipts":1}],"first_statement_date":"2022-05-31","related":[],"current":{"assessment_id":"marcus-dear-elon-musk-here-are-five-things-1663f2-a1","payload":{"claim_id":"marcus-dear-elon-musk-here-are-five-things-1663f2","subject_id":"gary-marcus","normalized_proposition":"Musk said in 2015 that truly self-driving cars were two years away and has pretty much said the same thing every year since, and truly self-driving cars still had not arrived.","kind":"observation","conditions":["pretty much every year since"],"original_horizon_text":null,"statement":{"excerpt":"You said, for instance in 2015, that (truly) self-driving cars were two years away; you’ve pretty much said the same thing every year since. It still hasn’t happened.","locator":"paragraph 9","url":"https://garymarcus.substack.com/p/dear-elon-musk-here-are-five-things","date":"2022-05-31","document_title":"Dear Elon Musk, here are five things you might want to consider about AGI","observed_at":"2026-10-28T22:49:04Z"},"criteria":{"from_speaker":["in 2015","(truly) self-driving cars were two years away","pretty much said the same thing every year since","It still hasn’t happened"],"our_interpretation":["The surrounding discussion makes Tesla the ordinary scope of the claim.","Truly self-driving means driving without a human continuously supervising and being ready to intervene.","Pretty much the same thing means repeated near-term forecasts, usually within about two years, rather than an identical two-year promise every calendar year.","The target period covers the explicitly named prediction history through the post date. The non-arrival assertion concerns the end of that period."],"support_condition":"Records establish the 2015 two-year prediction, substantially annual repetitions of near-term autonomy predictions, and Tesla's continued reliance on supervised driving assistance before the post.","contradiction_condition":"Records disprove the attributed 2015 statement or the recurring prediction pattern, or establish that Tesla had delivered the promised unsupervised driving capability before the post."},"axis":"historical_observation","outcome":"supported","horizon_state":"not_applicable","evaluation_as_of":"2026-10-28","rationale":"The 2015 quotation is accurate. Dated accounts document recurring near-term forecasts through 2022, while contemporary testing still required human supervision. This supports the ordinary Tesla-specific reading. The timelines and promised milestones varied, so the evidence does not establish an identical two-year promise every year. Waymo's earlier driverless service means the statement should not be generalized to every company.","limitations":["Several predictions are documented through contemporary reporting because readable original transcripts were unavailable.","The direct 2022 driving observations cover January and February rather than May 31 itself.","The outcome depends on the contextual Tesla-specific scope."],"evidence":[{"id":"e1","relation":"supports","excerpt":"“We’re going to end up with complete autonomy, and I think we will have complete autonomy in approximately two years.”","locator":"The Path to Full Autonomy","url":"https://fortune.com/2015/12/21/elon-musk-interview","publisher":"Fortune","event_date":"2015-12","available_at":"2015-12-21","explanation":"The original interview establishes the attributed prediction. Musk separately allowed additional time for regulatory approval, so the prediction concerned technical capability first."},{"id":"e2","relation":"supports","excerpt":"I feel very confident predicting autonomous robo taxis for Tesla by next year.","locator":"Opening account of Autonomy Day; subsequent dated chronology","url":"https://www.inverse.com/science/55141-elon-musk-autonomous-driving","publisher":"Inverse","event_date":"2019-04-22","available_at":"2019-04-22","explanation":"Reports the 2019 robotaxi forecast. Its chronology also documents a 2016 coast-to-coast forecast, a 2017 two-year sleeping-at-the-wheel forecast, and a 2018 forecast of full self-driving the following year."},{"id":"e3","relation":"supports","excerpt":"Tesla will be able to make its vehicles completely autonomous by the end of this year, founder Elon Musk has said.","locator":"Opening paragraph","url":"https://www.bbc.com/news/technology-53349313","publisher":"BBC News","event_date":"2020-07","available_at":"2020-07-09","explanation":"Documents another near-term prediction in 2020, explicitly concerning autonomy without driver input."},{"id":"e4","relation":"supports","excerpt":"“I’m confident based on my understanding of the technical roadmap and the progress that we’re making between each beta iteration,” Musk said.","locator":"Opening two paragraphs","url":"https://arstechnica.com/cars/2021/05/tesla-autopilot-director-contradicts-musks-self-driving-timeline","publisher":"Ars Technica","event_date":"2021-01","available_at":"2021-05-07","explanation":"The preceding paragraph identifies this as Musk's answer about achieving Level 5 by the end of 2021. This extends the recurring short-timeline pattern into 2021."},{"id":"e5","relation":"supports","excerpt":"All three drivers understood that the technology does not make their Tesla electric cars fully self driving, despite the brand name.","locator":"Introduction describing January and February ride-alongs","url":"https://www.cnbc.com/2022/02/15/tesla-fsd-beta-users-show-cnbc-how-the-system-works-and-doesnt.html","publisher":"CNBC","event_date":"2022-02","available_at":"2022-02-15","explanation":"Direct ride-alongs documented continuing supervision and interventions. The article also reports Musk's renewed prediction, on the 2021 year-end earnings call, of achieving full self-driving during 2022."},{"id":"e6","relation":"context","excerpt":"Beginning today, October 8, we’re excited to open up our fully driverless offering to Waymo One riders.","locator":"Public opening announcement","url":"https://waymo.com/blog/2020/10/waymo-is-opening-its-fully-driverless-service-in-phoenix","publisher":"Waymo","event_date":"2020-10-08","available_at":"2020-10-08","explanation":"Driverless service existed at another company within a limited service area. This is material to the wording's scope, but does not establish that Tesla fulfilled Musk's predictions."}],"what_would_resolve":null,"retrospective":true,"plain_reading":true},"payload_hash":"f5350ba7175a7e7351b2244fa5a2e36d6722e7880a4b991e7b8bd50a6b3a40cd","policy_version":"2026-10-28.7","recorded_at":"2026-10-29T06:55:52Z","drafter":{"provider":"openai","model":"gpt-6-astra"},"reviewers":[{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-dear-elon-musk-here-are-five-things-056704-4x-review-anthropic-1"},{"provider":"openai","model":"gpt-6-astra","run_id":"marcus-dear-elon-musk-here-are-five-things-056704-4x-review-openai-1"}],"final_text":{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-dear-elon-musk-here-are-five-things-056704-4x-final-anthropic-1"},"status":"current","notice":null,"superseded_by":null,"evidence_meta":[{"id":"e1","source_type":"news_report","audited":null,"derived":null,"metric":null},{"id":"e2","source_type":"news_report","audited":null,"derived":null,"metric":null},{"id":"e3","source_type":"news_report","audited":null,"derived":null,"metric":null},{"id":"e4","source_type":"news_report","audited":null,"derived":null,"metric":null},{"id":"e5","source_type":"news_report","audited":null,"derived":null,"metric":null},{"id":"e6","source_type":"company_disclosure","audited":null,"derived":null,"metric":null}],"public_from":null,"public_until":null},"history":[],"review":{"review_due":false,"triggers":[],"last_completed_review_at":"2026-10-29T06:55:52Z","next_review_due_at":null,"horizon_now":"not_applicable"},"event_ids":["marcus-dear-elon-musk-here-are-five-things-1663f2-assessment-published-a1"],"last_substantive_change_at":"2026-10-29T17:00:18Z"}},{"url":"https://punditclaims.ai/claims/marcus-dear-elon-musk-here-are-five-things-1c4f0d","standing":"current","assessment_label":"Can’t be checked","versions":[{"assessment_id":"marcus-dear-elon-musk-here-are-five-things-1c4f0d-a1","status":"current","url":"https://punditclaims.ai/claims/marcus-dear-elon-musk-here-are-five-things-1c4f0d#assessment-marcus-dear-elon-musk-here-are-five-things-1c4f0d-a1"}],"claim":{"claim_id":"marcus-dear-elon-musk-here-are-five-things-1c4f0d","subject_id":"gary-marcus","revision":1,"content_hash":"e45a183695a52a5b0b259a383861d2ec6f8437619e273fd1362f7ffcb3b4c493","normalized_proposition":"Unless the field invests in ideas beyond current techniques like deep learning, outliers alone may prevent reaching AGI before the end of the decade.","kind":"conditional_forecast","target":"the AI field","conditions":[{"text":"unless the field invests in ideas beyond deep learning","type":"antecedent","from_source":true}],"original_horizon_text":"before the end of the decade","topic_tags":["agi-timelines","reliability"],"concerns_providers":[],"selection_stratum":"declared-lead","occurrences":[{"id":"marcus-dear-elon-musk-here-are-five-things-1c4f0d-o1","statement_date":"2022-05-31","statement_date_precision":"day","excerpt":"Seven years is a long time, but the field is going to need to invest in other ideas if we are going to get to AGI before the end of the decade. Or else outliers alone might be enough to keep us from getting there.","locator":"paragraph 17","document":{"url":"https://garymarcus.substack.com/p/dear-elon-musk-here-are-five-things","title":"Dear Elon Musk, here are five things you might want to consider about AGI","publisher":"Marcus on AI","published_at":"2022-05-31T18:32:41Z","date_uncertain":false,"date_provenance":"JSON-LD datePublished"},"observed_at":"2026-10-28T22:49:04Z","observed_via":"live","archive_url":null,"access_state":"full_public","snapshot_hash":"d64c073bb595b86ed0d5a52382668c601d5a8e7009353d00f0e54061bdd4f0f7","receipts":1}],"first_statement_date":"2022-05-31","related":[],"current":{"assessment_id":"marcus-dear-elon-musk-here-are-five-things-1c4f0d-a1","payload":{"claim_id":"marcus-dear-elon-musk-here-are-five-things-1c4f0d","subject_id":"gary-marcus","normalized_proposition":"Unless the field invests in ideas beyond current techniques like deep learning, outliers alone may prevent reaching AGI before the end of the decade.","kind":"conditional_forecast","conditions":["unless the field invests in ideas beyond deep learning"],"original_horizon_text":"before the end of the decade","statement":{"excerpt":"Seven years is a long time, but the field is going to need to invest in other ideas if we are going to get to AGI before the end of the decade. Or else outliers alone might be enough to keep us from getting there.","locator":"paragraph 17","url":"https://garymarcus.substack.com/p/dear-elon-musk-here-are-five-things","date":"2022-05-31","document_title":"Dear Elon Musk, here are five things you might want to consider about AGI","observed_at":"2026-10-28T22:49:04Z"},"criteria":{"from_speaker":["unless the field invests in ideas beyond deep learning","before the end of the decade"],"our_interpretation":[],"support_condition":"Not checked against evidence: both screens found that public evidence could not settle this claim.","contradiction_condition":"Not checked against evidence: both screens found that public evidence could not settle this claim."},"axis":"not_applicable","outcome":"not_adjudicable","horizon_state":"not_applicable","evaluation_as_of":"2026-10-28","rationale":"Not checked against evidence. Two independent screens, one from each provider, both found this stated as a chance or a possibility, or at medium or low confidence.","limitations":[],"evidence":[],"what_would_resolve":null,"retrospective":true,"uncheckable":["probability_language"]},"payload_hash":"f25705cbfa26f7dc3058b5b9ef1c2f4c146722358cd7dd8ef24a02e9df83f60f","policy_version":"2026-10-28.7","recorded_at":"2026-10-29T01:10:54Z","drafter":{"provider":"anthropic","model":"claude-opus-5-5"},"reviewers":[{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-dear-elon-musk-here-are-five-things-screen-anthropic-1"},{"provider":"openai","model":"gpt-6-astra","run_id":"marcus-dear-elon-musk-here-are-five-things-screen-openai-1"}],"final_text":null,"status":"current","notice":null,"superseded_by":null,"evidence_meta":[],"public_from":null,"public_until":null},"history":[],"review":{"review_due":false,"triggers":[],"last_completed_review_at":"2026-10-29T01:10:54Z","next_review_due_at":null,"horizon_now":"not_applicable"},"event_ids":["marcus-dear-elon-musk-here-are-five-things-1c4f0d-assessment-published-a1"],"last_substantive_change_at":"2026-10-29T17:00:18Z"}},{"url":"https://punditclaims.ai/claims/marcus-dear-elon-musk-here-are-five-things-2a322d","standing":"current","assessment_label":"Can’t be checked","versions":[{"assessment_id":"marcus-dear-elon-musk-here-are-five-things-2a322d-a1","status":"current","url":"https://punditclaims.ai/claims/marcus-dear-elon-musk-here-are-five-things-2a322d#assessment-marcus-dear-elon-musk-here-are-five-things-2a322d-a1"}],"claim":{"claim_id":"marcus-dear-elon-musk-here-are-five-things-2a322d","subject_id":"gary-marcus","revision":2,"content_hash":"cf39669098791bbd2823ee296b1cc635551d67eb66aab23f56973423b2fa0b00","normalized_proposition":"Unexpected circumstances have been and continue to be the bane of contemporary AI techniques, and probably will remain so until there is a real revolution.","kind":"forecast","target":"contemporary AI techniques","conditions":[{"text":"until there is a real revolution","type":"qualifier","from_source":true},{"text":"probably","type":"qualifier","from_source":true}],"original_horizon_text":null,"topic_tags":["reliability"],"concerns_providers":[],"selection_stratum":"declared-lead","occurrences":[{"id":"marcus-dear-elon-musk-here-are-five-things-2a322d-o1","statement_date":"2022-05-31","statement_date_precision":"day","excerpt":"Unexpected circumstances have been and continue to be the bane of contemporary AI techniques, and probably will be until there is a real revolution.","locator":"paragraph 16","document":{"url":"https://garymarcus.substack.com/p/dear-elon-musk-here-are-five-things","title":"Dear Elon Musk, here are five things you might want to consider about AGI","publisher":"Marcus on AI","published_at":"2022-05-31T18:32:41Z","date_uncertain":false,"date_provenance":"JSON-LD datePublished"},"observed_at":"2026-10-28T22:49:04Z","observed_via":"live","archive_url":null,"access_state":"full_public","snapshot_hash":"d64c073bb595b86ed0d5a52382668c601d5a8e7009353d00f0e54061bdd4f0f7","receipts":1}],"first_statement_date":"2022-05-31","related":[],"current":{"assessment_id":"marcus-dear-elon-musk-here-are-five-things-2a322d-a1","payload":{"claim_id":"marcus-dear-elon-musk-here-are-five-things-2a322d","subject_id":"gary-marcus","normalized_proposition":"Unexpected circumstances have been and continue to be the bane of contemporary AI techniques, and probably will remain so until there is a real revolution.","kind":"forecast","conditions":["until there is a real revolution","probably"],"original_horizon_text":null,"statement":{"excerpt":"Unexpected circumstances have been and continue to be the bane of contemporary AI techniques, and probably will be until there is a real revolution.","locator":"paragraph 16","url":"https://garymarcus.substack.com/p/dear-elon-musk-here-are-five-things","date":"2022-05-31","document_title":"Dear Elon Musk, here are five things you might want to consider about AGI","observed_at":"2026-10-28T22:49:04Z"},"criteria":{"from_speaker":["until there is a real revolution","probably"],"our_interpretation":[],"support_condition":"Not checked against evidence: both screens found that public evidence could not settle this claim.","contradiction_condition":"Not checked against evidence: both screens found that public evidence could not settle this claim."},"axis":"not_applicable","outcome":"not_adjudicable","horizon_state":"not_applicable","evaluation_as_of":"2026-10-28","rationale":"Not checked against evidence. Two independent screens, one from each provider, both found this stated as a chance or a possibility, or at medium or low confidence.","limitations":[],"evidence":[],"what_would_resolve":null,"retrospective":true,"uncheckable":["probability_language"]},"payload_hash":"484ca8b6683af4cf85ce9288aeec2b87fb6aa20cc4d763276e200d4880b140f1","policy_version":"2026-10-28.7","recorded_at":"2026-10-29T02:32:05Z","drafter":{"provider":"anthropic","model":"claude-opus-5-5"},"reviewers":[{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-dear-elon-musk-here-are-five-things-screen-anthropic-2"},{"provider":"openai","model":"gpt-6-astra","run_id":"marcus-dear-elon-musk-here-are-five-things-screen-openai-2"}],"final_text":null,"status":"current","notice":null,"superseded_by":null,"evidence_meta":[],"public_from":null,"public_until":null},"history":[],"review":{"review_due":false,"triggers":[],"last_completed_review_at":"2026-10-29T02:32:05Z","next_review_due_at":null,"horizon_now":"not_applicable"},"event_ids":["marcus-dear-elon-musk-here-are-five-things-2a322d-assessment-published-a1"],"last_substantive_change_at":"2026-10-29T17:00:18Z"}},{"url":"https://punditclaims.ai/claims/marcus-dear-elon-musk-here-are-five-things-2c8978","standing":"current","assessment_label":"Unresolved","versions":[{"assessment_id":"marcus-dear-elon-musk-here-are-five-things-2c8978-a1","status":"current","url":"https://punditclaims.ai/claims/marcus-dear-elon-musk-here-are-five-things-2c8978#assessment-marcus-dear-elon-musk-here-are-five-things-2c8978-a1"}],"claim":{"claim_id":"marcus-dear-elon-musk-here-are-five-things-2c8978","subject_id":"gary-marcus","revision":2,"content_hash":"aa8cb29f940f07b70bf5967ec9689abe3f0c01250a82ce544668581224d29026","normalized_proposition":"The AGI needed for a general-purpose domestic robot, where every home is different and poses its own safety risks, is way beyond what is needed for a car driving on roads engineered more or less the same from town to town.","kind":"observation","target":"general-purpose domestic robots (Tesla Optimus)","conditions":[],"original_horizon_text":null,"topic_tags":["capabilities","agi-timelines"],"concerns_providers":[],"selection_stratum":"declared-lead","occurrences":[{"id":"marcus-dear-elon-musk-here-are-five-things-2c8978-o1","statement_date":"2022-05-31","statement_date_precision":"day","excerpt":"I don’t know what you have planned for Optimus, but I can promise you that the AGI that you would need for a general-purpose domestic robot (where every home is different, and each poses its own safety risks) is way beyond what you would need for a car that drives on roads that are more or less engineered the same way from one town to the next.","locator":"paragraph 26","document":{"url":"https://garymarcus.substack.com/p/dear-elon-musk-here-are-five-things","title":"Dear Elon Musk, here are five things you might want to consider about AGI","publisher":"Marcus on AI","published_at":"2022-05-31T18:32:41Z","date_uncertain":false,"date_provenance":"JSON-LD datePublished"},"observed_at":"2026-10-28T22:49:04Z","observed_via":"live","archive_url":null,"access_state":"full_public","snapshot_hash":"d64c073bb595b86ed0d5a52382668c601d5a8e7009353d00f0e54061bdd4f0f7","receipts":1}],"first_statement_date":"2022-05-31","related":[],"current":{"assessment_id":"marcus-dear-elon-musk-here-are-five-things-2c8978-a1","payload":{"claim_id":"marcus-dear-elon-musk-here-are-five-things-2c8978","subject_id":"gary-marcus","normalized_proposition":"The AGI needed for a general-purpose domestic robot, where every home is different and poses its own safety risks, is way beyond what is needed for a car driving on roads engineered more or less the same from town to town.","kind":"observation","conditions":[],"original_horizon_text":null,"statement":{"excerpt":"I don’t know what you have planned for Optimus, but I can promise you that the AGI that you would need for a general-purpose domestic robot (where every home is different, and each poses its own safety risks) is way beyond what you would need for a car that drives on roads that are more or less engineered the same way from one town to the next.","locator":"paragraph 26","url":"https://garymarcus.substack.com/p/dear-elon-musk-here-are-five-things","date":"2022-05-31","document_title":"Dear Elon Musk, here are five things you might want to consider about AGI","observed_at":"2026-10-28T22:49:04Z"},"criteria":{"from_speaker":["general-purpose domestic robot","every home is different","each poses its own safety risks","way beyond","a car that drives on roads"],"our_interpretation":["The ordinary reading is that safe, general domestic assistance requires substantially broader adaptable cognitive capabilities than autonomous road driving.","This compares required capabilities, not market adoption, commercial success or release dates.","The claim describes the state of knowledge at publication. With no reporting cycle for this research field, the current observation window is calendar 2022 through May 31; earlier studies are context.","The driving comparator is autonomous road driving as described, not merely one fixed route or a specially restricted demonstration."],"support_condition":"Comparative task analyses and capability evaluations establish substantially broader cognitive requirements for safe general domestic work than for autonomous road driving.","contradiction_condition":"Comparable analyses or demonstrations show that general domestic work needs no substantially broader cognitive capabilities, or that driving requires greater generality under comparable assumptions."},"axis":"historical_observation","outcome":"unresolved","horizon_state":"not_applicable","evaluation_as_of":"2026-10-28","rationale":"The sources make the comparison plausible: household assistance spans many activities, while driving skills can transfer between locations. They also show meaningful household-robot progress and substantial complexity in driving. None directly establishes that the intelligence required for general domestic work is 'way beyond' that required for road driving. Deployment differences would be an insufficient substitute for that comparison.","limitations":["Household robotics evidence includes simulation and constrained kitchen tasks.","Waymo's account is a company disclosure, not an independent comparison with domestic robotics.","Task breadth and overall difficulty are related but distinct."],"evidence":[{"id":"e1","relation":"context","excerpt":"demand generalizable solutions.","locator":"Section 3, Diversity in BEHAVIOR Activities","url":"https://arxiv.org/html/2108.03332v1","publisher":"arXiv; Sanjana Srivastava and colleagues","event_date":"2021","available_at":"2021-08-06","explanation":"The benchmark spans varied household activities, objects and environments, supporting the premise of broad domestic requirements. Its experiments predate the current observation window and do not directly compare cognitive requirements with driving."},{"id":"e2","relation":"context","excerpt":"improves their robustness","locator":"Section 7, Conclusions, Limitations and Future Work","url":"https://arxiv.org/html/2204.01691v1","publisher":"arXiv; Michael Ahn and colleagues","event_date":"2022-04","available_at":"2022-04-04","explanation":"SayCan demonstrates language-guided sequences of real kitchen tasks while identifying limited skills and failure recovery as obstacles. It shows both progress and remaining difficulty, but does not establish the comparative amount of intelligence required."},{"id":"e3","relation":"context","excerpt":"fundamentals of driving","locator":"Section Using Machine Learning to Build a Generalizable System","url":"https://waymo.com/blog/2022/05/howwevebuiltascalableautonomousdriver","publisher":"Waymo","event_date":"2022-05","available_at":"2022-05-18","explanation":"Waymo describes transferable driving skills and autonomous rides, consistent with Marcus's premise of shared road-driving requirements. The same disclosure describes rare events, local rules and climate differences, limiting any inference that driving environments are simple or uniform."}],"what_would_resolve":"A comparative requirements analysis or controlled capability study addressing general domestic work and autonomous driving with explicit operating scope and comparable reliability requirements.","retrospective":true,"plain_reading":true},"payload_hash":"4f07eed9ae188c8bc238caaa2d347700f19ce765c1b66c0870e801619a985d79","policy_version":"2026-10-28.7","recorded_at":"2026-10-29T06:55:52Z","drafter":{"provider":"openai","model":"gpt-6-astra"},"reviewers":[{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-dear-elon-musk-here-are-five-things-056704-4x-review-anthropic-1"},{"provider":"openai","model":"gpt-6-astra","run_id":"marcus-dear-elon-musk-here-are-five-things-056704-4x-review-openai-1"}],"final_text":{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-dear-elon-musk-here-are-five-things-056704-4x-final-anthropic-1"},"status":"current","notice":null,"superseded_by":null,"evidence_meta":[{"id":"e1","source_type":"preprint","audited":null,"derived":null,"metric":null},{"id":"e2","source_type":"preprint","audited":null,"derived":null,"metric":null},{"id":"e3","source_type":"company_disclosure","audited":null,"derived":null,"metric":null}],"public_from":null,"public_until":null},"history":[],"review":{"review_due":false,"triggers":[],"last_completed_review_at":"2026-10-29T06:55:52Z","next_review_due_at":null,"horizon_now":"not_applicable"},"event_ids":["marcus-dear-elon-musk-here-are-five-things-2c8978-assessment-published-a1"],"last_substantive_change_at":"2026-10-29T17:00:18Z"}},{"url":"https://punditclaims.ai/claims/marcus-dear-elon-musk-here-are-five-things-31b9db","standing":"current","assessment_label":"Unresolved","versions":[{"assessment_id":"marcus-dear-elon-musk-here-are-five-things-31b9db-a2","status":"current","url":"https://punditclaims.ai/claims/marcus-dear-elon-musk-here-are-five-things-31b9db#assessment-marcus-dear-elon-musk-here-are-five-things-31b9db-a2"}],"claim":{"claim_id":"marcus-dear-elon-musk-here-are-five-things-31b9db","subject_id":"gary-marcus","revision":2,"content_hash":"a7a43d09b5d8179e23fe50996567894a36bc2bafb1eb7c2eb7c39f40d5468025","normalized_proposition":"In 2029, AI will not be able to read a novel and reliably answer questions about plot, character, conflicts, motivations, etc., where the key is going beyond the literal text.","kind":"forecast","target":"AI","conditions":[{"text":"key will be going beyond the literal text","type":"qualifier","from_source":true}],"original_horizon_text":"In 2029","topic_tags":["capabilities","agi-timelines"],"concerns_providers":[],"selection_stratum":"declared-lead","occurrences":[{"id":"marcus-dear-elon-musk-here-are-five-things-31b9db-o1","statement_date":"2022-05-31","statement_date_precision":"day","excerpt":"In 2029, AI will not be able to read a novel and reliably answer questions about plot, character, conflicts, motivations, etc.","locator":"prediction 2 of 5","document":{"url":"https://garymarcus.substack.com/p/dear-elon-musk-here-are-five-things","title":"Dear Elon Musk, here are five things you might want to consider about AGI","publisher":"Marcus on AI","published_at":"2022-05-31T18:32:41Z","date_uncertain":false,"date_provenance":"JSON-LD datePublished"},"observed_at":"2026-10-28T22:49:04Z","observed_via":"live","archive_url":null,"access_state":"full_public","snapshot_hash":"d64c073bb595b86ed0d5a52382668c601d5a8e7009353d00f0e54061bdd4f0f7","receipts":1}],"first_statement_date":"2022-05-31","related":[],"current":{"assessment_id":"marcus-dear-elon-musk-here-are-five-things-31b9db-a2","payload":{"claim_id":"marcus-dear-elon-musk-here-are-five-things-31b9db","subject_id":"gary-marcus","normalized_proposition":"In 2029, AI will not be able to read a novel and reliably answer questions about plot, character, conflicts, motivations, etc., where the key is going beyond the literal text.","kind":"forecast","conditions":["key will be going beyond the literal text"],"original_horizon_text":"In 2029","statement":{"excerpt":"In 2029, AI will not be able to read a novel and reliably answer questions about plot, character, conflicts, motivations, etc.","locator":"prediction 2 of 5","url":"https://garymarcus.substack.com/p/dear-elon-musk-here-are-five-things","date":"2022-05-31","document_title":"Dear Elon Musk, here are five things you might want to consider about AGI","observed_at":"2026-10-28T22:49:04Z"},"criteria":{"from_speaker":["In 2029","read a novel","reliably answer questions about plot, character, conflicts, motivations, etc.","going beyond the literal text"],"our_interpretation":["The ordinary reading is dependable comprehension of whole novels, including reasonable inferences about motives, relationships and conflicts.","Going beyond the literal text defines the task. It is not an external antecedent whose occurrence must be established separately.","Test varied novels and questions with controls for memorized summaries. Reliability does not require agreement on every debatable literary interpretation.","Use 2029-12-31 as the latest deadline, allowing an earlier reproducible demonstration of the complete capability to refute the forecast.","Assess this individual forecast independently of the surrounding multi-challenge wager."],"support_condition":"Evidence covering the end of 2029 shows that leading available systems still fail to answer questions dependably about whole novels, including questions requiring inference about plot, characters, conflicts and motivations.","contradiction_condition":"At least one AI system reliably answers questions across varied whole novels, including nonliteral questions, under a credible evaluation with human-validated answers."},"axis":"forecast_outcome","outcome":"unresolved","horizon_state":"not_due","evaluation_as_of":"2026-10-28","rationale":"The August 2025 PRELUDE study reports that tested systems lagged behind humans when judging whether character backstories fit a novel [e1]. The September 2026 detective-novel study tests one local 9B model on multiple-choice questions and reports limited accuracy [e2]. Neither establishes the complete, independently assessed and reproducible whole-novel capability required for early contradiction, and these earlier results cannot establish failure in 2029 [e1; e2].","limitations":["The studies do not establish current frontier performance across the complete task or what systems will achieve in 2029.","The packet contains no verified Fiction.LiveBench results; their relevance to early resolution remains unassessed.","The claim supplies no numerical reliability threshold, and literary questions can admit multiple defensible answers."],"evidence":[{"id":"e1","relation":"context","excerpt":"room for improvement","locator":"Abstract, final sentence","url":"https://arxiv.org/abs/2508.09848v2","publisher":"arXiv; Mo Yu and collaborators","event_date":"2025","available_at":"2025-08-14","explanation":"PRELUDE tests whether proposed character backstories fit a novel, requiring indirect evidence and inference. Its reported human-model gaps concern a demanding related task and do not settle the broader 2029 forecast."},{"id":"e2","relation":"context","excerpt":"the graph route reaches 42.86%.","locator":"Abstract, results discussion","url":"https://arxiv.org/abs/2609.22939","publisher":"arXiv; Wenji Fu","event_date":"2026-09","available_at":"2026-09-19","explanation":"This study evaluates a fixed local model on detective-novel questions and reports limited performance. It neither establishes general reliable comprehension nor measures the strongest available systems."}],"what_would_resolve":"A dated, independently assessed and reproducible evaluation demonstrating reliable answers across varied whole novels, including nonliteral questions, would contradict the forecast early. It must supply the novels, disclose outputs and grading, use human-validated answers, accept text-supported alternative interpretations, and control for prior familiarity and answer leakage. Otherwise, evidence covering leading systems through December 31, 2029 would be needed to assess whether dependable comprehension remains beyond them.","retrospective":false,"plain_reading":true},"payload_hash":"d20454339b0bea4567351aff39ce1801ed0d45f618aa44ca8a083e5d43bc7909","policy_version":"2026-10-28.7","recorded_at":"2026-10-29T07:09:09Z","drafter":{"provider":"openai","model":"gpt-6-astra"},"reviewers":[{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-dear-elon-musk-here-are-five-things-31b9db-4x-review-anthropic-1"},{"provider":"openai","model":"gpt-6-astra","run_id":"marcus-dear-elon-musk-here-are-five-things-31b9db-4x-review-openai-1"}],"final_text":{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-dear-elon-musk-here-are-five-things-31b9db-2x-final-anthropic-1"},"status":"current","notice":null,"superseded_by":null,"evidence_meta":[{"id":"e1","source_type":"preprint","audited":null,"derived":null,"metric":null},{"id":"e2","source_type":"preprint","audited":null,"derived":null,"metric":null}],"public_from":null,"public_until":null},"history":[],"review":{"review_due":false,"triggers":[],"last_completed_review_at":"2026-10-29T07:09:09Z","next_review_due_at":null,"horizon_now":"not_due"},"event_ids":["marcus-dear-elon-musk-here-are-five-things-31b9db-assessment-published-a2"],"last_substantive_change_at":"2026-10-29T17:00:18Z"}},{"url":"https://punditclaims.ai/claims/marcus-dear-elon-musk-here-are-five-things-536f05","standing":"current","assessment_label":"Unresolved","versions":[{"assessment_id":"marcus-dear-elon-musk-here-are-five-things-536f05-a1","status":"current","url":"https://punditclaims.ai/claims/marcus-dear-elon-musk-here-are-five-things-536f05#assessment-marcus-dear-elon-musk-here-are-five-things-536f05-a1"}],"claim":{"claim_id":"marcus-dear-elon-musk-here-are-five-things-536f05","subject_id":"gary-marcus","revision":1,"content_hash":"904472ff23993c9299e4a8224f7574865f5b60fb630d229d48e335a4647df2fe","normalized_proposition":"AI has no stable or trustworthy solutions for common sense, reasoning, language or analogy, and this has scarcely changed in about five years.","kind":"observation","target":"current AI","conditions":[],"original_horizon_text":null,"topic_tags":["capabilities","reliability"],"concerns_providers":[],"selection_stratum":"declared-lead","occurrences":[{"id":"marcus-dear-elon-musk-here-are-five-things-536f05-o1","statement_date":"2022-05-31","statement_date_precision":"day","excerpt":"We still don’t have anything like stable or trustworthy solutions for common sense, reasoning, language, or analogy. Truth is, I have been using this same pie chart for about 5 years, and the situation has scarcely changed.","locator":"paragraph 21","document":{"url":"https://garymarcus.substack.com/p/dear-elon-musk-here-are-five-things","title":"Dear Elon Musk, here are five things you might want to consider about AGI","publisher":"Marcus on AI","published_at":"2022-05-31T18:32:41Z","date_uncertain":false,"date_provenance":"JSON-LD datePublished"},"observed_at":"2026-10-28T22:49:04Z","observed_via":"live","archive_url":null,"access_state":"full_public","snapshot_hash":"d64c073bb595b86ed0d5a52382668c601d5a8e7009353d00f0e54061bdd4f0f7","receipts":1}],"first_statement_date":"2022-05-31","related":[],"current":{"assessment_id":"marcus-dear-elon-musk-here-are-five-things-536f05-a1","payload":{"claim_id":"marcus-dear-elon-musk-here-are-five-things-536f05","subject_id":"gary-marcus","normalized_proposition":"AI has no stable or trustworthy solutions for common sense, reasoning, language or analogy, and this has scarcely changed in about five years.","kind":"observation","conditions":[],"original_horizon_text":null,"statement":{"excerpt":"We still don’t have anything like stable or trustworthy solutions for common sense, reasoning, language, or analogy. Truth is, I have been using this same pie chart for about 5 years, and the situation has scarcely changed.","locator":"paragraph 21","url":"https://garymarcus.substack.com/p/dear-elon-musk-here-are-five-things","date":"2022-05-31","document_title":"Dear Elon Musk, here are five things you might want to consider about AGI","observed_at":"2026-10-28T22:49:04Z"},"criteria":{"from_speaker":["stable or trustworthy solutions","common sense, reasoning, language, or analogy","about 5 years","the situation has scarcely changed"],"our_interpretation":["A trustworthy solution means dependable performance across varied, unfamiliar instances of a domain, rather than success on one benchmark. It does not require perfect performance.","The comparison covers approximately 2017 through May 2022. Calendar 2017 is the approximate baseline; the endpoint is the post date.","In context, 'scarcely changed' concerns the availability of dependable general solutions. It does not necessarily deny substantial improvements in benchmark scores."],"support_condition":"Contemporary evaluations show persistent, substantial reliability limitations across the four domains, and comparable longitudinal evidence shows little change in the availability of dependable general solutions.","contradiction_condition":"A dependable general solution existed in a named domain by May 2022, or comparable evaluations demonstrate substantial improvement in broad reliability since approximately 2017."},"axis":"historical_observation","outcome":"unresolved","horizon_state":"not_applicable","evaluation_as_of":"2026-10-28","rationale":"Contemporary research supports persistent weaknesses in language, commonsense reasoning and analogy. PaLM also documents substantial benchmark progress before the post. Those findings can coexist: improved scores do not establish trustworthy general solutions. The evidence does not settle the combined claim that dependable solutions were absent across all four domains and that this situation had scarcely changed over five years.","limitations":["The studies use different tasks, models and evaluation methods.","Scoring the claim as mixed solely because benchmark scores improved would substitute a different meaning of progress."],"evidence":[{"id":"e1","relation":"supports","excerpt":"perform poorly under adversarial evaluation.","locator":"Section 5, Discussion and Related Work","url":"https://arxiv.org/html/1707.07328v1","publisher":"arXiv; Robin Jia and Percy Liang","event_date":"2017","available_at":"2017-07-23","explanation":"The experiments establish substantial reading-comprehension brittleness near the five-year baseline. They do not measure subsequent improvement."},{"id":"e2","relation":"supports","excerpt":"word embedding models.","locator":"Abstract","url":"https://arxiv.org/abs/2105.04949v1","publisher":"arXiv; Asahi Ushio and colleagues","event_date":"2021","available_at":"2021-05-11","explanation":"The study reports difficulty with abstract analogies and sensitivity to architecture and settings. This supports a limited finding of unreliable analogy performance among the tested models, not the absence of every possible solution in May 2022."},{"id":"e3","relation":"supports","excerpt":"partially understood","locator":"Discussion, Exploring new components of commonsense knowledge","url":"https://arxiv.org/html/2201.12438v1","publisher":"arXiv; Prajjwal Bhargava and Vincent Ng","event_date":"2022-01","available_at":"2022-01-28","explanation":"This contemporary survey identifies unaddressed commonsense capabilities and weaknesses in existing evaluations. It supports continued incompleteness but supplies no common five-year measure of reliability."},{"id":"e4","relation":"context","excerpt":"language understanding and generation benchmarks.","locator":"Abstract","url":"https://arxiv.org/abs/2204.02311v1","publisher":"arXiv; Aakanksha Chowdhery and colleagues","event_date":"2022-04","available_at":"2022-04-05","explanation":"PaLM supplies material counterevidence to a reading that little capability progress occurred. These benchmark gains do not by themselves establish dependable general solutions under the contract."}],"what_would_resolve":"Comparable historical robustness evaluations across the four domains, together with a defensible threshold for dependable performance and meaningful five-year change.","retrospective":true,"plain_reading":true},"payload_hash":"3ee8bb0ead86ed2168d2ad0642f8cae62bc3aa16300b14b0a22b5ce884ad64b9","policy_version":"2026-10-28.7","recorded_at":"2026-10-29T06:55:56Z","drafter":{"provider":"openai","model":"gpt-6-astra"},"reviewers":[{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-dear-elon-musk-here-are-five-things-31b9db-4x-review-anthropic-1"},{"provider":"openai","model":"gpt-6-astra","run_id":"marcus-dear-elon-musk-here-are-five-things-31b9db-4x-review-openai-1"}],"final_text":{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-dear-elon-musk-here-are-five-things-31b9db-4x-final-anthropic-1"},"status":"current","notice":null,"superseded_by":null,"evidence_meta":[{"id":"e1","source_type":"preprint","audited":null,"derived":null,"metric":null},{"id":"e2","source_type":"preprint","audited":null,"derived":null,"metric":null},{"id":"e3","source_type":"preprint","audited":null,"derived":null,"metric":null},{"id":"e4","source_type":"preprint","audited":null,"derived":null,"metric":null}],"public_from":null,"public_until":null},"history":[],"review":{"review_due":false,"triggers":[],"last_completed_review_at":"2026-10-29T06:55:56Z","next_review_due_at":null,"horizon_now":"not_applicable"},"event_ids":["marcus-dear-elon-musk-here-are-five-things-536f05-assessment-published-a1"],"last_substantive_change_at":"2026-10-29T17:00:18Z"}},{"url":"https://punditclaims.ai/claims/marcus-dear-elon-musk-here-are-five-things-5dc69d","standing":"current","assessment_label":"Unresolved","versions":[{"assessment_id":"marcus-dear-elon-musk-here-are-five-things-5dc69d-a1","status":"current","url":"https://punditclaims.ai/claims/marcus-dear-elon-musk-here-are-five-things-5dc69d#assessment-marcus-dear-elon-musk-here-are-five-things-5dc69d-a1"}],"claim":{"claim_id":"marcus-dear-elon-musk-here-are-five-things-5dc69d","subject_id":"gary-marcus","revision":1,"content_hash":"1f7efb5811b3455c2c1bdf3235c768ef4ba3d38aef9ea59260ee156cc5868046","normalized_proposition":"Deep learning is decent at object recognition but much worse at planning, reading and language comprehension.","kind":"observation","target":"deep learning","conditions":[],"original_horizon_text":null,"topic_tags":["capabilities"],"concerns_providers":[],"selection_stratum":"declared-lead","occurrences":[{"id":"marcus-dear-elon-musk-here-are-five-things-5dc69d-o1","statement_date":"2022-05-31","statement_date_precision":"day","excerpt":"Deep learning is pretty decent at, for example, recognizing objects, but not nearly as good at planning, or reading, or language comprehension.","locator":"paragraph 20","document":{"url":"https://garymarcus.substack.com/p/dear-elon-musk-here-are-five-things","title":"Dear Elon Musk, here are five things you might want to consider about AGI","publisher":"Marcus on AI","published_at":"2022-05-31T18:32:41Z","date_uncertain":false,"date_provenance":"JSON-LD datePublished"},"observed_at":"2026-10-28T22:49:04Z","observed_via":"live","archive_url":null,"access_state":"full_public","snapshot_hash":"d64c073bb595b86ed0d5a52382668c601d5a8e7009353d00f0e54061bdd4f0f7","receipts":1}],"first_statement_date":"2022-05-31","related":[],"current":{"assessment_id":"marcus-dear-elon-musk-here-are-five-things-5dc69d-a1","payload":{"claim_id":"marcus-dear-elon-musk-here-are-five-things-5dc69d","subject_id":"gary-marcus","normalized_proposition":"Deep learning is decent at object recognition but much worse at planning, reading and language comprehension.","kind":"observation","conditions":[],"original_horizon_text":null,"statement":{"excerpt":"Deep learning is pretty decent at, for example, recognizing objects, but not nearly as good at planning, or reading, or language comprehension.","locator":"paragraph 20","url":"https://garymarcus.substack.com/p/dear-elon-musk-here-are-five-things","date":"2022-05-31","document_title":"Dear Elon Musk, here are five things you might want to consider about AGI","observed_at":"2026-10-28T22:49:04Z"},"criteria":{"from_speaker":["Deep learning","pretty decent at","recognizing objects","not nearly as good at planning, or reading, or language comprehension"],"our_interpretation":["The ordinary reading compares the maturity and reliability of deep learning across the named abilities, rather than comparing unrelated benchmark percentages.","Reading is understood as understanding written material, given its pairing with language comprehension.","Human-relative task reliability is a useful comparison standard, but Marcus supplies neither this measure nor a required size of gap.","The observation concerns the current 2022 research record through May 31; academic research has no fixed reporting quarter."],"support_condition":"Representative, reasonably comparable evaluations establish useful object recognition and substantially weaker reliability in planning, reading, and language comprehension.","contradiction_condition":"Representative, reasonably comparable evaluations establish that the claimed broad disadvantage in the other abilities does not hold."},"axis":"historical_observation","outcome":"unresolved","horizon_state":"not_applicable","evaluation_as_of":"2026-10-28","rationale":"The evidence establishes strong object recognition and real planning limitations. It also establishes strong language benchmark performance before the post. These findings do not settle the comparative claim: the tasks, metrics, and evaluation conditions differ. Calling the whole statement supported or contradicted would depend too heavily on which tasks the evaluator selected.","limitations":["Benchmark success does not establish broad real-world comprehension.","A planning failure does not by itself establish that planning was substantially worse than recognition.","The evidence supports individual observations without resolving the claimed ranking."],"evidence":[{"id":"e1","relation":"context","excerpt":"91.0% on ImageNet","locator":"Section 1, Introduction, page 2","url":"https://arxiv.org/pdf/2205.01917v1","publisher":"arXiv; Jiahui Yu and colleagues","event_date":"2022-05-04","available_at":"2022-05-04","explanation":"This supports the component claim that deep learning was effective at object recognition. The different task scores do not establish a common scale for comparing recognition with planning or comprehension."},{"id":"e2","relation":"context","excerpt":"admissible actions","locator":"Abstract","url":"https://arxiv.org/abs/2201.07207v1","publisher":"arXiv; Wenlong Huang and colleagues","event_date":"2022-01-18","available_at":"2022-01-18","explanation":"The study documents a practical planning limitation while also finding useful planning ability and improvements from translating plans into admissible actions. It does not compare planning reliability with object recognition."},{"id":"e3","relation":"context","excerpt":"language understanding","locator":"Abstract","url":"https://arxiv.org/abs/2204.02311v1","publisher":"arXiv; Aakanksha Chowdhery and colleagues","event_date":"2022-04-05","available_at":"2022-04-05","explanation":"PaLM reports strong language results, including performance above the average human result on its BIG-bench evaluation. This challenges a blanket characterization of weak language ability but does not supply the required comparison with object recognition."}],"what_would_resolve":"A representative evaluation of systems available by May 2022 using a defensible common standard across all four abilities, with an explicit treatment of task difficulty and generalization.","retrospective":true,"plain_reading":true},"payload_hash":"20800551b7adca7e351869cfea55ca3f86eaa660d8274b8afc7a675a32d2e2b7","policy_version":"2026-10-28.7","recorded_at":"2026-10-29T06:56:00Z","drafter":{"provider":"openai","model":"gpt-6-astra"},"reviewers":[{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-dear-elon-musk-here-are-five-things-5dc69d-4x-review-anthropic-1"},{"provider":"openai","model":"gpt-6-astra","run_id":"marcus-dear-elon-musk-here-are-five-things-5dc69d-4x-review-openai-1"}],"final_text":{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-dear-elon-musk-here-are-five-things-5dc69d-4x-final-anthropic-1"},"status":"current","notice":null,"superseded_by":null,"evidence_meta":[{"id":"e1","source_type":"preprint","audited":null,"derived":null,"metric":null},{"id":"e2","source_type":"preprint","audited":null,"derived":null,"metric":null},{"id":"e3","source_type":"preprint","audited":null,"derived":null,"metric":null}],"public_from":null,"public_until":null},"history":[],"review":{"review_due":false,"triggers":[],"last_completed_review_at":"2026-10-29T06:56:00Z","next_review_due_at":null,"horizon_now":"not_applicable"},"event_ids":["marcus-dear-elon-musk-here-are-five-things-5dc69d-assessment-published-a1"],"last_substantive_change_at":"2026-10-29T17:00:18Z"}},{"url":"https://punditclaims.ai/claims/marcus-dear-elon-musk-here-are-five-things-720a9f","standing":"current","assessment_label":"Unresolved","versions":[{"assessment_id":"marcus-dear-elon-musk-here-are-five-things-720a9f-a1","status":"current","url":"https://punditclaims.ai/claims/marcus-dear-elon-musk-here-are-five-things-720a9f#assessment-marcus-dear-elon-musk-here-are-five-things-720a9f-a1"}],"claim":{"claim_id":"marcus-dear-elon-musk-here-are-five-things-720a9f","subject_id":"gary-marcus","revision":2,"content_hash":"0beaa7fcdd33ecfe7f808a708664406fa65964d36531554eef2afe46791f8d96","normalized_proposition":"In 2029, AI will not be able to take arbitrary proofs written in natural language from the mathematical literature and convert them into a symbolic form suitable for symbolic verification.","kind":"forecast","target":"AI","conditions":[{"text":"from the mathematical literature","type":"scope","from_source":true}],"original_horizon_text":"In 2029","topic_tags":["capabilities","agi-timelines"],"concerns_providers":[],"selection_stratum":"declared-lead","occurrences":[{"id":"marcus-dear-elon-musk-here-are-five-things-720a9f-o1","statement_date":"2022-05-31","statement_date_precision":"day","excerpt":"In 2029, AI will not be able to take arbitrary proofs from the mathematical literature written in natural language and convert them into a symbolic form suitable for symbolic verification.","locator":"prediction 5 of 5","document":{"url":"https://garymarcus.substack.com/p/dear-elon-musk-here-are-five-things","title":"Dear Elon Musk, here are five things you might want to consider about AGI","publisher":"Marcus on AI","published_at":"2022-05-31T18:32:41Z","date_uncertain":false,"date_provenance":"JSON-LD datePublished"},"observed_at":"2026-10-28T22:49:04Z","observed_via":"live","archive_url":null,"access_state":"full_public","snapshot_hash":"d64c073bb595b86ed0d5a52382668c601d5a8e7009353d00f0e54061bdd4f0f7","receipts":1}],"first_statement_date":"2022-05-31","related":[],"current":{"assessment_id":"marcus-dear-elon-musk-here-are-five-things-720a9f-a1","payload":{"claim_id":"marcus-dear-elon-musk-here-are-five-things-720a9f","subject_id":"gary-marcus","normalized_proposition":"In 2029, AI will not be able to take arbitrary proofs written in natural language from the mathematical literature and convert them into a symbolic form suitable for symbolic verification.","kind":"forecast","conditions":["from the mathematical literature"],"original_horizon_text":"In 2029","statement":{"excerpt":"In 2029, AI will not be able to take arbitrary proofs from the mathematical literature written in natural language and convert them into a symbolic form suitable for symbolic verification.","locator":"prediction 5 of 5","url":"https://garymarcus.substack.com/p/dear-elon-musk-here-are-five-things","date":"2022-05-31","document_title":"Dear Elon Musk, here are five things you might want to consider about AGI","observed_at":"2026-10-28T22:49:04Z"},"criteria":{"from_speaker":["In 2029","AI will not be able to take arbitrary proofs","from the mathematical literature","written in natural language","convert them into a symbolic form suitable for symbolic verification"],"our_interpretation":["The deadline is December 31, 2029, allowing the whole named year.","Arbitrary means broad applicability to independently selected proofs across mathematical literature, rather than selected showcases or one narrow class. It does not require successfully verifying false arguments.","Converting proofs means preserving their mathematical content and substantive reasoning while supplying formal details. Producing a different proof of the same theorem alone does not demonstrate faithful conversion.","Successful checking by a proof assistant is the practical test of suitability for symbolic verification. Statements alone, unchecked proof holes and unsupported assumptions do not establish success.","Marcus specifies neither a representative sampling method nor an acceptable failure rate; those must be disclosed rather than silently invented.","The literature restriction defines the task, rather than a conditional forecast requiring a separate antecedent.","This assessment concerns the individual formalization prediction, not winning the broader three-of-five AGI bet."],"support_condition":"Broad evaluations during 2029 find that leading systems still cannot faithfully formalize independently selected natural-language proofs from diverse mathematical literature, with recurring substantive failures rather than merely occasional errors. Absence of a demonstration alone is insufficient.","contradiction_condition":"Reproducible evaluations establish broad successful conversion of independently selected literature proofs into checked symbolic proofs, preserving the original mathematical content and reasoning without substantial expert completion."},"axis":"forecast_outcome","outcome":"unresolved","horizon_state":"not_due","evaluation_as_of":"2026-10-28","rationale":"The deadline is December 31, 2029. Textbook-scale formalization and the reported checked proof of Fermat's Last Theorem are substantial counterevidence to any suggestion that literature-proof formalization is wholly beyond AI. They do not establish the broad arbitrary-proof capability in this forecast. The broader benchmark also supplies human-formalized statements. No reviewed evidence meets the early-resolution rule, and current gaps cannot establish failure in 2029.","limitations":["The research and developer reports were read, but their generated proof artifacts were not independently rerun in this assessment.","Breadth and proof fidelity require judgments the speaker did not fully specify.","A checked proof of a theorem does not alone establish faithful translation of its source argument."],"evidence":[{"id":"e3","relation":"context","excerpt":"large-scale automated formalization of mathematical literature","locator":"Abstract, penultimate sentence","url":"https://arxiv.org/abs/2602.17016","publisher":"arXiv","event_date":"2026-02-19","available_at":"2026-02-19","explanation":"M2F reports 153,853 lines of Lean from 479 textbook pages on real and convex analysis. This is material progress, but those selected corpora and the reported statement audits do not establish faithful conversion of arbitrary literature proofs across mathematics."},{"id":"e4","relation":"context","excerpt":"computer-checked proof of Fermat’s Last Theorem.","locator":"Opening paragraph","url":"https://www.anthropic.com/research/formalizing-fermats-last-theorem","publisher":"Anthropic","event_date":"2026-08-18","available_at":"2026-09-04","explanation":"Anthropic reports a checked formalization following a published exposition of Wiles's proof, with limited high-level human input; its quoted completion log dates the achievement to August 18 UTC. This strongly challenges a reading that AI cannot formalize major literature proofs, but a selected project does not establish the forecast's arbitrary-proof breadth."},{"id":"e5","relation":"context","excerpt":"leaving the corresponding Lean proof","locator":"Abstract, fifth sentence","url":"https://ai.meta.com/research/publications/autoformbench-a-lean-formalization-benchmark-for-advanced-mathematics","publisher":"Meta AI","event_date":"2026-10-13","available_at":"2026-10-13","explanation":"AutoformBench covers 341 tasks from 24 advanced mathematical sources and supplies informal proofs, making it relevant to breadth. Human-formalized target statements provide part of the translation, and the announcement does not establish general faithful conversion of arbitrary literature proofs."}],"what_would_resolve":"An independent, reproducible evaluation across diverse literature proofs, selected without favoring the system, with source-fidelity review, complete checker results and disclosed human interventions. Such evidence could establish capability early; otherwise evaluations during 2029 must settle the forecast.","retrospective":false,"plain_reading":true},"payload_hash":"950bb3d8a76b601128c29aaa431e15a7d374332414f154b8744d2c7b33380169","policy_version":"2026-10-28.7","recorded_at":"2026-10-29T06:56:00Z","drafter":{"provider":"openai","model":"gpt-6-astra"},"reviewers":[{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-dear-elon-musk-here-are-five-things-5dc69d-4x-review-anthropic-1"},{"provider":"openai","model":"gpt-6-astra","run_id":"marcus-dear-elon-musk-here-are-five-things-5dc69d-4x-review-openai-1"}],"final_text":{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-dear-elon-musk-here-are-five-things-5dc69d-4x-final-anthropic-1"},"status":"current","notice":null,"superseded_by":null,"evidence_meta":[{"id":"e3","source_type":"preprint","audited":null,"derived":null,"metric":null},{"id":"e4","source_type":"company_disclosure","audited":null,"derived":null,"metric":null},{"id":"e5","source_type":"company_disclosure","audited":null,"derived":null,"metric":null}],"public_from":null,"public_until":null},"history":[],"review":{"review_due":false,"triggers":[],"last_completed_review_at":"2026-10-29T06:56:00Z","next_review_due_at":null,"horizon_now":"not_due"},"event_ids":["marcus-dear-elon-musk-here-are-five-things-720a9f-assessment-published-a1"],"last_substantive_change_at":"2026-10-29T17:00:18Z"}},{"url":"https://punditclaims.ai/claims/marcus-dear-elon-musk-here-are-five-things-8818ae","standing":"current","assessment_label":"Supported","versions":[{"assessment_id":"marcus-dear-elon-musk-here-are-five-things-8818ae-a1","status":"current","url":"https://punditclaims.ai/claims/marcus-dear-elon-musk-here-are-five-things-8818ae#assessment-marcus-dear-elon-musk-here-are-five-things-8818ae-a1"}],"claim":{"claim_id":"marcus-dear-elon-musk-here-are-five-things-8818ae","subject_id":"gary-marcus","revision":1,"content_hash":"239ae72d5c42d78f6f93a08323fb938516e5fd90607e65bf63a0672ac2ad6b8d","normalized_proposition":"3D perception and scene understanding remain unsolved for current AI.","kind":"observation","target":"current AI","conditions":[],"original_horizon_text":null,"topic_tags":["capabilities"],"concerns_providers":[],"selection_stratum":"declared-lead","occurrences":[{"id":"marcus-dear-elon-musk-here-are-five-things-8818ae-o1","statement_date":"2022-05-31","statement_date_precision":"day","excerpt":"Even within perception 3D perception remains a challenge, and scene understanding is by no means solved.","locator":"paragraph 21","document":{"url":"https://garymarcus.substack.com/p/dear-elon-musk-here-are-five-things","title":"Dear Elon Musk, here are five things you might want to consider about AGI","publisher":"Marcus on AI","published_at":"2022-05-31T18:32:41Z","date_uncertain":false,"date_provenance":"JSON-LD datePublished"},"observed_at":"2026-10-28T22:49:04Z","observed_via":"live","archive_url":null,"access_state":"full_public","snapshot_hash":"d64c073bb595b86ed0d5a52382668c601d5a8e7009353d00f0e54061bdd4f0f7","receipts":1}],"first_statement_date":"2022-05-31","related":[],"current":{"assessment_id":"marcus-dear-elon-musk-here-are-five-things-8818ae-a1","payload":{"claim_id":"marcus-dear-elon-musk-here-are-five-things-8818ae","subject_id":"gary-marcus","normalized_proposition":"3D perception and scene understanding remain unsolved for current AI.","kind":"observation","conditions":[],"original_horizon_text":null,"statement":{"excerpt":"Even within perception 3D perception remains a challenge, and scene understanding is by no means solved.","locator":"paragraph 21","url":"https://garymarcus.substack.com/p/dear-elon-musk-here-are-five-things","date":"2022-05-31","document_title":"Dear Elon Musk, here are five things you might want to consider about AGI","observed_at":"2026-10-28T22:49:04Z"},"criteria":{"from_speaker":["Even within perception","3D perception remains a challenge","scene understanding is by no means solved"],"our_interpretation":["The ordinary reading is that substantial technical failures remained in reliable 3D perception and scene understanding.","Solved does not require flawless performance, and the claim does not deny useful systems or progress.","Scene understanding includes identifying objects and their spatial relationships within a scene.","The observation concerns the current 2022 research record through May 31; academic research has no fixed reporting quarter."],"support_condition":"Contemporaneous evaluations document material remaining limitations in both 3D perception and scene understanding.","contradiction_condition":"Contemporaneous evidence establishes broadly reliable solutions in both areas, with remaining errors too limited to sustain the ordinary claim."},"axis":"historical_observation","outcome":"supported","horizon_state":"not_applicable","evaluation_as_of":"2026-10-28","rationale":"Research available before the post documents significant weaknesses in 3D recognition under common corruptions and a remaining object-identification bottleneck in spatial scene question answering. These are substantive limitations in both named areas. The studies also show progress, which is compatible with Marcus's claim that the problems remained challenging and unsolved.","limitations":["The studies cover particular tasks rather than every form of perception or scene understanding.","ScanQA combines perception and language; its limitations cannot all be attributed to perception alone.","The ScanQA work first appeared in 2021; the cited evidence is its publicly available May 2022 revision."],"evidence":[{"id":"e1","relation":"supports","excerpt":"a significant gap","locator":"Abstract","url":"https://arxiv.org/abs/2201.12296v1","publisher":"arXiv; Jiachen Sun and colleagues","event_date":"2022-01-28","available_at":"2022-01-28","explanation":"Testing fifteen common corruptions exposed substantial robustness weaknesses in leading 3D recognition models. The authors also report mitigation methods, so the result establishes remaining challenges rather than an absence of progress."},{"id":"e2","relation":"supports","excerpt":"oracle setting","locator":"Section 5.2, comparison of ScanQA with image-based question-answering baselines","url":"https://arxiv.org/html/2112.10482v3","publisher":"arXiv; Daichi Azuma and colleagues","event_date":"2022-05-07","available_at":"2022-05-07","explanation":"The May revision reports that supplying correct object identification can make an image-based baseline competitive with or better than ScanQA. Its authors identify that assistance as unavailable in real-world use, demonstrating a remaining scene-understanding bottleneck."}],"what_would_resolve":null,"retrospective":true,"plain_reading":true},"payload_hash":"2a48ff2270d1705604bb2b7902283e468eadadf0d83aa1c4dd2760d353d32de9","policy_version":"2026-10-28.7","recorded_at":"2026-10-29T06:56:00Z","drafter":{"provider":"openai","model":"gpt-6-astra"},"reviewers":[{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-dear-elon-musk-here-are-five-things-5dc69d-4x-review-anthropic-1"},{"provider":"openai","model":"gpt-6-astra","run_id":"marcus-dear-elon-musk-here-are-five-things-5dc69d-4x-review-openai-1"}],"final_text":{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-dear-elon-musk-here-are-five-things-5dc69d-4x-final-anthropic-1"},"status":"current","notice":null,"superseded_by":null,"evidence_meta":[{"id":"e1","source_type":"preprint","audited":null,"derived":null,"metric":null},{"id":"e2","source_type":"preprint","audited":null,"derived":null,"metric":null}],"public_from":null,"public_until":null},"history":[],"review":{"review_due":false,"triggers":[],"last_completed_review_at":"2026-10-29T06:56:00Z","next_review_due_at":null,"horizon_now":"not_applicable"},"event_ids":["marcus-dear-elon-musk-here-are-five-things-8818ae-assessment-published-a1"],"last_substantive_change_at":"2026-10-29T17:00:18Z"}},{"url":"https://punditclaims.ai/claims/marcus-dear-elon-musk-here-are-five-things-a53d89","standing":"current","assessment_label":"Supported","versions":[{"assessment_id":"marcus-dear-elon-musk-here-are-five-things-a53d89-a1","status":"current","url":"https://punditclaims.ai/claims/marcus-dear-elon-musk-here-are-five-things-a53d89#assessment-marcus-dear-elon-musk-here-are-five-things-a53d89-a1"}],"claim":{"claim_id":"marcus-dear-elon-musk-here-are-five-things-a53d89","subject_id":"gary-marcus","revision":2,"content_hash":"3a4b2181b4efda74b150d3e516b9d9940414780159e8b855c2b31ced2dc3bee9","normalized_proposition":"The long-tail problem of rare events is a very serious issue for the currently popular AI techniques, which rely more on big data than on reasoning.","kind":"observation","target":"current big-data AI techniques","conditions":[{"text":"for the kinds of AI techniques that are currently popular","type":"scope","from_source":true}],"original_horizon_text":null,"topic_tags":["reliability"],"concerns_providers":[],"selection_stratum":"declared-lead","occurrences":[{"id":"marcus-dear-elon-musk-here-are-five-things-a53d89-o1","statement_date":"2022-05-31","statement_date_precision":"day","excerpt":"But for the kinds of AI techniques that are currently popular, and which rely more on big data than on reasoning, it’s a very serious issue.","locator":"paragraph 10","document":{"url":"https://garymarcus.substack.com/p/dear-elon-musk-here-are-five-things","title":"Dear Elon Musk, here are five things you might want to consider about AGI","publisher":"Marcus on AI","published_at":"2022-05-31T18:32:41Z","date_uncertain":false,"date_provenance":"JSON-LD datePublished"},"observed_at":"2026-10-28T22:49:04Z","observed_via":"live","archive_url":null,"access_state":"full_public","snapshot_hash":"d64c073bb595b86ed0d5a52382668c601d5a8e7009353d00f0e54061bdd4f0f7","receipts":1}],"first_statement_date":"2022-05-31","related":[],"current":{"assessment_id":"marcus-dear-elon-musk-here-are-five-things-a53d89-a1","payload":{"claim_id":"marcus-dear-elon-musk-here-are-five-things-a53d89","subject_id":"gary-marcus","normalized_proposition":"The long-tail problem of rare events is a very serious issue for the currently popular AI techniques, which rely more on big data than on reasoning.","kind":"observation","conditions":["for the kinds of AI techniques that are currently popular"],"original_horizon_text":null,"statement":{"excerpt":"But for the kinds of AI techniques that are currently popular, and which rely more on big data than on reasoning, it’s a very serious issue.","locator":"paragraph 10","url":"https://garymarcus.substack.com/p/dear-elon-musk-here-are-five-things","date":"2022-05-31","document_title":"Dear Elon Musk, here are five things you might want to consider about AGI","observed_at":"2026-10-28T22:49:04Z"},"criteria":{"from_speaker":["AI techniques that are currently popular","rely more on big data than on reasoning","a very serious issue"],"our_interpretation":["The surrounding text identifies the issue as poor handling of rare cases with little training data.","The ordinary reading concerns mainstream data-driven deep learning, not every AI method or an assertion that neural networks cannot reason at all.","Very serious means substantial performance or reliability limitations, not a claim that the problem is impossible to mitigate.","For this continuously reported technical field, current is operationalized as the 2022 state of the art up to the post date, rather than an accounting quarter."],"support_condition":"Contemporary research documents substantial deficiencies on rare classes or situations in mainstream deep-learning systems, despite stronger performance where training data are plentiful.","contradiction_condition":"Representative contemporary evaluations show that rare-case limitations are generally small or robustly resolved in the relevant methods."},"axis":"historical_observation","outcome":"supported","horizon_state":"not_applicable","evaluation_as_of":"2026-10-28","rationale":"A research synthesis publicly available four days before the post identifies poor rare-class performance as a substantial limitation of conventional deep learning. That supports the ordinary reading of a serious practical issue. Existing mitigation methods show progress, so the evidence does not support stronger claims of universal failure or impossibility.","limitations":["Evidence is strongest for visual recognition.","Seriousness is qualitative rather than tied to a speaker-specified threshold.","The comparison with reasoning is a description of the targeted methods, not a separately established ranking of all AI approaches."],"evidence":[{"id":"e1","relation":"supports","excerpt":"Using conventional methods (e.g., Cross-Entropy loss, or simple fine-tuning, etc.) will result in the poor performance of the tail classes.","locator":"Section 2.1, discussion of transfer learning","url":"https://ar5iv.labs.arxiv.org/html/2205.13775","publisher":"International Journal of Computer Vision; author manuscript rendered by ar5iv","event_date":"2022-05","available_at":"2022-05-27","explanation":"The contemporary synthesis documents poor learning of rare classes and substantial limitations in conventional deep-learning methods. It also reviews mitigation methods, supporting a serious practical problem without implying that every method fails."},{"id":"e2","relation":"context","excerpt":"Experiments on long-tailed CIFAR, ImageNet, Places, and iNaturalist 2018 manifest the new state-of-the-art for long-tailed recognition.","locator":"Abstract","url":"https://arxiv.org/abs/2107.12028v1","publisher":"arXiv","event_date":"2021-07","available_at":"2021-07-26","explanation":"PaCo demonstrates progress on several long-tailed benchmarks using changes within deep learning. This qualifies any stronger interpretation that the problem had no useful mitigation, but does not show it was generally solved in 2022."}],"what_would_resolve":null,"retrospective":true,"plain_reading":true},"payload_hash":"1d6c76b6910875bb289036929e676670c7438a8b03e4b61f0142c26bd11aa53c","policy_version":"2026-10-28.7","recorded_at":"2026-10-29T06:56:00Z","drafter":{"provider":"openai","model":"gpt-6-astra"},"reviewers":[{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-dear-elon-musk-here-are-five-things-5dc69d-4x-review-anthropic-1"},{"provider":"openai","model":"gpt-6-astra","run_id":"marcus-dear-elon-musk-here-are-five-things-5dc69d-4x-review-openai-1"}],"final_text":{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-dear-elon-musk-here-are-five-things-5dc69d-4x-final-anthropic-1"},"status":"current","notice":null,"superseded_by":null,"evidence_meta":[{"id":"e1","source_type":"peer_reviewed","audited":null,"derived":null,"metric":null},{"id":"e2","source_type":"preprint","audited":null,"derived":null,"metric":null}],"public_from":null,"public_until":null},"history":[],"review":{"review_due":false,"triggers":[],"last_completed_review_at":"2026-10-29T06:56:00Z","next_review_due_at":null,"horizon_now":"not_applicable"},"event_ids":["marcus-dear-elon-musk-here-are-five-things-a53d89-assessment-published-a1"],"last_substantive_change_at":"2026-10-29T17:00:18Z"}},{"url":"https://punditclaims.ai/claims/marcus-dear-elon-musk-here-are-five-things-acced4","standing":"current","assessment_label":"Unresolved","versions":[{"assessment_id":"marcus-dear-elon-musk-here-are-five-things-acced4-a1","status":"current","url":"https://punditclaims.ai/claims/marcus-dear-elon-musk-here-are-five-things-acced4#assessment-marcus-dear-elon-musk-here-are-five-things-acced4-a1"}],"claim":{"claim_id":"marcus-dear-elon-musk-here-are-five-things-acced4","subject_id":"gary-marcus","revision":1,"content_hash":"8b24025cf4d649799ddc9c0766949014604bd7cfcfe5df34aa496821cbca733b","normalized_proposition":"In 2029, AI will not be able to work as a competent cook in an arbitrary kitchen.","kind":"forecast","target":"AI","conditions":[],"original_horizon_text":"In 2029","topic_tags":["capabilities","agi-timelines"],"concerns_providers":[],"selection_stratum":"declared-lead","occurrences":[{"id":"marcus-dear-elon-musk-here-are-five-things-acced4-o1","statement_date":"2022-05-31","statement_date_precision":"day","excerpt":"In 2029, AI will not be able to work as a competent cook in an arbitrary kitchen (extending Steve Wozniak’s cup of coffee benchmark).","locator":"prediction 3 of 5","document":{"url":"https://garymarcus.substack.com/p/dear-elon-musk-here-are-five-things","title":"Dear Elon Musk, here are five things you might want to consider about AGI","publisher":"Marcus on AI","published_at":"2022-05-31T18:32:41Z","date_uncertain":false,"date_provenance":"JSON-LD datePublished"},"observed_at":"2026-10-28T22:49:04Z","observed_via":"live","archive_url":null,"access_state":"full_public","snapshot_hash":"d64c073bb595b86ed0d5a52382668c601d5a8e7009353d00f0e54061bdd4f0f7","receipts":1}],"first_statement_date":"2022-05-31","related":[],"current":{"assessment_id":"marcus-dear-elon-musk-here-are-five-things-acced4-a1","payload":{"claim_id":"marcus-dear-elon-musk-here-are-five-things-acced4","subject_id":"gary-marcus","normalized_proposition":"In 2029, AI will not be able to work as a competent cook in an arbitrary kitchen.","kind":"forecast","conditions":[],"original_horizon_text":"In 2029","statement":{"excerpt":"In 2029, AI will not be able to work as a competent cook in an arbitrary kitchen (extending Steve Wozniak’s cup of coffee benchmark).","locator":"prediction 3 of 5","url":"https://garymarcus.substack.com/p/dear-elon-musk-here-are-five-things","date":"2022-05-31","document_title":"Dear Elon Musk, here are five things you might want to consider about AGI","observed_at":"2026-10-28T22:49:04Z"},"criteria":{"from_speaker":["In 2029","work as a competent cook","in an arbitrary kitchen","extending Steve Wozniak’s cup of coffee benchmark"],"our_interpretation":["The ordinary reading requires an embodied AI system that can prepare ordinary meals competently in unfamiliar, usable kitchens.","'Arbitrary' means varied ordinary kitchens without installation tailored to the robot; it does not mean impossible or unequipped environments.","Competence includes locating and using equipment, handling ingredients, managing cooking stages, and producing safe, edible food with reasonable consistency.","Human meal requests are allowed, but teleoperation or human completion of cooking steps does not demonstrate the robot's competence.","Use 2029-12-31 as the latest deadline and assess this forecast independently of the surrounding wager."],"support_condition":"Evidence covering the end of 2029 shows that available AI robots still cannot consistently complete ordinary cooking tasks in varied unfamiliar kitchens without substantial human assistance or robot-specific installation.","contradiction_condition":"An AI robot repeatedly prepares ordinary meals competently and safely across varied unfamiliar kitchens without teleoperation, task completion by people, or bespoke kitchen installation."},"axis":"forecast_outcome","outcome":"unresolved","horizon_state":"not_due","evaluation_as_of":"2026-10-28","rationale":"The forecast is not due. YORI demonstrates cooking in a structured installation, while Figure reports unfamiliar-home performance on other household tasks. These are relevant advances, but neither establishes competent cooking in arbitrary kitchens. No inspected evidence meets the early-resolution rule.","limitations":["The cooking demonstrations and unfamiliar-home results concern different systems and tasks.","A company report is not independent confirmation.","The absence of a qualifying result in this search does not establish that none exists."],"evidence":[{"id":"e1","relation":"context","excerpt":"reliably preparing steak-frites","locator":"Abstract, sixth sentence","url":"https://arxiv.org/abs/2405.11094v4","publisher":"arXiv; Donghun Noh and collaborators","event_date":"2025","available_at":"2025-11-24","explanation":"YORI provides relevant counterpressure to a broad claim that robots cannot cook. The report specifies custom tools, appliances and a structured kitchen, so it does not demonstrate competence in arbitrary kitchens."},{"id":"e2","relation":"context","excerpt":"tidying a room","locator":"Section The Whole-Body Generalization Problem, first paragraph","url":"https://www.figure.ai/news/helix-2-5-zero-shot-30-home-generalization","publisher":"Figure","event_date":"2026-09","available_at":"2026-09-17","explanation":"Figure reports evaluation across 30 unseen homes, which is relevant to generalization. The evaluated behaviors exclude cooking, so this does not satisfy the forecast's combined task and environment requirements."}],"what_would_resolve":"An independently observed, repeatable evaluation showing autonomous meal preparation across varied unfamiliar kitchens would contradict the forecast early. Otherwise assess the complete capability through the end of 2029.","retrospective":false,"plain_reading":true},"payload_hash":"fb033f83c8127334af5522d5bf03c0524f8d40d45209582280974c5536584b0d","policy_version":"2026-10-28.7","recorded_at":"2026-10-29T06:56:13Z","drafter":{"provider":"openai","model":"gpt-6-astra"},"reviewers":[{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-dear-elon-musk-here-are-five-things-acced4-4x-review-anthropic-1"},{"provider":"openai","model":"gpt-6-astra","run_id":"marcus-dear-elon-musk-here-are-five-things-acced4-4x-review-openai-1"}],"final_text":{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-dear-elon-musk-here-are-five-things-acced4-4x-final-anthropic-1"},"status":"current","notice":null,"superseded_by":null,"evidence_meta":[{"id":"e1","source_type":"preprint","audited":null,"derived":null,"metric":null},{"id":"e2","source_type":"company_disclosure","audited":null,"derived":null,"metric":null}],"public_from":null,"public_until":null},"history":[],"review":{"review_due":false,"triggers":[],"last_completed_review_at":"2026-10-29T06:56:13Z","next_review_due_at":null,"horizon_now":"not_due"},"event_ids":["marcus-dear-elon-musk-here-are-five-things-acced4-assessment-published-a1"],"last_substantive_change_at":"2026-10-29T17:00:18Z"}},{"url":"https://punditclaims.ai/claims/marcus-dear-elon-musk-here-are-five-things-b5e529","standing":"current","assessment_label":"Can’t be checked","versions":[{"assessment_id":"marcus-dear-elon-musk-here-are-five-things-b5e529-a1","status":"current","url":"https://punditclaims.ai/claims/marcus-dear-elon-musk-here-are-five-things-b5e529#assessment-marcus-dear-elon-musk-here-are-five-things-b5e529-a1"}],"claim":{"claim_id":"marcus-dear-elon-musk-here-are-five-things-b5e529","subject_id":"gary-marcus","revision":2,"content_hash":"40f0a7ca178b45cbbeff0368ce40bd2cdf26bad520a486c7d84cb378acdef944","normalized_proposition":"Seven years might well not be enough to invent the further techniques needed, if they do not already exist, and to take them from the lab into production.","kind":"forecast","target":"the AI field","conditions":[{"text":"if they don't already exist","type":"qualifier","from_source":true},{"text":"being realistic","type":"qualifier","from_source":true}],"original_horizon_text":"7 years","topic_tags":["agi-timelines"],"concerns_providers":[],"selection_stratum":"declared-lead","occurrences":[{"id":"marcus-dear-elon-musk-here-are-five-things-b5e529-o1","statement_date":"2022-05-31","statement_date_precision":"day","excerpt":"Being realistic, 7 years might well not be enough to (a) invent those tools (if they are don’t already exist) and (b) take them from the lab into production.","locator":"paragraph 25","document":{"url":"https://garymarcus.substack.com/p/dear-elon-musk-here-are-five-things","title":"Dear Elon Musk, here are five things you might want to consider about AGI","publisher":"Marcus on AI","published_at":"2022-05-31T18:32:41Z","date_uncertain":false,"date_provenance":"JSON-LD datePublished"},"observed_at":"2026-10-28T22:49:04Z","observed_via":"live","archive_url":null,"access_state":"full_public","snapshot_hash":"d64c073bb595b86ed0d5a52382668c601d5a8e7009353d00f0e54061bdd4f0f7","receipts":1}],"first_statement_date":"2022-05-31","related":[],"current":{"assessment_id":"marcus-dear-elon-musk-here-are-five-things-b5e529-a1","payload":{"claim_id":"marcus-dear-elon-musk-here-are-five-things-b5e529","subject_id":"gary-marcus","normalized_proposition":"Seven years might well not be enough to invent the further techniques needed, if they do not already exist, and to take them from the lab into production.","kind":"forecast","conditions":["if they don't already exist","being realistic"],"original_horizon_text":"7 years","statement":{"excerpt":"Being realistic, 7 years might well not be enough to (a) invent those tools (if they are don’t already exist) and (b) take them from the lab into production.","locator":"paragraph 25","url":"https://garymarcus.substack.com/p/dear-elon-musk-here-are-five-things","date":"2022-05-31","document_title":"Dear Elon Musk, here are five things you might want to consider about AGI","observed_at":"2026-10-28T22:49:04Z"},"criteria":{"from_speaker":["if they don't already exist","being realistic","7 years"],"our_interpretation":[],"support_condition":"Not checked against evidence: both screens found that public evidence could not settle this claim.","contradiction_condition":"Not checked against evidence: both screens found that public evidence could not settle this claim."},"axis":"not_applicable","outcome":"not_adjudicable","horizon_state":"not_applicable","evaluation_as_of":"2026-10-28","rationale":"Not checked against evidence. Two independent screens, one from each provider, both found this stated as a chance or a possibility, or at medium or low confidence.","limitations":[],"evidence":[],"what_would_resolve":null,"retrospective":true,"uncheckable":["probability_language"]},"payload_hash":"e808aec6a7248a1457bc60751b595d55536b402c98b3ff44e1202f4a4f05545f","policy_version":"2026-10-28.7","recorded_at":"2026-10-29T02:32:05Z","drafter":{"provider":"anthropic","model":"claude-opus-5-5"},"reviewers":[{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-dear-elon-musk-here-are-five-things-screen-anthropic-2"},{"provider":"openai","model":"gpt-6-astra","run_id":"marcus-dear-elon-musk-here-are-five-things-screen-openai-2"}],"final_text":null,"status":"current","notice":null,"superseded_by":null,"evidence_meta":[],"public_from":null,"public_until":null},"history":[],"review":{"review_due":false,"triggers":[],"last_completed_review_at":"2026-10-29T02:32:05Z","next_review_due_at":null,"horizon_now":"not_applicable"},"event_ids":["marcus-dear-elon-musk-here-are-five-things-b5e529-assessment-published-a1"],"last_substantive_change_at":"2026-10-29T17:00:18Z"}},{"url":"https://punditclaims.ai/claims/marcus-dear-elon-musk-here-are-five-things-d1f821","standing":"current","assessment_label":"Mixed","versions":[{"assessment_id":"marcus-dear-elon-musk-here-are-five-things-d1f821-a1","status":"current","url":"https://punditclaims.ai/claims/marcus-dear-elon-musk-here-are-five-things-d1f821#assessment-marcus-dear-elon-musk-here-are-five-things-d1f821-a1"}],"claim":{"claim_id":"marcus-dear-elon-musk-here-are-five-things-d1f821","subject_id":"gary-marcus","revision":2,"content_hash":"63f0fc406a4670dbad93f488de9833a508771d8870bcd0216fc136933981fbf9","normalized_proposition":"Debugging deep learning is wicked hard because nobody really understands how it works or knows how to fix problems other than collecting more data, adding more layers and so forth.","kind":"causal","target":"deep learning","conditions":[],"original_horizon_text":null,"topic_tags":["reliability"],"concerns_providers":[],"selection_stratum":"declared-lead","occurrences":[{"id":"marcus-dear-elon-musk-here-are-five-things-d1f821-o1","statement_date":"2022-05-31","statement_date_precision":"day","excerpt":"Debugging with deep learning is wicked hard, because nobody (a) really understands how it works, and (b) knows how to fix problems other to collect more data and add more layers and so forth.","locator":"paragraph 29","document":{"url":"https://garymarcus.substack.com/p/dear-elon-musk-here-are-five-things","title":"Dear Elon Musk, here are five things you might want to consider about AGI","publisher":"Marcus on AI","published_at":"2022-05-31T18:32:41Z","date_uncertain":false,"date_provenance":"JSON-LD datePublished"},"observed_at":"2026-10-28T22:49:04Z","observed_via":"live","archive_url":null,"access_state":"full_public","snapshot_hash":"d64c073bb595b86ed0d5a52382668c601d5a8e7009353d00f0e54061bdd4f0f7","receipts":1}],"first_statement_date":"2022-05-31","related":[],"current":{"assessment_id":"marcus-dear-elon-musk-here-are-five-things-d1f821-a1","payload":{"claim_id":"marcus-dear-elon-musk-here-are-five-things-d1f821","subject_id":"gary-marcus","normalized_proposition":"Debugging deep learning is wicked hard because nobody really understands how it works or knows how to fix problems other than collecting more data, adding more layers and so forth.","kind":"causal","conditions":[],"original_horizon_text":null,"statement":{"excerpt":"Debugging with deep learning is wicked hard, because nobody (a) really understands how it works, and (b) knows how to fix problems other to collect more data and add more layers and so forth.","locator":"paragraph 29","url":"https://garymarcus.substack.com/p/dear-elon-musk-here-are-five-things","date":"2022-05-31","document_title":"Dear Elon Musk, here are five things you might want to consider about AGI","observed_at":"2026-10-28T22:49:04Z"},"criteria":{"from_speaker":["Debugging with deep learning is wicked hard","nobody (a) really understands how it works","knows how to fix problems other to collect more data and add more layers and so forth"],"our_interpretation":["Assess the state of knowledge available when the post appeared, on 2022-05-31. The target period has no lower bound because established earlier methods remain relevant to that state.","The ordinary reading is that limited understanding makes diagnosis and repair difficult, and that practitioners lack effective targeted remedies beyond broadly changing data, training, or model size.","Assess the difficulty and its proposed explanation separately. A targeted repair need not solve debugging generally to count against the assertion that such remedies were unknown.","Do not interpret 'understands' as requiring complete mathematical understanding, or 'and so forth' as automatically encompassing every possible repair."],"support_condition":"Contemporaneous research or engineering evidence shows that model opacity materially obstructed diagnosis and repair, and that effective remedies were generally limited to broad data or training changes.","contradiction_condition":"Contemporaneous work demonstrates useful understanding of internal mechanisms and effective targeted repairs beyond collecting more data or enlarging models. Evidence that debugging was routinely straightforward would also contradict the difficulty claim."},"axis":"causal_assessment","outcome":"mixed","horizon_state":"not_applicable","evaluation_as_of":"2026-10-28","rationale":"Contemporary research supports Marcus's point that opacity makes debugging harder. But the stronger explanation overstates the limits of existing methods: before the post, ROME demonstrated causal investigation of factual recall and targeted edits to model weights. The difficulty claim holds in this narrower sense; the assertion that useful understanding and targeted remedies were absent does not.","limitations":["The evidence concerns neural language models, not every deep learning application.","The supporting survey gives a qualitative assessment, not a controlled estimate of the causal contribution of opacity.","The mixed verdict depends on treating the assertion about absent remedies as a substantive commitment rather than loose rhetoric."],"evidence":[{"id":"e1","relation":"supports","excerpt":"opaque deep learning models.","locator":"Abstract, second sentence","url":"https://arxiv.org/abs/2104.15135v1","publisher":"arXiv; Piyawat Lertvittayakumjorn and Francesca Toni","event_date":"2021-04","available_at":"2021-04-30","explanation":"The survey supports the narrower causal point that opacity makes locating and repairing problems harder. Its review of explanation-based debugging also shows that researchers were already developing remedies."},{"id":"e2","relation":"contradicts","excerpt":"editing facts stored in model weights.","locator":"Abstract, fourth sentence","url":"https://arxiv.org/abs/2202.05262v1","publisher":"arXiv; Kevin Meng, David Bau, Alex Andonian and Yonatan Belinkov","event_date":"2022-02","available_at":"2022-02-10","explanation":"The authors used causal interventions to investigate factual recall and evaluated targeted factual edits on CounterFact. This supplies a concrete pre-post example of useful mechanistic understanding and repair without adding data or layers."}],"what_would_resolve":null,"retrospective":true,"plain_reading":true},"payload_hash":"204f8a6d91b7a4e2f1ab3067cf06e0a35ca013c6d6b0cb883ac8cd961bf2fe63","policy_version":"2026-10-28.7","recorded_at":"2026-10-29T06:56:13Z","drafter":{"provider":"openai","model":"gpt-6-astra"},"reviewers":[{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-dear-elon-musk-here-are-five-things-acced4-4x-review-anthropic-1"},{"provider":"openai","model":"gpt-6-astra","run_id":"marcus-dear-elon-musk-here-are-five-things-acced4-4x-review-openai-1"}],"final_text":{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-dear-elon-musk-here-are-five-things-acced4-4x-final-anthropic-1"},"status":"current","notice":null,"superseded_by":null,"evidence_meta":[{"id":"e1","source_type":"preprint","audited":null,"derived":null,"metric":null},{"id":"e2","source_type":"preprint","audited":null,"derived":null,"metric":null}],"public_from":null,"public_until":null},"history":[],"review":{"review_due":false,"triggers":[],"last_completed_review_at":"2026-10-29T06:56:13Z","next_review_due_at":null,"horizon_now":"not_applicable"},"event_ids":["marcus-dear-elon-musk-here-are-five-things-d1f821-assessment-published-a1"],"last_substantive_change_at":"2026-10-29T17:00:18Z"}},{"url":"https://punditclaims.ai/claims/marcus-dear-elon-musk-here-are-five-things-d56a36","standing":"current","assessment_label":"Unresolved","versions":[{"assessment_id":"marcus-dear-elon-musk-here-are-five-things-d56a36-a1","status":"current","url":"https://punditclaims.ai/claims/marcus-dear-elon-musk-here-are-five-things-d56a36#assessment-marcus-dear-elon-musk-here-are-five-things-d56a36-a1"}],"claim":{"claim_id":"marcus-dear-elon-musk-here-are-five-things-d56a36","subject_id":"gary-marcus","revision":1,"content_hash":"f9cf90f4d5f44e7c7e5488002fb96c4bdba4cf17f1433e1d7a9fbe6904c12945","normalized_proposition":"The long tail problem makes it easy to believe AI problems are much easier than they actually are.","kind":"causal","target":"the long tail problem","conditions":[],"original_horizon_text":null,"topic_tags":["reliability","capabilities"],"concerns_providers":[],"selection_stratum":"declared-lead","occurrences":[{"id":"marcus-dear-elon-musk-here-are-five-things-d56a36-o1","statement_date":"2022-05-31","statement_date_precision":"day","excerpt":"The thing is, it’s easy to convince yourself that AI problems are much easier than they are actually are, because of the long tail problem.","locator":"paragraph 10","document":{"url":"https://garymarcus.substack.com/p/dear-elon-musk-here-are-five-things","title":"Dear Elon Musk, here are five things you might want to consider about AGI","publisher":"Marcus on AI","published_at":"2022-05-31T18:32:41Z","date_uncertain":false,"date_provenance":"JSON-LD datePublished"},"observed_at":"2026-10-28T22:49:04Z","observed_via":"live","archive_url":null,"access_state":"full_public","snapshot_hash":"d64c073bb595b86ed0d5a52382668c601d5a8e7009353d00f0e54061bdd4f0f7","receipts":1}],"first_statement_date":"2022-05-31","related":[],"current":{"assessment_id":"marcus-dear-elon-musk-here-are-five-things-d56a36-a1","payload":{"claim_id":"marcus-dear-elon-musk-here-are-five-things-d56a36","subject_id":"gary-marcus","normalized_proposition":"The long tail problem makes it easy to believe AI problems are much easier than they actually are.","kind":"causal","conditions":[],"original_horizon_text":null,"statement":{"excerpt":"The thing is, it’s easy to convince yourself that AI problems are much easier than they are actually are, because of the long tail problem.","locator":"paragraph 10","url":"https://garymarcus.substack.com/p/dear-elon-musk-here-are-five-things","date":"2022-05-31","document_title":"Dear Elon Musk, here are five things you might want to consider about AGI","observed_at":"2026-10-28T22:49:04Z"},"criteria":{"from_speaker":["easy to convince yourself","AI problems are much easier than they are actually are","because of the long tail problem"],"our_interpretation":["The ordinary reading is that success on frequently encountered cases can cause people to underestimate the difficulty of making AI reliable across uncommon cases.","Underestimating development difficulty is distinct from trusting an existing system too much.","The claim describes a general mechanism without a deadline or a specified population."],"support_condition":"Studies or documented causal analyses show that exposure dominated by successful common cases produces underestimation of AI problem difficulty, and connect that misjudgment to overlooked rare cases.","contradiction_condition":"Well-matched studies show that this exposure does not produce the proposed underestimation, or that another factor accounts for the effect while rare-case exposure has no material influence."},"axis":"causal_assessment","outcome":"unresolved","horizon_state":"not_applicable","evaluation_as_of":"2026-10-28","rationale":"Research on overtrust is consistent with part of Marcus's explanation: repeated successful interactions can encourage excessive confidence. But that does not establish that the long-tail problem causes people to underestimate AI engineering difficulty. The searches did not locate evidence that directly settles that causal claim.","limitations":["The closest empirical evidence uses a simulated robot task.","The speaker supplies neither a population nor a measure of perceived difficulty."],"evidence":[{"id":"e1","relation":"context","excerpt":"System failure is a rare occurrence.","locator":"General paradigm of overtrust, page 6","url":"https://www.mmi.ifi.lmu.de/pubdb/publications/pub/ullrich2021DevelopmentOfOvertrust/ullrich2021DevelopmentOfOvertrust.pdf","publisher":"Frontiers in Robotics and AI; institutional copy hosted by LMU Munich","event_date":"2021","available_at":"2021-04-13","explanation":"The study models successful interactions followed by uncommon failures and reports increasing trust during repeated successes. Its simulated pet-feeding task measures trust and checking behavior, not beliefs about how difficult AI engineering problems are."}],"what_would_resolve":"A study comparing common-case-only exposure with exposure including rare failures, measuring resulting estimates of AI development difficulty or remaining work against independently established performance limitations.","retrospective":true,"plain_reading":true},"payload_hash":"871b2d9884d1f58427c1cac370cd73ec3b37fb0ad201893a30765c42676580c8","policy_version":"2026-10-28.7","recorded_at":"2026-10-29T06:56:13Z","drafter":{"provider":"openai","model":"gpt-6-astra"},"reviewers":[{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-dear-elon-musk-here-are-five-things-acced4-4x-review-anthropic-1"},{"provider":"openai","model":"gpt-6-astra","run_id":"marcus-dear-elon-musk-here-are-five-things-acced4-4x-review-openai-1"}],"final_text":{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-dear-elon-musk-here-are-five-things-acced4-4x-final-anthropic-1"},"status":"current","notice":null,"superseded_by":null,"evidence_meta":[{"id":"e1","source_type":"peer_reviewed","audited":null,"derived":null,"metric":null}],"public_from":null,"public_until":null},"history":[],"review":{"review_due":false,"triggers":[],"last_completed_review_at":"2026-10-29T06:56:13Z","next_review_due_at":null,"horizon_now":"not_applicable"},"event_ids":["marcus-dear-elon-musk-here-are-five-things-d56a36-assessment-published-a1"],"last_substantive_change_at":"2026-10-29T17:00:18Z"}},{"url":"https://punditclaims.ai/claims/marcus-dear-elon-musk-here-are-five-things-ed6b53","standing":"current","assessment_label":"Supported","versions":[{"assessment_id":"marcus-dear-elon-musk-here-are-five-things-ed6b53-a1","status":"current","url":"https://punditclaims.ai/claims/marcus-dear-elon-musk-here-are-five-things-ed6b53#assessment-marcus-dear-elon-musk-here-are-five-things-ed6b53-a1"}],"claim":{"claim_id":"marcus-dear-elon-musk-here-are-five-things-ed6b53","subject_id":"gary-marcus","revision":1,"content_hash":"967264432a6b0ca36acbe5f2bf2fa4eb02af6b8c33dfbcba70f0cc11f9658c51","normalized_proposition":"Tesla will not ship level 5 self-driving cars in 2022 or 2023.","kind":"forecast","target":"Tesla","conditions":[],"original_horizon_text":"this year or next","topic_tags":["capabilities","reliability"],"concerns_providers":[],"selection_stratum":"declared-lead","occurrences":[{"id":"marcus-dear-elon-musk-here-are-five-things-ed6b53-o1","statement_date":"2022-05-31","statement_date_precision":"day","excerpt":"And it’s why I absolutely guarantee you won’t be shipping level 5 self-driving cars this year or next, no matter what you might have told Chris Anderson at TED.","locator":"paragraph 16","document":{"url":"https://garymarcus.substack.com/p/dear-elon-musk-here-are-five-things","title":"Dear Elon Musk, here are five things you might want to consider about AGI","publisher":"Marcus on AI","published_at":"2022-05-31T18:32:41Z","date_uncertain":false,"date_provenance":"JSON-LD datePublished"},"observed_at":"2026-10-28T22:49:04Z","observed_via":"live","archive_url":null,"access_state":"full_public","snapshot_hash":"d64c073bb595b86ed0d5a52382668c601d5a8e7009353d00f0e54061bdd4f0f7","receipts":1}],"first_statement_date":"2022-05-31","related":[],"current":{"assessment_id":"marcus-dear-elon-musk-here-are-five-things-ed6b53-a1","payload":{"claim_id":"marcus-dear-elon-musk-here-are-five-things-ed6b53","subject_id":"gary-marcus","normalized_proposition":"Tesla will not ship level 5 self-driving cars in 2022 or 2023.","kind":"forecast","conditions":[],"original_horizon_text":"this year or next","statement":{"excerpt":"And it’s why I absolutely guarantee you won’t be shipping level 5 self-driving cars this year or next, no matter what you might have told Chris Anderson at TED.","locator":"paragraph 16","url":"https://garymarcus.substack.com/p/dear-elon-musk-here-are-five-things","date":"2022-05-31","document_title":"Dear Elon Musk, here are five things you might want to consider about AGI","observed_at":"2026-10-28T22:49:04Z"},"criteria":{"from_speaker":["absolutely guarantee","won’t be shipping","level 5 self-driving cars","this year or next"],"our_interpretation":["The addressee and context identify Tesla as the manufacturer.","This year or next means 2022 or 2023.","Shipping means supplying customers with functioning Level 5 capability, including activation through software updates.","Level 5 means full driving automation without human supervision across driver-manageable road conditions. Selling hardware intended for future autonomy does not qualify."],"support_condition":"Tesla's documented customer offerings remained below Level 5 throughout 2022 and 2023.","contradiction_condition":"Tesla supplied at least one customer with a vehicle offering functioning Level 5 automation during 2022 or 2023."},"axis":"forecast_outcome","outcome":"supported","horizon_state":"elapsed","evaluation_as_of":"2026-10-28","rationale":"Tesla's annual filings for both forecast years describe its available systems as driver assistance requiring a responsible human driver. These documented capabilities fall below Level 5. The deadline has elapsed, and the records support Marcus's prediction.","limitations":["The assessment concerns delivered capability, not whether Tesla sold an option named Full Self-Driving.","Annual disclosures establish the product offering rather than independently testing every vehicle.","This outcome does not establish Marcus's proposed explanation for the delay."],"evidence":[{"id":"e1","relation":"supports","excerpt":"advanced driver assist systems","locator":"Item 1, Self-Driving Development and Artificial Intelligence","url":"https://www.annualreports.com/HostedData/AnnualReportArchive/t/NASDAQ_TSLA_2022.pdf","publisher":"Tesla, Inc., mirrored by AnnualReports.com","event_date":"2022-12-31","available_at":"2023-01-31","explanation":"The annual filing describes Tesla's offerings as driver assistance and says the driver remains responsible for controlling the vehicle. That capability does not meet Level 5."},{"id":"e2","relation":"supports","excerpt":"advanced driver assist systems","locator":"Item 1, Self-Driving Development and Artificial Intelligence, page 5","url":"https://s203.q4cdn.com/655419027/files/doc_financials/2023/ar/tsla-20231231-gen.pdf","publisher":"Tesla, Inc.","event_date":"2023-12-31","available_at":"2024-01-29","explanation":"The following annual filing still describes the delivered systems as driver assistance, with the driver responsible for control. Together, the filings support non-delivery of Level 5 capability through the deadline."}],"what_would_resolve":null,"retrospective":true,"plain_reading":true},"payload_hash":"6bc5b8fabe289c22eddf037ac43e6d0347c8fe038c1052b9269ed52f9257853f","policy_version":"2026-10-28.7","recorded_at":"2026-10-29T06:56:13Z","drafter":{"provider":"openai","model":"gpt-6-astra"},"reviewers":[{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-dear-elon-musk-here-are-five-things-acced4-4x-review-anthropic-1"},{"provider":"openai","model":"gpt-6-astra","run_id":"marcus-dear-elon-musk-here-are-five-things-acced4-4x-review-openai-1"}],"final_text":{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-dear-elon-musk-here-are-five-things-acced4-4x-final-anthropic-1"},"status":"current","notice":null,"superseded_by":null,"evidence_meta":[{"id":"e1","source_type":"filing","audited":null,"derived":null,"metric":null},{"id":"e2","source_type":"filing","audited":null,"derived":null,"metric":null}],"public_from":null,"public_until":null},"history":[],"review":{"review_due":false,"triggers":[],"last_completed_review_at":"2026-10-29T06:56:13Z","next_review_due_at":null,"horizon_now":"elapsed"},"event_ids":["marcus-dear-elon-musk-here-are-five-things-ed6b53-assessment-published-a1"],"last_substantive_change_at":"2026-10-29T17:00:18Z"}},{"url":"https://punditclaims.ai/claims/marcus-dear-elon-musk-here-are-five-things-f025f6","standing":"current","assessment_label":"Unresolved","versions":[{"assessment_id":"marcus-dear-elon-musk-here-are-five-things-f025f6-a1","status":"current","url":"https://punditclaims.ai/claims/marcus-dear-elon-musk-here-are-five-things-f025f6#assessment-marcus-dear-elon-musk-here-are-five-things-f025f6-a1"}],"claim":{"claim_id":"marcus-dear-elon-musk-here-are-five-things-f025f6","subject_id":"gary-marcus","revision":1,"content_hash":"3f77b22fdbbd6cbc9d8049b8a41863f6862e09826baa1e21c38b830bcd743b12","normalized_proposition":"In 2029, AI will not be able to watch a movie and accurately describe what is going on.","kind":"forecast","target":"AI","conditions":[],"original_horizon_text":"In 2029","topic_tags":["capabilities","agi-timelines"],"concerns_providers":[],"selection_stratum":"declared-lead","occurrences":[{"id":"marcus-dear-elon-musk-here-are-five-things-f025f6-o1","statement_date":"2022-05-31","statement_date_precision":"day","excerpt":"In 2029, AI will not be able to watch a movie and tell you accurately what is going on (what I called the comprehension challenge in The New Yorker, in 2014).","locator":"prediction 1 of 5","document":{"url":"https://garymarcus.substack.com/p/dear-elon-musk-here-are-five-things","title":"Dear Elon Musk, here are five things you might want to consider about AGI","publisher":"Marcus on AI","published_at":"2022-05-31T18:32:41Z","date_uncertain":false,"date_provenance":"JSON-LD datePublished"},"observed_at":"2026-10-28T22:49:04Z","observed_via":"live","archive_url":null,"access_state":"full_public","snapshot_hash":"d64c073bb595b86ed0d5a52382668c601d5a8e7009353d00f0e54061bdd4f0f7","receipts":1}],"first_statement_date":"2022-05-31","related":[],"current":{"assessment_id":"marcus-dear-elon-musk-here-are-five-things-f025f6-a1","payload":{"claim_id":"marcus-dear-elon-musk-here-are-five-things-f025f6","subject_id":"gary-marcus","normalized_proposition":"In 2029, AI will not be able to watch a movie and accurately describe what is going on.","kind":"forecast","conditions":[],"original_horizon_text":"In 2029","statement":{"excerpt":"In 2029, AI will not be able to watch a movie and tell you accurately what is going on (what I called the comprehension challenge in The New Yorker, in 2014).","locator":"prediction 1 of 5","url":"https://garymarcus.substack.com/p/dear-elon-musk-here-are-five-things","date":"2022-05-31","document_title":"Dear Elon Musk, here are five things you might want to consider about AGI","observed_at":"2026-10-28T22:49:04Z"},"criteria":{"from_speaker":["In 2029","AI will not be able to watch a movie","tell you accurately what is going on","Who are the characters?","What are their conflicts and motivations?"],"our_interpretation":["Read this as a forecast about general movie comprehension, including events, characters, conflicts and inferred motivations.","A suitable test uses varied full movies and open-ended explanations, with controls against answering from memorized summaries. A hand-picked clip or isolated correct answer is insufficient.","Accuracy means ordinary competent comprehension, allowing reasonable differences in interpretation; it does not require perfect recall of every detail.","Use 2029-12-31 as the latest deadline. An earlier reproducible demonstration of the complete capability can refute the forecast under the early-resolution rule.","Assess this individual forecast independently of the surrounding wager about passing three challenges."],"support_condition":"Evidence covering the end of 2029 shows that leading available systems still cannot accurately explain full movies across a representative evaluation of events, characters, conflicts and motivations.","contradiction_condition":"At least one AI system demonstrates accurate full-movie comprehension across varied movies under a credible evaluation that tests the stated narrative elements."},"axis":"forecast_outcome","outcome":"unresolved","horizon_state":"not_due","evaluation_as_of":"2026-10-28","rationale":"The deadline is still ahead. The inspected movie-understanding research does not provide a decisive early demonstration of the complete task. Its reported shortcomings are evidence about tested systems, not proof that the forecast will hold in 2029.","limitations":["The search does not establish that no qualifying system exists.","There is no speaker-specified accuracy threshold.","Benchmark results cover selected models and films."],"evidence":[{"id":"e1","relation":"context","excerpt":"a substantial gap","locator":"Abstract, fourth sentence","url":"https://arxiv.org/abs/2506.06275v2","publisher":"arXiv; Emmanouil Zaranis and collaborators","event_date":"2026","available_at":"2026-10-19","explanation":"The study reports persistent difficulties in narrative understanding and improvements when relevant visual content is selected. It does not establish the complete capability required for early contradiction, and its results cannot settle performance in 2029."}],"what_would_resolve":"A reproducible full-movie evaluation meeting the early-resolution rule would contradict the forecast. Otherwise, evaluate available systems through 2029 using a disclosed rubric covering events, characters, conflicts and motivations.","retrospective":false,"plain_reading":true},"payload_hash":"ebe9e0119daf8367a5f1a9d3b11931bf53ef42252e546684809c81b984c09dcd","policy_version":"2026-10-28.7","recorded_at":"2026-10-29T06:55:39Z","drafter":{"provider":"openai","model":"gpt-6-astra"},"reviewers":[{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-dear-elon-musk-here-are-five-things-f025f6-2x-review-anthropic-1"},{"provider":"openai","model":"gpt-6-astra","run_id":"marcus-dear-elon-musk-here-are-five-things-f025f6-2x-review-openai-1"}],"final_text":{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-dear-elon-musk-here-are-five-things-f025f6-2x-final-anthropic-1"},"status":"current","notice":null,"superseded_by":null,"evidence_meta":[{"id":"e1","source_type":"preprint","audited":null,"derived":null,"metric":null}],"public_from":null,"public_until":null},"history":[],"review":{"review_due":false,"triggers":[],"last_completed_review_at":"2026-10-29T06:55:39Z","next_review_due_at":null,"horizon_now":"not_due"},"event_ids":["marcus-dear-elon-musk-here-are-five-things-f025f6-assessment-published-a1"],"last_substantive_change_at":"2026-10-29T17:00:18Z"}},{"url":"https://punditclaims.ai/claims/marcus-dear-elon-musk-here-are-five-things-fcf450","standing":"current","assessment_label":"Can’t be checked","versions":[{"assessment_id":"marcus-dear-elon-musk-here-are-five-things-fcf450-a1","status":"current","url":"https://punditclaims.ai/claims/marcus-dear-elon-musk-here-are-five-things-fcf450#assessment-marcus-dear-elon-musk-here-are-five-things-fcf450-a1"}],"claim":{"claim_id":"marcus-dear-elon-musk-here-are-five-things-fcf450","subject_id":"gary-marcus","revision":1,"content_hash":"432bd40fbf5bbf86ce148c9296d666d6cdb4797605f74f23a3771ee3d00e3e06","normalized_proposition":"Deep learning lacks tools for verifiable code, and AGI by 2029 would probably require solving that soon.","kind":"conditional_forecast","target":"AGI by 2029","conditions":[{"text":"if AGI is to arrive by 2029","type":"antecedent","from_source":true}],"original_horizon_text":"soon","topic_tags":["agi-timelines","reliability"],"concerns_providers":[],"selection_stratum":"declared-lead","occurrences":[{"id":"marcus-dear-elon-musk-here-are-five-things-fcf450-o1","statement_date":"2022-05-31","statement_date_precision":"day","excerpt":"Things like programming verification might eventually help, but again, in deep learning we don’t yet have the tools for writing verifiable code. If you are to win the bet, we are probably going to have to solve that problem too, and soon.","locator":"paragraph 31","document":{"url":"https://garymarcus.substack.com/p/dear-elon-musk-here-are-five-things","title":"Dear Elon Musk, here are five things you might want to consider about AGI","publisher":"Marcus on AI","published_at":"2022-05-31T18:32:41Z","date_uncertain":false,"date_provenance":"JSON-LD datePublished"},"observed_at":"2026-10-28T22:49:04Z","observed_via":"live","archive_url":null,"access_state":"full_public","snapshot_hash":"d64c073bb595b86ed0d5a52382668c601d5a8e7009353d00f0e54061bdd4f0f7","receipts":1}],"first_statement_date":"2022-05-31","related":[],"current":{"assessment_id":"marcus-dear-elon-musk-here-are-five-things-fcf450-a1","payload":{"claim_id":"marcus-dear-elon-musk-here-are-five-things-fcf450","subject_id":"gary-marcus","normalized_proposition":"Deep learning lacks tools for verifiable code, and AGI by 2029 would probably require solving that soon.","kind":"conditional_forecast","conditions":["if AGI is to arrive by 2029"],"original_horizon_text":"soon","statement":{"excerpt":"Things like programming verification might eventually help, but again, in deep learning we don’t yet have the tools for writing verifiable code. If you are to win the bet, we are probably going to have to solve that problem too, and soon.","locator":"paragraph 31","url":"https://garymarcus.substack.com/p/dear-elon-musk-here-are-five-things","date":"2022-05-31","document_title":"Dear Elon Musk, here are five things you might want to consider about AGI","observed_at":"2026-10-28T22:49:04Z"},"criteria":{"from_speaker":["if AGI is to arrive by 2029","soon"],"our_interpretation":[],"support_condition":"Not checked against evidence: both screens found that public evidence could not settle this claim.","contradiction_condition":"Not checked against evidence: both screens found that public evidence could not settle this claim."},"axis":"not_applicable","outcome":"not_adjudicable","horizon_state":"not_applicable","evaluation_as_of":"2026-10-28","rationale":"Not checked against evidence. Two independent screens, one from each provider, both found this stated as a chance or a possibility, or at medium or low confidence.","limitations":[],"evidence":[],"what_would_resolve":null,"retrospective":true,"uncheckable":["probability_language"]},"payload_hash":"1ccda1859a18749416a0ee80758ee6b3e95865ddfc9fb0f8ac71c43961962612","policy_version":"2026-10-28.7","recorded_at":"2026-10-29T01:10:54Z","drafter":{"provider":"anthropic","model":"claude-opus-5-5"},"reviewers":[{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-dear-elon-musk-here-are-five-things-screen-anthropic-1"},{"provider":"openai","model":"gpt-6-astra","run_id":"marcus-dear-elon-musk-here-are-five-things-screen-openai-1"}],"final_text":null,"status":"current","notice":null,"superseded_by":null,"evidence_meta":[],"public_from":null,"public_until":null},"history":[],"review":{"review_due":false,"triggers":[],"last_completed_review_at":"2026-10-29T01:10:54Z","next_review_due_at":null,"horizon_now":"not_applicable"},"event_ids":["marcus-dear-elon-musk-here-are-five-things-fcf450-assessment-published-a1"],"last_substantive_change_at":"2026-10-29T17:00:18Z"}},{"url":"https://punditclaims.ai/claims/marcus-dear-elon-musk-here-are-five-things-ffdca9","standing":"current","assessment_label":"Supported","versions":[{"assessment_id":"marcus-dear-elon-musk-here-are-five-things-ffdca9-a2","status":"current","url":"https://punditclaims.ai/claims/marcus-dear-elon-musk-here-are-five-things-ffdca9#assessment-marcus-dear-elon-musk-here-are-five-things-ffdca9-a2"}],"claim":{"claim_id":"marcus-dear-elon-musk-here-are-five-things-ffdca9","subject_id":"gary-marcus","revision":1,"content_hash":"8b842ed5a0924c6ee36960c80a17807aad851b929caacd44a946c43bc4eeaf23","normalized_proposition":"Outliers remain a major AI problem with no known robust solution.","kind":"observation","target":"outliers in AI","conditions":[],"original_horizon_text":null,"topic_tags":["reliability"],"concerns_providers":[],"selection_stratum":"declared-lead","occurrences":[{"id":"marcus-dear-elon-musk-here-are-five-things-ffdca9-o1","statement_date":"2022-05-31","statement_date_precision":"day","excerpt":"But you and I both know that they continue to be a major problem that as yet has no known robust solution.","locator":"paragraph 17","document":{"url":"https://garymarcus.substack.com/p/dear-elon-musk-here-are-five-things","title":"Dear Elon Musk, here are five things you might want to consider about AGI","publisher":"Marcus on AI","published_at":"2022-05-31T18:32:41Z","date_uncertain":false,"date_provenance":"JSON-LD datePublished"},"observed_at":"2026-10-28T22:49:04Z","observed_via":"live","archive_url":null,"access_state":"full_public","snapshot_hash":"d64c073bb595b86ed0d5a52382668c601d5a8e7009353d00f0e54061bdd4f0f7","receipts":1}],"first_statement_date":"2022-05-31","related":[],"current":{"assessment_id":"marcus-dear-elon-musk-here-are-five-things-ffdca9-a2","payload":{"claim_id":"marcus-dear-elon-musk-here-are-five-things-ffdca9","subject_id":"gary-marcus","normalized_proposition":"Outliers remain a major AI problem with no known robust solution.","kind":"observation","conditions":[],"original_horizon_text":null,"statement":{"excerpt":"But you and I both know that they continue to be a major problem that as yet has no known robust solution.","locator":"paragraph 17","url":"https://garymarcus.substack.com/p/dear-elon-musk-here-are-five-things","date":"2022-05-31","document_title":"Dear Elon Musk, here are five things you might want to consider about AGI","observed_at":"2026-10-28T22:49:04Z"},"criteria":{"from_speaker":["they continue to be a major problem","as yet","no known robust solution"],"our_interpretation":["They refers to outliers and unexpected circumstances discussed immediately before this sentence.","The ordinary reading concerns reliable handling of unfamiliar inputs and distribution shifts across applications. It does not deny effective methods for particular cases.","Because academic research has no fixed reporting quarter, the observation is assessed against the current 2022 research record through May 31. Earlier research supplies background.","A robust solution means broadly demonstrated reliability, not perfect performance on every mathematically possible input."],"support_condition":"Contemporaneous research documents substantial failures under unexpected inputs or distribution shifts, with available remedies remaining limited rather than constituting a broadly validated solution.","contradiction_condition":"Research available by the post establishes a known method that reliably resolves these failures across representative applications and unfamiliar conditions."},"axis":"historical_observation","outcome":"supported","horizon_state":"not_applicable","evaluation_as_of":"2026-10-28","rationale":"The February 16, 2022 survey identifies continuing obstacles to handling distribution shifts in graph data, including differences in graph structure, tasks and shift types [e1]. The February 21, 2022 experiments find that fine-tuning can worsen accuracy under distribution shift across ten tested datasets, while a two-stage training method improves those results [e2]. These findings support the observation, as of May 31, 2022, that unfamiliar inputs remained a substantial unresolved problem: they document obstacles and useful remedies without establishing a broadly reliable solution [e1; e2].","limitations":["The absence of a broadly validated solution is an inference from the inspected public research record, not proof that every possible solution was absent.","The graph survey and selected distribution-shift datasets do not cover every application or kind of outlier.","The evidence does not establish that outlier problems are inherently unsolvable or validate every surrounding criticism of deep learning."],"evidence":[{"id":"e1","relation":"supports","excerpt":"highly non-trivial","locator":"Section 1, Introduction","url":"https://arxiv.org/pdf/2202.07987v1","publisher":"arXiv; Haoyang Li and colleagues","event_date":"2022-02-16","available_at":"2022-02-16","explanation":"The survey identifies unresolved obstacles involving graph structure, differing tasks, and different kinds of distribution shift. It also describes progress, supporting the distinction between useful remedies and a general solution."},{"id":"e2","relation":"supports","excerpt":"tradeoff","locator":"Abstract","url":"https://arxiv.org/abs/2202.10054v1","publisher":"arXiv; Ananya Kumar and colleagues","event_date":"2022-02-21","available_at":"2022-02-21","explanation":"Experiments across ten distribution-shift datasets show that improving ordinary accuracy can reduce accuracy after a shift. The proposed two-stage training method improves results, but the paper does not establish a general solution to unexpected inputs."},{"id":"e3","relation":"context","excerpt":"open challenges","locator":"Abstract, final sentence","url":"https://arxiv.org/abs/2110.11334v1","publisher":"arXiv; Jingkang Yang and colleagues","event_date":"2021-10-21","available_at":"2021-10-21","explanation":"This earlier survey covers anomaly, novelty, open-set, outlier, and out-of-distribution detection. It supplies background on the breadth of the problem and the existence of many partial approaches."}],"what_would_resolve":null,"retrospective":true,"plain_reading":true},"payload_hash":"e9434f4c415086f41afe30f709cb72a7e300304f2202dbbc1a2e9719ad7dccb5","policy_version":"2026-10-28.7","recorded_at":"2026-10-29T07:09:09Z","drafter":{"provider":"openai","model":"gpt-6-astra"},"reviewers":[{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-dear-elon-musk-here-are-five-things-f025f6-2x-review-anthropic-1"},{"provider":"openai","model":"gpt-6-astra","run_id":"marcus-dear-elon-musk-here-are-five-things-f025f6-2x-review-openai-1"}],"final_text":{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-dear-elon-musk-here-are-five-things-31b9db-2x-final-anthropic-1"},"status":"current","notice":null,"superseded_by":null,"evidence_meta":[{"id":"e1","source_type":"preprint","audited":null,"derived":null,"metric":null},{"id":"e2","source_type":"preprint","audited":null,"derived":null,"metric":null},{"id":"e3","source_type":"preprint","audited":null,"derived":null,"metric":null}],"public_from":null,"public_until":null},"history":[],"review":{"review_due":false,"triggers":[],"last_completed_review_at":"2026-10-29T07:09:09Z","next_review_due_at":null,"horizon_now":"not_applicable"},"event_ids":["marcus-dear-elon-musk-here-are-five-things-ffdca9-assessment-published-a2"],"last_substantive_change_at":"2026-10-29T17:00:18Z"}},{"url":"https://punditclaims.ai/claims/marcus-horse-rides-astronaut-1e0fda","standing":"current","assessment_label":"Can’t be checked","versions":[{"assessment_id":"marcus-horse-rides-astronaut-1e0fda-a1","status":"current","url":"https://punditclaims.ai/claims/marcus-horse-rides-astronaut-1e0fda#assessment-marcus-horse-rides-astronaut-1e0fda-a1"}],"claim":{"claim_id":"marcus-horse-rides-astronaut-1e0fda","subject_id":"gary-marcus","revision":2,"content_hash":"280279d00436a50549932e8a3c19eef6f0b40d02fb248546544c7fbb6f9bb1fe","normalized_proposition":"From the few clues that have filtered out, there is enormous reason to be skeptical of Google's claim that Imagen has a deep level of language understanding, though without access to the system skeptics can only make guesses.","kind":"normative","target":"Imagen","conditions":[{"text":"from the few clues that have filtered out","type":"qualifier","from_source":true},{"text":"Google won't allow skeptics to try the system, so we can only make guesses","type":"qualifier","from_source":true}],"original_horizon_text":null,"topic_tags":["capabilities"],"concerns_providers":[],"selection_stratum":"declared-lead","occurrences":[{"id":"marcus-horse-rides-astronaut-1e0fda-o1","statement_date":"2022-05-28","statement_date_precision":"day","excerpt":"Against a long history of overclaiming in AI, such strong statements demand an awful lot of evidence. Because Google won’t allow skeptics to try the system, we can only make guesses. But from the few clues that have filtered out, there is enormous reason for skepticism.","locator":"paragraph 31","document":{"url":"https://garymarcus.substack.com/p/horse-rides-astronaut","title":"Horse rides astronaut","publisher":"Marcus on AI","published_at":"2022-05-28T14:29:44Z","date_uncertain":false,"date_provenance":"JSON-LD datePublished"},"observed_at":"2026-10-29T00:05:21Z","observed_via":"live","archive_url":null,"access_state":"full_public","snapshot_hash":"e4a8233fec988ccb75c5e8a45ac1ad325ab13313a43cd661957d6e57ebb49d48","receipts":1}],"first_statement_date":"2022-05-28","related":[],"current":{"assessment_id":"marcus-horse-rides-astronaut-1e0fda-a1","payload":{"claim_id":"marcus-horse-rides-astronaut-1e0fda","subject_id":"gary-marcus","normalized_proposition":"From the few clues that have filtered out, there is enormous reason to be skeptical of Google's claim that Imagen has a deep level of language understanding, though without access to the system skeptics can only make guesses.","kind":"normative","conditions":["from the few clues that have filtered out","Google won't allow skeptics to try the system, so we can only make guesses"],"original_horizon_text":null,"statement":{"excerpt":"Against a long history of overclaiming in AI, such strong statements demand an awful lot of evidence. Because Google won’t allow skeptics to try the system, we can only make guesses. But from the few clues that have filtered out, there is enormous reason for skepticism.","locator":"paragraph 31","url":"https://garymarcus.substack.com/p/horse-rides-astronaut","date":"2022-05-28","document_title":"Horse rides astronaut","observed_at":"2026-10-29T00:05:21Z"},"criteria":{"from_speaker":["from the few clues that have filtered out","Google won't allow skeptics to try the system, so we can only make guesses"],"our_interpretation":[],"support_condition":"Not checked against evidence: both screens found that public evidence could not settle this claim.","contradiction_condition":"Not checked against evidence: both screens found that public evidence could not settle this claim."},"axis":"not_applicable","outcome":"not_adjudicable","horizon_state":"not_applicable","evaluation_as_of":"2026-10-28","rationale":"Not checked against evidence. Two independent screens, one from each provider, both found this a recommendation or a value judgment.","limitations":[],"evidence":[],"what_would_resolve":null,"retrospective":true,"uncheckable":["value_judgment"]},"payload_hash":"102741a784a3d69bd6daee21f92b0382627a56cac5c5fdbc6ecf5d4a4cd19307","policy_version":"2026-10-28.7","recorded_at":"2026-10-29T02:32:05Z","drafter":{"provider":"anthropic","model":"claude-opus-5-5"},"reviewers":[{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-horse-rides-astronaut-screen-anthropic-2"},{"provider":"openai","model":"gpt-6-astra","run_id":"marcus-horse-rides-astronaut-screen-openai-2"}],"final_text":null,"status":"current","notice":null,"superseded_by":null,"evidence_meta":[],"public_from":null,"public_until":null},"history":[],"review":{"review_due":false,"triggers":[],"last_completed_review_at":"2026-10-29T02:32:05Z","next_review_due_at":null,"horizon_now":"not_applicable"},"event_ids":["marcus-horse-rides-astronaut-1e0fda-assessment-published-a1"],"last_substantive_change_at":"2026-10-29T17:00:18Z"}},{"url":"https://punditclaims.ai/claims/marcus-horse-rides-astronaut-21930b","standing":"current","assessment_label":"Can’t be checked","versions":[{"assessment_id":"marcus-horse-rides-astronaut-21930b-a1","status":"current","url":"https://punditclaims.ai/claims/marcus-horse-rides-astronaut-21930b#assessment-marcus-horse-rides-astronaut-21930b-a1"}],"claim":{"claim_id":"marcus-horse-rides-astronaut-21930b","subject_id":"gary-marcus","revision":2,"content_hash":"769d1bd2a4e67f11b26c7a5bdad290a80d97a8dacc50a7366b0ac558c65a6d94","normalized_proposition":"Imagen's failure on 'a horse riding an astronaut' casts doubt on its success with 'an astronaut riding a horse' and suggests that this success comes not from extracting word meanings and combining them by syntax, but from something more holistic and approximate, a bit like keyword matching.","kind":"causal","target":"Imagen","conditions":[{"text":"casts doubt / suggests","type":"qualifier","from_source":true}],"original_horizon_text":null,"topic_tags":["capabilities"],"concerns_providers":[],"selection_stratum":"declared-lead","occurrences":[{"id":"marcus-horse-rides-astronaut-21930b-o1","statement_date":"2022-05-28","statement_date_precision":"day","excerpt":"it casts doubt on “an astronaut riding a horse”, and suggests success there is not because the network knows how to extract meanings from individual words","locator":"paragraph 91","document":{"url":"https://garymarcus.substack.com/p/horse-rides-astronaut","title":"Horse rides astronaut","publisher":"Marcus on AI","published_at":"2022-05-28T14:29:44Z","date_uncertain":false,"date_provenance":"JSON-LD datePublished"},"observed_at":"2026-10-29T00:05:21Z","observed_via":"live","archive_url":null,"access_state":"full_public","snapshot_hash":"e4a8233fec988ccb75c5e8a45ac1ad325ab13313a43cd661957d6e57ebb49d48","receipts":1}],"first_statement_date":"2022-05-28","related":[],"current":{"assessment_id":"marcus-horse-rides-astronaut-21930b-a1","payload":{"claim_id":"marcus-horse-rides-astronaut-21930b","subject_id":"gary-marcus","normalized_proposition":"Imagen's failure on 'a horse riding an astronaut' casts doubt on its success with 'an astronaut riding a horse' and suggests that this success comes not from extracting word meanings and combining them by syntax, but from something more holistic and approximate, a bit like keyword matching.","kind":"causal","conditions":["casts doubt / suggests"],"original_horizon_text":null,"statement":{"excerpt":"it casts doubt on “an astronaut riding a horse”, and suggests success there is not because the network knows how to extract meanings from individual words","locator":"paragraph 91","url":"https://garymarcus.substack.com/p/horse-rides-astronaut","date":"2022-05-28","document_title":"Horse rides astronaut","observed_at":"2026-10-29T00:05:21Z"},"criteria":{"from_speaker":["casts doubt / suggests"],"our_interpretation":[],"support_condition":"Not checked against evidence: both screens found that public evidence could not settle this claim.","contradiction_condition":"Not checked against evidence: both screens found that public evidence could not settle this claim."},"axis":"not_applicable","outcome":"not_adjudicable","horizon_state":"not_applicable","evaluation_as_of":"2026-10-28","rationale":"Not checked against evidence. Two independent screens, one from each provider, both found this stated as a chance or a possibility, or at medium or low confidence.","limitations":[],"evidence":[],"what_would_resolve":null,"retrospective":true,"uncheckable":["probability_language"]},"payload_hash":"f8330d842eab0545918b2dd76e094b225c2890e6e67bbd4178d27e2a33238524","policy_version":"2026-10-28.7","recorded_at":"2026-10-29T02:32:05Z","drafter":{"provider":"anthropic","model":"claude-opus-5-5"},"reviewers":[{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-horse-rides-astronaut-screen-anthropic-2"},{"provider":"openai","model":"gpt-6-astra","run_id":"marcus-horse-rides-astronaut-screen-openai-2"}],"final_text":null,"status":"current","notice":null,"superseded_by":null,"evidence_meta":[],"public_from":null,"public_until":null},"history":[],"review":{"review_due":false,"triggers":[],"last_completed_review_at":"2026-10-29T02:32:05Z","next_review_due_at":null,"horizon_now":"not_applicable"},"event_ids":["marcus-horse-rides-astronaut-21930b-assessment-published-a1"],"last_substantive_change_at":"2026-10-29T17:00:18Z"}},{"url":"https://punditclaims.ai/claims/marcus-horse-rides-astronaut-4f1bbd","standing":"current","assessment_label":"Supported","versions":[{"assessment_id":"marcus-horse-rides-astronaut-4f1bbd-a1","status":"current","url":"https://punditclaims.ai/claims/marcus-horse-rides-astronaut-4f1bbd#assessment-marcus-horse-rides-astronaut-4f1bbd-a1"}],"claim":{"claim_id":"marcus-horse-rides-astronaut-4f1bbd","subject_id":"gary-marcus","revision":1,"content_hash":"2b29b617b3d214832bc4d7b1c6ce03c0ef9d227b91d2e1dfae32061760104f91","normalized_proposition":"Imagen lacks deep understanding of language despite Google's PR.","kind":"observation","target":"Imagen","conditions":[],"original_horizon_text":null,"topic_tags":["capabilities"],"concerns_providers":[],"selection_stratum":"declared-lead","occurrences":[{"id":"marcus-horse-rides-astronaut-4f1bbd-o1","statement_date":"2022-05-28","statement_date_precision":"day","excerpt":"That’s where the deep understanding of language should live—but, despite the PR, it isn’t there.","locator":"paragraph 83","document":{"url":"https://garymarcus.substack.com/p/horse-rides-astronaut","title":"Horse rides astronaut","publisher":"Marcus on AI","published_at":"2022-05-28T14:29:44Z","date_uncertain":false,"date_provenance":"JSON-LD datePublished"},"observed_at":"2026-10-29T00:05:21Z","observed_via":"live","archive_url":null,"access_state":"full_public","snapshot_hash":"e4a8233fec988ccb75c5e8a45ac1ad325ab13313a43cd661957d6e57ebb49d48","receipts":1}],"first_statement_date":"2022-05-28","related":[],"current":{"assessment_id":"marcus-horse-rides-astronaut-4f1bbd-a1","payload":{"claim_id":"marcus-horse-rides-astronaut-4f1bbd","subject_id":"gary-marcus","normalized_proposition":"Imagen lacks deep understanding of language despite Google's PR.","kind":"observation","conditions":[],"original_horizon_text":null,"statement":{"excerpt":"That’s where the deep understanding of language should live—but, despite the PR, it isn’t there.","locator":"paragraph 83","url":"https://garymarcus.substack.com/p/horse-rides-astronaut","date":"2022-05-28","document_title":"Horse rides astronaut","observed_at":"2026-10-29T00:05:21Z"},"criteria":{"from_speaker":["deep understanding of language","despite the PR","it isn’t there"],"our_interpretation":["In this context, the ordinary reading is reliable understanding of who does what to whom in compositional image instructions.","This is a behavioral claim about Imagen as a complete system, not a claim about consciousness or the absence of every language capability.","I use the original model's public evaluation window, from its May 23 announcement through the May 28 post, because the sentence describes an ongoing capability without specifying a reporting period.","Neither occasional mistakes nor occasional successes alone settle the claim; evidence should show a recurring weakness or robust success on the relevant instruction types."],"support_condition":"Evaluations of the original Imagen show recurring failures to preserve basic roles or relationships in compositional instructions, inconsistent with reliable understanding in the sense discussed.","contradiction_condition":"Controlled evaluations show reliable handling of the relevant role reversals, relationships, and equivalent paraphrases, with the cited failures shown to be isolated exceptions."},"axis":"historical_observation","outcome":"supported","horizon_state":"not_applicable","evaluation_as_of":"2026-10-28","rationale":"Under the contextual reading of reliable compositional instruction understanding, the original paper supports Marcus: its authors acknowledge alignment difficulties in the category containing the disputed prompt. Imagen's strong comparative results are material qualifications, but outperforming other models does not demonstrate reliable handling of these instructions. This finding does not establish that Imagen had no language capabilities or identify the internal cause of its failures.","limitations":["The result depends on interpreting deep understanding through the role-and-relationship behavior discussed in the post.","The evidence is the authors' qualitative category assessment, not a comprehensive paraphrase robustness study.","It applies to the original Imagen, not subsequent versions."],"evidence":[{"id":"e1","relation":"supports","excerpt":"struggle generating\nwell aligned images","locator":"Figure A.19 caption, page 33, Conflicting category","url":"https://www.3dvar.com/Saharia2022Photorealistic.pdf","publisher":"Google Research, Brain Team; copy hosted by 3DVAR","event_date":"2022-05-23","available_at":"2022-05-23","explanation":"The original authors identify a category-level alignment weakness in the figure containing the horse-riding-an-astronaut prompt. This supports the contextual reading that reliable compositional instruction understanding was missing."},{"id":"e2","relation":"context","excerpt":"Human raters strongly prefer Imagen","locator":"DrawBench: new comprehensive challenging benchmark","url":"https://imagen.research.google/","publisher":"Google Research","event_date":"2022-05-23","available_at":"2022-05-23","explanation":"The favorable comparative evaluation cautions against interpreting Marcus as showing no useful language capability. Preference over competing systems does not establish reliable understanding of the specific relationships at issue."}],"what_would_resolve":null,"retrospective":true,"plain_reading":true},"payload_hash":"6ef86bd16286ef70c1a77403b95013abf137e2bf02f31ee1d37664337d64a029","policy_version":"2026-10-28.7","recorded_at":"2026-10-29T07:30:34Z","drafter":{"provider":"openai","model":"gpt-6-astra"},"reviewers":[{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-horse-rides-astronaut-20af65-4x-review-anthropic-1"},{"provider":"openai","model":"gpt-6-astra","run_id":"marcus-horse-rides-astronaut-20af65-4x-review-openai-1"}],"final_text":{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-horse-rides-astronaut-20af65-4x-final-anthropic-1"},"status":"current","notice":null,"superseded_by":null,"evidence_meta":[{"id":"e1","source_type":"preprint","audited":null,"derived":null,"metric":null},{"id":"e2","source_type":"company_disclosure","audited":null,"derived":null,"metric":null}],"public_from":null,"public_until":null},"history":[],"review":{"review_due":false,"triggers":[],"last_completed_review_at":"2026-10-29T07:30:34Z","next_review_due_at":null,"horizon_now":"not_applicable"},"event_ids":["marcus-horse-rides-astronaut-4f1bbd-assessment-published-a1"],"last_substantive_change_at":"2026-10-29T17:00:18Z"}},{"url":"https://punditclaims.ai/claims/marcus-horse-rides-astronaut-61347d","standing":"current","assessment_label":"Unresolved","versions":[{"assessment_id":"marcus-horse-rides-astronaut-61347d-a1","status":"current","url":"https://punditclaims.ai/claims/marcus-horse-rides-astronaut-61347d#assessment-marcus-horse-rides-astronaut-61347d-a1"}],"claim":{"claim_id":"marcus-horse-rides-astronaut-61347d","subject_id":"gary-marcus","revision":1,"content_hash":"554434e21db75dd44089593f4ae422884b2c1ffd369df53f8daf4f9f69a6db33","normalized_proposition":"DALL-E 2 is easily fooled, especially by complex sentences and words requiring deep language understanding.","kind":"observation","target":"DALL-E 2","conditions":[{"text":"especially complex sentences and words like 'similar'","type":"qualifier","from_source":true}],"original_horizon_text":null,"topic_tags":["capabilities","reliability"],"concerns_providers":["openai"],"selection_stratum":"declared-lead","occurrences":[{"id":"marcus-horse-rides-astronaut-61347d-o1","statement_date":"2022-05-28","statement_date_precision":"day","excerpt":"As Ernie Davis, Scott Aaronson, and I showed, a week or so after the release, it’s easily fooled, especially with complex sentences and words like “similar” that would require a deep understanding of language to be interpreted correctly","locator":"paragraph 18","document":{"url":"https://garymarcus.substack.com/p/horse-rides-astronaut","title":"Horse rides astronaut","publisher":"Marcus on AI","published_at":"2022-05-28T14:29:44Z","date_uncertain":false,"date_provenance":"JSON-LD datePublished"},"observed_at":"2026-10-29T00:05:21Z","observed_via":"live","archive_url":null,"access_state":"full_public","snapshot_hash":"e4a8233fec988ccb75c5e8a45ac1ad325ab13313a43cd661957d6e57ebb49d48","receipts":1}],"first_statement_date":"2022-05-28","related":[],"current":{"assessment_id":"marcus-horse-rides-astronaut-61347d-a1","payload":{"claim_id":"marcus-horse-rides-astronaut-61347d","subject_id":"gary-marcus","normalized_proposition":"DALL-E 2 is easily fooled, especially by complex sentences and words requiring deep language understanding.","kind":"observation","conditions":["especially complex sentences and words like 'similar'"],"original_horizon_text":null,"statement":{"excerpt":"As Ernie Davis, Scott Aaronson, and I showed, a week or so after the release, it’s easily fooled, especially with complex sentences and words like “similar” that would require a deep understanding of language to be interpreted correctly","locator":"paragraph 18","url":"https://garymarcus.substack.com/p/horse-rides-astronaut","date":"2022-05-28","document_title":"Horse rides astronaut","observed_at":"2026-10-29T00:05:21Z"},"criteria":{"from_speaker":["a week or so after the release","it’s easily fooled","especially with complex sentences and words like “similar”"],"our_interpretation":["The context explicitly places these observations in April 2022. The target is the original DALL-E 2.","The plain reading is that complex captions repeatedly produce substantial errors in objects, attributes or relationships, with particular difficulty interpreting relational language.","“Similar” is an illustrative example. No numerical failure rate or precise meaning of “deep understanding” is supplied.","Correctness concerns whether images satisfy the caption, rather than their visual quality."],"support_condition":"Independent records of the April 2022 model show repeated substantive failures on complex relational captions, including evidence that linguistic complexity or relational expressions create particular difficulty.","contradiction_condition":"Comparable controlled tests of the April 2022 model show reliable interpretation of complex relational captions, without the claimed particular difficulty."},"axis":"historical_observation","outcome":"unresolved","horizon_state":"not_applicable","evaluation_as_of":"2026-10-28","rationale":"The original model paper documents attribute-binding problems, so there is evidence for a narrower weakness. The retrieved independent evidence does not settle the stronger claim about frequent failures, especially on complex sentences and relational expressions, during April 2022. Marcus’s own study is excluded by the evidence rules. Later evaluations cannot automatically establish the earlier state.","limitations":["Unresolved does not mean the claim was false.","The evidence supports some compositional weaknesses without establishing the full comparison Marcus makes."],"evidence":[{"id":"e1","relation":"context","excerpt":"worse at binding attributes to objects","locator":"Section 7, Limitations and Risks","url":"https://ar5iv.labs.arxiv.org/html/2204.06125","publisher":"OpenAI authors, published through arXiv","event_date":"2022-04","available_at":"2022-04-13","explanation":"The original model paper documents difficulty assigning properties to objects. This supports a narrower weakness, but does not establish frequent failures specifically associated with complex syntax or expressions such as “similar.”"}],"what_would_resolve":"Independent April prompt-output records, or a documented evaluation of the preserved April model, testing complex relational captions against simpler controls with all outputs retained.","retrospective":true,"plain_reading":true},"payload_hash":"04dc2958e8a30ee187052d2a03b4f3786da4f17f5c3b23d8bb5ed75715a95832","policy_version":"2026-10-28.7","recorded_at":"2026-10-29T07:30:34Z","drafter":{"provider":"openai","model":"gpt-6-astra"},"reviewers":[{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-horse-rides-astronaut-20af65-4x-review-anthropic-1"},{"provider":"openai","model":"gpt-6-astra","run_id":"marcus-horse-rides-astronaut-20af65-4x-review-openai-1"}],"final_text":{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-horse-rides-astronaut-20af65-4x-final-anthropic-1"},"status":"current","notice":null,"superseded_by":null,"evidence_meta":[{"id":"e1","source_type":"preprint","audited":null,"derived":null,"metric":null}],"public_from":null,"public_until":null},"history":[],"review":{"review_due":false,"triggers":[],"last_completed_review_at":"2026-10-29T07:30:34Z","next_review_due_at":null,"horizon_now":"not_applicable"},"event_ids":["marcus-horse-rides-astronaut-61347d-assessment-published-a1"],"last_substantive_change_at":"2026-10-29T17:00:18Z"}},{"url":"https://punditclaims.ai/claims/marcus-horse-rides-astronaut-63b833","standing":"current","assessment_label":"Unresolved","versions":[{"assessment_id":"marcus-horse-rides-astronaut-63b833-a1","status":"current","url":"https://punditclaims.ai/claims/marcus-horse-rides-astronaut-63b833#assessment-marcus-horse-rides-astronaut-63b833-a1"}],"claim":{"claim_id":"marcus-horse-rides-astronaut-63b833","subject_id":"gary-marcus","revision":2,"content_hash":"b2129fdb8fff457d2cae48e575ebb3cc9682c87fddd6593070a7ed8ac7cd52d0","normalized_proposition":"Since Imagen can draw horses riding astronauts when the instructions are specific enough, its failure to draw one from the prompt 'a horse riding an astronaut' can only be explained by the system not understanding what that prompt means.","kind":"causal","target":"Imagen","conditions":[{"text":"since Imagen can draw horses riding astronauts if the instructions are specific enough","type":"qualifier","from_source":true}],"original_horizon_text":null,"topic_tags":["capabilities"],"concerns_providers":[],"selection_stratum":"declared-lead","occurrences":[{"id":"marcus-horse-rides-astronaut-63b833-o1","statement_date":"2022-05-28","statement_date_precision":"day","excerpt":"Since we know that Imagen can draw images of horses riding astronauts (if the instructions are specific enough) we know that the failure to draw a horse riding an astronaut given the prompt “a horse riding an astronaut” can only have one explanation: the system doesn’t understand what the prompt “a horse riding an astronaut” means.","locator":"paragraph 90","document":{"url":"https://garymarcus.substack.com/p/horse-rides-astronaut","title":"Horse rides astronaut","publisher":"Marcus on AI","published_at":"2022-05-28T14:29:44Z","date_uncertain":false,"date_provenance":"JSON-LD datePublished"},"observed_at":"2026-10-29T00:05:21Z","observed_via":"live","archive_url":null,"access_state":"full_public","snapshot_hash":"e4a8233fec988ccb75c5e8a45ac1ad325ab13313a43cd661957d6e57ebb49d48","receipts":1}],"first_statement_date":"2022-05-28","related":[],"current":{"assessment_id":"marcus-horse-rides-astronaut-63b833-a1","payload":{"claim_id":"marcus-horse-rides-astronaut-63b833","subject_id":"gary-marcus","normalized_proposition":"Since Imagen can draw horses riding astronauts when the instructions are specific enough, its failure to draw one from the prompt 'a horse riding an astronaut' can only be explained by the system not understanding what that prompt means.","kind":"causal","conditions":["since Imagen can draw horses riding astronauts if the instructions are specific enough"],"original_horizon_text":null,"statement":{"excerpt":"Since we know that Imagen can draw images of horses riding astronauts (if the instructions are specific enough) we know that the failure to draw a horse riding an astronaut given the prompt “a horse riding an astronaut” can only have one explanation: the system doesn’t understand what the prompt “a horse riding an astronaut” means.","locator":"paragraph 90","url":"https://garymarcus.substack.com/p/horse-rides-astronaut","date":"2022-05-28","document_title":"Horse rides astronaut","observed_at":"2026-10-29T00:05:21Z"},"criteria":{"from_speaker":["Imagen can draw images of horses riding astronauts","if the instructions are specific enough","can only have one explanation","the system doesn’t understand what the prompt","a horse riding an astronaut","means"],"our_interpretation":["This concerns the original Imagen's May 2022 prompt failure.","The success of more specific instructions is an asserted premise, not a future conditional forecast.","Only one explanation commits to causal exclusivity. Evidence that some prompt changes help is insufficient unless competing causes of the original failure are excluded.","The claim does not specifically identify a neural-network component, so understanding is assessed through preservation and use of the requested semantic roles."],"support_condition":"Tests of the original Imagen verify the premise and establish failed semantic interpretation as the explanation of the original failure while excluding downstream conditioning, generation, sampling, and selection explanations.","contradiction_condition":"Tests of the original Imagen establish another cause of the failure, including cases where the requested roles are represented correctly but the generation process fails to express them."},"axis":"causal_assessment","outcome":"unresolved","horizon_state":"not_applicable","evaluation_as_of":"2026-10-28","rationale":"The premise is supported by Neyshabur's report that more specific wording improves the chance of success. The exclusive explanation is not established. Successful rewording does not by itself show where the original attempt failed. Research on other generators cannot supply the missing diagnosis for Imagen, and merely proposing alternatives would not justify calling Marcus contradicted.","limitations":["The available demonstration is not a controlled causal experiment.","An output-level failure cannot by itself distinguish absent semantic interpretation from failure to use an adequate representation.","The post's acknowledged uncertainty limits confidence but does not erase its explicit exclusivity claim."],"evidence":[{"id":"e1","relation":"supports","excerpt":"helps increase the chance","locator":"Neyshabur's second sentence","url":"https://twitter.com/bneyshabur/status/1529506107655303168","publisher":"Behnam Neyshabur","event_date":"2022-05-25","available_at":"2022-05-25","explanation":"The firsthand report supports the premise that more specific instructions can produce the intended scene. It supplies no test establishing a unique cause for failures with the original wording."},{"id":"e2","relation":"context","excerpt":"stablediffusion 2.1 text-to-image model.","locator":"Abstract, third sentence","url":"https://arxiv.org/abs/2306.01755","publisher":"Charles Lovering and Ellie Pavlick, arXiv","event_date":"2023","available_at":"2023-10-25","explanation":"This study relates training frequencies to success on reversed relationships. Because it tests another model, it cannot establish an alternative cause of the original Imagen failure."}],"what_would_resolve":"A controlled investigation of the original Imagen that identifies the cause of the disputed failures and either excludes or demonstrates competing explanations.","retrospective":true,"plain_reading":true},"payload_hash":"faa6c48fd3b9ed6c0520d23624de08e8df83cc4ddd2d1f3de7043d49d793216a","policy_version":"2026-10-28.7","recorded_at":"2026-10-29T07:30:19Z","drafter":{"provider":"openai","model":"gpt-6-astra"},"reviewers":[{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-horse-rides-astronaut-63b833-4x-review-anthropic-1"},{"provider":"openai","model":"gpt-6-astra","run_id":"marcus-horse-rides-astronaut-63b833-4x-review-openai-1"}],"final_text":{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-horse-rides-astronaut-63b833-4x-final-anthropic-1"},"status":"current","notice":null,"superseded_by":null,"evidence_meta":[{"id":"e1","source_type":"demonstration","audited":null,"derived":null,"metric":null},{"id":"e2","source_type":"preprint","audited":null,"derived":null,"metric":null}],"public_from":null,"public_until":null},"history":[],"review":{"review_due":false,"triggers":[],"last_completed_review_at":"2026-10-29T07:30:19Z","next_review_due_at":null,"horizon_now":"not_applicable"},"event_ids":["marcus-horse-rides-astronaut-63b833-assessment-published-a1"],"last_substantive_change_at":"2026-10-29T17:00:18Z"}},{"url":"https://punditclaims.ai/claims/marcus-horse-rides-astronaut-6501e4","standing":"current","assessment_label":"Unresolved","versions":[{"assessment_id":"marcus-horse-rides-astronaut-6501e4-a2","status":"current","url":"https://punditclaims.ai/claims/marcus-horse-rides-astronaut-6501e4#assessment-marcus-horse-rides-astronaut-6501e4-a2"}],"claim":{"claim_id":"marcus-horse-rides-astronaut-6501e4","subject_id":"gary-marcus","revision":1,"content_hash":"8c2b7ba16d693fc0c40a69fcc68899d4271d73b191962aeac77947b8f71712f7","normalized_proposition":"Imagen's failure on 'a horse riding an astronaut' was not due to common sense.","kind":"causal","target":"Imagen","conditions":[],"original_horizon_text":null,"topic_tags":["capabilities"],"concerns_providers":[],"selection_stratum":"declared-lead","occurrences":[{"id":"marcus-horse-rides-astronaut-6501e4-o1","statement_date":"2022-05-28","statement_date_precision":"day","excerpt":"And, no, it wasn’t about common sense at all.","locator":"paragraph 65","document":{"url":"https://garymarcus.substack.com/p/horse-rides-astronaut","title":"Horse rides astronaut","publisher":"Marcus on AI","published_at":"2022-05-28T14:29:44Z","date_uncertain":false,"date_provenance":"JSON-LD datePublished"},"observed_at":"2026-10-29T00:05:21Z","observed_via":"live","archive_url":null,"access_state":"full_public","snapshot_hash":"e4a8233fec988ccb75c5e8a45ac1ad325ab13313a43cd661957d6e57ebb49d48","receipts":1}],"first_statement_date":"2022-05-28","related":[],"current":{"assessment_id":"marcus-horse-rides-astronaut-6501e4-a2","payload":{"claim_id":"marcus-horse-rides-astronaut-6501e4","subject_id":"gary-marcus","normalized_proposition":"Imagen's failure on 'a horse riding an astronaut' was not due to common sense.","kind":"causal","conditions":[],"original_horizon_text":null,"statement":{"excerpt":"And, no, it wasn’t about common sense at all.","locator":"paragraph 65","url":"https://garymarcus.substack.com/p/horse-rides-astronaut","date":"2022-05-28","document_title":"Horse rides astronaut","observed_at":"2026-10-29T00:05:21Z"},"criteria":{"from_speaker":["it wasn’t about common sense at all","All good guesses, all wrong"],"our_interpretation":["The claim concerns the cause of original Imagen’s May 2022 failures on the horse-riding-astronaut prompt.","In context, “common sense” means a plausibility judgment or learned expectation that makes the model favor ordinary riding arrangements.","Successful rewording can rule out an absolute inability or unconditional refusal. It does not by itself rule out a contributing plausibility preference.","The categorical phrase “at all” requires stronger evidence than showing that wording affects the result."],"support_condition":"Controlled comparisons or model interventions show that plausibility-based judgment did not cause the original failure, while identifying a sufficient alternative mechanism.","contradiction_condition":"Controlled evidence shows that a plausibility-based judgment or expectation causally contributed to the role reversal."},"axis":"causal_assessment","outcome":"unresolved","horizon_state":"not_applicable","evaluation_as_of":"2026-10-28","rationale":"On May 25, 2022, Neyshabur reported that adding “on back of” or “on shoulders of” improved Imagen’s chance of depicting a horse riding an astronaut [e5]. The demonstration does not separate wording effects from learned plausibility preferences, so it cannot establish whether those preferences contributed to the original failure [e5]. The claim that common sense played no causal role remains unresolved [e5].","limitations":["The demonstration provides no repeated-sample success rates or controlled comparison separating wording from scene plausibility [e5].","Successful rewording establishes neither the presence nor the absence of a plausibility-based contribution to the original failure [e5]."],"evidence":[{"id":"e5","relation":"context","excerpt":"helps increase the chance of getting it right!","locator":"Neyshabur’s post, second sentence","url":"https://twitter.com/bneyshabur/status/1529506107655303168","publisher":"Behnam Neyshabur","event_date":"2022-05-25","available_at":"2022-05-25","explanation":"The researcher reports improved success after specifying the riding position. This challenges an unconditional refusal explanation, but does not isolate or exclude a contributing preference for plausible scenes."}],"what_would_resolve":"Controlled tests on original Imagen, using repeated samples and matched prompt variants, that separate wording from plausibility effects. These would need to establish a causal contribution from plausibility preferences, or exclude that contribution while identifying a sufficient alternative mechanism.","retrospective":true,"plain_reading":true},"payload_hash":"534d8c7db1298d6795bd9bf2718f9ea04fc90c01fc0a316812a786c13be3c56a","policy_version":"2026-10-28.7","recorded_at":"2026-10-29T07:41:18Z","drafter":{"provider":"openai","model":"gpt-6-astra"},"reviewers":[{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-horse-rides-astronaut-63b833-4x-review-anthropic-1"},{"provider":"openai","model":"gpt-6-astra","run_id":"marcus-horse-rides-astronaut-63b833-4x-review-openai-1"}],"final_text":{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-horse-rides-astronaut-6501e4-2x-final-anthropic-1"},"status":"current","notice":null,"superseded_by":null,"evidence_meta":[{"id":"e5","source_type":"demonstration","audited":null,"derived":null,"metric":null}],"public_from":null,"public_until":null},"history":[],"review":{"review_due":false,"triggers":[],"last_completed_review_at":"2026-10-29T07:41:18Z","next_review_due_at":null,"horizon_now":"not_applicable"},"event_ids":["marcus-horse-rides-astronaut-6501e4-assessment-published-a2"],"last_substantive_change_at":"2026-10-29T17:00:18Z"}},{"url":"https://punditclaims.ai/claims/marcus-horse-rides-astronaut-792402","standing":"current","assessment_label":"Can’t be checked","versions":[{"assessment_id":"marcus-horse-rides-astronaut-792402-a1","status":"current","url":"https://punditclaims.ai/claims/marcus-horse-rides-astronaut-792402#assessment-marcus-horse-rides-astronaut-792402-a1"}],"claim":{"claim_id":"marcus-horse-rides-astronaut-792402","subject_id":"gary-marcus","revision":1,"content_hash":"165dd46574db8584124f3850efbb6e152d0babd6eb41dcacabea2c762268cd8d","normalized_proposition":"Imagen almost certainly does not have a deep understanding of language.","kind":"observation","target":"Imagen","conditions":[],"original_horizon_text":null,"topic_tags":["capabilities"],"concerns_providers":[],"selection_stratum":"declared-lead","occurrences":[{"id":"marcus-horse-rides-astronaut-792402-o1","statement_date":"2022-05-28","statement_date_precision":"day","excerpt":"The answer to the second question—does Imagen have a deep understanding of language—is almost certainly “no”: if a system can draw a given picture with incantation X, but not with perfectly sensible alternative incantation Y, it’s pretty hard to say that the system understands Y.","locator":"paragraph 89","document":{"url":"https://garymarcus.substack.com/p/horse-rides-astronaut","title":"Horse rides astronaut","publisher":"Marcus on AI","published_at":"2022-05-28T14:29:44Z","date_uncertain":false,"date_provenance":"JSON-LD datePublished"},"observed_at":"2026-10-29T00:05:21Z","observed_via":"live","archive_url":null,"access_state":"full_public","snapshot_hash":"e4a8233fec988ccb75c5e8a45ac1ad325ab13313a43cd661957d6e57ebb49d48","receipts":1}],"first_statement_date":"2022-05-28","related":[],"current":{"assessment_id":"marcus-horse-rides-astronaut-792402-a1","payload":{"claim_id":"marcus-horse-rides-astronaut-792402","subject_id":"gary-marcus","normalized_proposition":"Imagen almost certainly does not have a deep understanding of language.","kind":"observation","conditions":[],"original_horizon_text":null,"statement":{"excerpt":"The answer to the second question—does Imagen have a deep understanding of language—is almost certainly “no”: if a system can draw a given picture with incantation X, but not with perfectly sensible alternative incantation Y, it’s pretty hard to say that the system understands Y.","locator":"paragraph 89","url":"https://garymarcus.substack.com/p/horse-rides-astronaut","date":"2022-05-28","document_title":"Horse rides astronaut","observed_at":"2026-10-29T00:05:21Z"},"criteria":{"from_speaker":[],"our_interpretation":[],"support_condition":"Not checked against evidence: both screens found that public evidence could not settle this claim.","contradiction_condition":"Not checked against evidence: both screens found that public evidence could not settle this claim."},"axis":"not_applicable","outcome":"not_adjudicable","horizon_state":"not_applicable","evaluation_as_of":"2026-10-28","rationale":"Not checked against evidence. Two independent screens, one from each provider, both found that evidence could not settle it: one found it too vague to test: informed readers would test it in different ways and reach different answers; the other found it stated as a chance or a possibility, or at medium or low confidence.","limitations":[],"evidence":[],"what_would_resolve":null,"retrospective":true,"uncheckable":["no_plain_reading","probability_language"]},"payload_hash":"fc32f20150d5cffcb93e14d792c8bf7f080c91da00e7117200738d2dc0d03c37","policy_version":"2026-10-28.7","recorded_at":"2026-10-29T01:10:54Z","drafter":{"provider":"anthropic","model":"claude-opus-5-5"},"reviewers":[{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-horse-rides-astronaut-screen-anthropic-1"},{"provider":"openai","model":"gpt-6-astra","run_id":"marcus-horse-rides-astronaut-screen-openai-1"}],"final_text":null,"status":"current","notice":null,"superseded_by":null,"evidence_meta":[],"public_from":null,"public_until":null},"history":[],"review":{"review_due":false,"triggers":[],"last_completed_review_at":"2026-10-29T01:10:54Z","next_review_due_at":null,"horizon_now":"not_applicable"},"event_ids":["marcus-horse-rides-astronaut-792402-assessment-published-a1"],"last_substantive_change_at":"2026-10-29T17:00:18Z"}},{"url":"https://punditclaims.ai/claims/marcus-horse-rides-astronaut-a6cd19","standing":"current","assessment_label":"Unresolved","versions":[{"assessment_id":"marcus-horse-rides-astronaut-a6cd19-a1","status":"current","url":"https://punditclaims.ai/claims/marcus-horse-rides-astronaut-a6cd19#assessment-marcus-horse-rides-astronaut-a6cd19-a1"}],"claim":{"claim_id":"marcus-horse-rides-astronaut-a6cd19","subject_id":"gary-marcus","revision":1,"content_hash":"ba73b1959d8fc9a8df29d73365ab37d1726d5c9370254932e0b0fa6806aed6ae","normalized_proposition":"Imagen has no understanding of weight, mass or density.","kind":"observation","target":"Imagen","conditions":[],"original_horizon_text":null,"topic_tags":["capabilities"],"concerns_providers":[],"selection_stratum":"declared-lead","occurrences":[{"id":"marcus-horse-rides-astronaut-a6cd19-o1","statement_date":"2022-05-28","statement_date_precision":"day","excerpt":"(In reality, Imagen has no idea what weight, mass, or density are.)","locator":"paragraph 63","document":{"url":"https://garymarcus.substack.com/p/horse-rides-astronaut","title":"Horse rides astronaut","publisher":"Marcus on AI","published_at":"2022-05-28T14:29:44Z","date_uncertain":false,"date_provenance":"JSON-LD datePublished"},"observed_at":"2026-10-29T00:05:21Z","observed_via":"live","archive_url":null,"access_state":"full_public","snapshot_hash":"e4a8233fec988ccb75c5e8a45ac1ad325ab13313a43cd661957d6e57ebb49d48","receipts":1}],"first_statement_date":"2022-05-28","related":[],"current":{"assessment_id":"marcus-horse-rides-astronaut-a6cd19-a1","payload":{"claim_id":"marcus-horse-rides-astronaut-a6cd19","subject_id":"gary-marcus","normalized_proposition":"Imagen has no understanding of weight, mass or density.","kind":"observation","conditions":[],"original_horizon_text":null,"statement":{"excerpt":"(In reality, Imagen has no idea what weight, mass, or density are.)","locator":"paragraph 63","url":"https://garymarcus.substack.com/p/horse-rides-astronaut","date":"2022-05-28","document_title":"Horse rides astronaut","observed_at":"2026-10-29T00:05:21Z"},"criteria":{"from_speaker":["Imagen has no idea","weight, mass, or density"],"our_interpretation":["The target is the original Imagen system discussed in May 2022, not later Imagen versions.","For this ongoing state, the target period is May 2022 through publication, the period of the model’s initial public report.","The ordinary behavioral reading is that Imagen lacks systematic usable competence with these physical concepts.","This does not require deciding whether a machine has subjective understanding. Neither a single implausible image nor the architecture alone establishes absence of competence."],"support_condition":"Controlled evaluations of the original model consistently fail to demonstrate usable knowledge of weight, mass and density across varied prompts, after accounting for general image-generation and language-following errors.","contradiction_condition":"Controlled evaluations demonstrate systematic, transferable competence with one or more of these concepts, contradicting the blanket assertion of no understanding."},"axis":"historical_observation","outcome":"unresolved","horizon_state":"not_applicable","evaluation_as_of":"2026-10-28","rationale":"The searched records do not settle whether original Imagen had usable competence with weight, mass or density. General image benchmarks and failures on a horse–astronaut prompt do not establish the claimed absence of all such competence. No controlled evaluation of these concepts in the original model was located.","limitations":["The behavioral interpretation makes the claim testable but does not settle philosophical questions about understanding.","Missing evaluations are not evidence that the model lacked the concepts."],"evidence":[{"id":"e4","relation":"context","excerpt":"a comprehensive and challenging benchmark","locator":"Abstract, DrawBench introduction","url":"https://arxiv.org/abs/2205.11487","publisher":"Google Research authors, published through arXiv","event_date":"2022-05","available_at":"2022-05-23","explanation":"The original report describes a text-to-image evaluation framework. The inspected evaluation does not establish either systematic competence or its absence for weight, mass and density."}],"what_would_resolve":"Controlled tests of the preserved original model across weight, mass and density relationships, with matched language and rendering controls and independently scored outputs.","retrospective":true,"plain_reading":true},"payload_hash":"11fdc15069d4624f75994d922af3cf57712a8dc2c132ad50df6d31390bce906e","policy_version":"2026-10-28.7","recorded_at":"2026-10-29T07:30:19Z","drafter":{"provider":"openai","model":"gpt-6-astra"},"reviewers":[{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-horse-rides-astronaut-63b833-4x-review-anthropic-1"},{"provider":"openai","model":"gpt-6-astra","run_id":"marcus-horse-rides-astronaut-63b833-4x-review-openai-1"}],"final_text":{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-horse-rides-astronaut-63b833-4x-final-anthropic-1"},"status":"current","notice":null,"superseded_by":null,"evidence_meta":[{"id":"e4","source_type":"preprint","audited":null,"derived":null,"metric":null}],"public_from":null,"public_until":null},"history":[],"review":{"review_due":false,"triggers":[],"last_completed_review_at":"2026-10-29T07:30:19Z","next_review_due_at":null,"horizon_now":"not_applicable"},"event_ids":["marcus-horse-rides-astronaut-a6cd19-assessment-published-a1"],"last_substantive_change_at":"2026-10-29T17:00:18Z"}},{"url":"https://punditclaims.ai/claims/marcus-horse-rides-astronaut-c336ee","standing":"current","assessment_label":"Can’t be checked","versions":[{"assessment_id":"marcus-horse-rides-astronaut-c336ee-a1","status":"current","url":"https://punditclaims.ai/claims/marcus-horse-rides-astronaut-c336ee#assessment-marcus-horse-rides-astronaut-c336ee-a1"}],"claim":{"claim_id":"marcus-horse-rides-astronaut-c336ee","subject_id":"gary-marcus","revision":1,"content_hash":"e654bb90172bad78e916f8c0d2568b85b1337ebe54320138c152ea851b469ff8","normalized_proposition":"DALL-E 2 is not as smart as it appears.","kind":"observation","target":"DALL-E 2","conditions":[],"original_horizon_text":null,"topic_tags":["capabilities"],"concerns_providers":["openai"],"selection_stratum":"declared-lead","occurrences":[{"id":"marcus-horse-rides-astronaut-c336ee-o1","statement_date":"2022-05-28","statement_date_precision":"day","excerpt":"DALL-E 2 is, I dare say, not as smart as it seems.","locator":"paragraph 18","document":{"url":"https://garymarcus.substack.com/p/horse-rides-astronaut","title":"Horse rides astronaut","publisher":"Marcus on AI","published_at":"2022-05-28T14:29:44Z","date_uncertain":false,"date_provenance":"JSON-LD datePublished"},"observed_at":"2026-10-29T00:05:21Z","observed_via":"live","archive_url":null,"access_state":"full_public","snapshot_hash":"e4a8233fec988ccb75c5e8a45ac1ad325ab13313a43cd661957d6e57ebb49d48","receipts":1}],"first_statement_date":"2022-05-28","related":[],"current":{"assessment_id":"marcus-horse-rides-astronaut-c336ee-a1","payload":{"claim_id":"marcus-horse-rides-astronaut-c336ee","subject_id":"gary-marcus","normalized_proposition":"DALL-E 2 is not as smart as it appears.","kind":"observation","conditions":[],"original_horizon_text":null,"statement":{"excerpt":"DALL-E 2 is, I dare say, not as smart as it seems.","locator":"paragraph 18","url":"https://garymarcus.substack.com/p/horse-rides-astronaut","date":"2022-05-28","document_title":"Horse rides astronaut","observed_at":"2026-10-29T00:05:21Z"},"criteria":{"from_speaker":[],"our_interpretation":[],"support_condition":"Not checked against evidence: both screens found that public evidence could not settle this claim.","contradiction_condition":"Not checked against evidence: both screens found that public evidence could not settle this claim."},"axis":"not_applicable","outcome":"not_adjudicable","horizon_state":"not_applicable","evaluation_as_of":"2026-10-28","rationale":"Not checked against evidence. Two independent screens, one from each provider, both found this too vague to test: informed readers would test it in different ways and reach different answers.","limitations":[],"evidence":[],"what_would_resolve":null,"retrospective":true,"uncheckable":["no_plain_reading"]},"payload_hash":"cebee152ce57c2555ae1dac83518ea67cbbb148933f6ac355eab92ef10db3127","policy_version":"2026-10-28.7","recorded_at":"2026-10-29T01:10:54Z","drafter":{"provider":"anthropic","model":"claude-opus-5-5"},"reviewers":[{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-horse-rides-astronaut-screen-anthropic-1"},{"provider":"openai","model":"gpt-6-astra","run_id":"marcus-horse-rides-astronaut-screen-openai-1"}],"final_text":null,"status":"current","notice":null,"superseded_by":null,"evidence_meta":[],"public_from":null,"public_until":null},"history":[],"review":{"review_due":false,"triggers":[],"last_completed_review_at":"2026-10-29T01:10:54Z","next_review_due_at":null,"horizon_now":"not_applicable"},"event_ids":["marcus-horse-rides-astronaut-c336ee-assessment-published-a1"],"last_substantive_change_at":"2026-10-29T17:00:18Z"}},{"url":"https://punditclaims.ai/claims/marcus-horse-rides-astronaut-d5bd09","standing":"current","assessment_label":"Can’t be checked","versions":[{"assessment_id":"marcus-horse-rides-astronaut-d5bd09-a1","status":"current","url":"https://punditclaims.ai/claims/marcus-horse-rides-astronaut-d5bd09#assessment-marcus-horse-rides-astronaut-d5bd09-a1"}],"claim":{"claim_id":"marcus-horse-rides-astronaut-d5bd09","subject_id":"gary-marcus","revision":1,"content_hash":"3b25bf2a7e9426cb99c3e2dfd2796f3667098a63df74417946b8cb96e1854364","normalized_proposition":"The machine learning community has recently made premature declarations and may not have learned the Clever Hans lesson about overclaiming.","kind":"observation","target":"the machine learning community","conditions":[],"original_horizon_text":null,"topic_tags":["other"],"concerns_providers":[],"selection_stratum":"declared-lead","occurrences":[{"id":"marcus-horse-rides-astronaut-d5bd09-o1","statement_date":"2022-05-28","statement_date_precision":"day","excerpt":"Judging by all the premature declarations I have seen from the machine learning community of late, I am not sure AI has yet fully learned it.","locator":"paragraph 17","document":{"url":"https://garymarcus.substack.com/p/horse-rides-astronaut","title":"Horse rides astronaut","publisher":"Marcus on AI","published_at":"2022-05-28T14:29:44Z","date_uncertain":false,"date_provenance":"JSON-LD datePublished"},"observed_at":"2026-10-29T00:05:21Z","observed_via":"live","archive_url":null,"access_state":"full_public","snapshot_hash":"e4a8233fec988ccb75c5e8a45ac1ad325ab13313a43cd661957d6e57ebb49d48","receipts":1}],"first_statement_date":"2022-05-28","related":[],"current":{"assessment_id":"marcus-horse-rides-astronaut-d5bd09-a1","payload":{"claim_id":"marcus-horse-rides-astronaut-d5bd09","subject_id":"gary-marcus","normalized_proposition":"The machine learning community has recently made premature declarations and may not have learned the Clever Hans lesson about overclaiming.","kind":"observation","conditions":[],"original_horizon_text":null,"statement":{"excerpt":"Judging by all the premature declarations I have seen from the machine learning community of late, I am not sure AI has yet fully learned it.","locator":"paragraph 17","url":"https://garymarcus.substack.com/p/horse-rides-astronaut","date":"2022-05-28","document_title":"Horse rides astronaut","observed_at":"2026-10-29T00:05:21Z"},"criteria":{"from_speaker":[],"our_interpretation":[],"support_condition":"Not checked against evidence: both screens found that public evidence could not settle this claim.","contradiction_condition":"Not checked against evidence: both screens found that public evidence could not settle this claim."},"axis":"not_applicable","outcome":"not_adjudicable","horizon_state":"not_applicable","evaluation_as_of":"2026-10-28","rationale":"Not checked against evidence. Two independent screens, one from each provider, both found this stated as a chance or a possibility, or at medium or low confidence.","limitations":[],"evidence":[],"what_would_resolve":null,"retrospective":true,"uncheckable":["probability_language"]},"payload_hash":"30a71a4b24c42d4b960ad1b8804f0e942e6f6aafe4c717169a76e229d92a2a2b","policy_version":"2026-10-28.7","recorded_at":"2026-10-29T01:10:54Z","drafter":{"provider":"anthropic","model":"claude-opus-5-5"},"reviewers":[{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-horse-rides-astronaut-screen-anthropic-1"},{"provider":"openai","model":"gpt-6-astra","run_id":"marcus-horse-rides-astronaut-screen-openai-1"}],"final_text":null,"status":"current","notice":null,"superseded_by":null,"evidence_meta":[],"public_from":null,"public_until":null},"history":[],"review":{"review_due":false,"triggers":[],"last_completed_review_at":"2026-10-29T01:10:54Z","next_review_due_at":null,"horizon_now":"not_applicable"},"event_ids":["marcus-horse-rides-astronaut-d5bd09-assessment-published-a1"],"last_substantive_change_at":"2026-10-29T17:00:18Z"}},{"url":"https://punditclaims.ai/claims/marcus-noam-chomsky-and-gpt-3-5ad6a3","standing":"current","assessment_label":"Can’t be checked","versions":[{"assessment_id":"marcus-noam-chomsky-and-gpt-3-5ad6a3-a1","status":"current","url":"https://punditclaims.ai/claims/marcus-noam-chomsky-and-gpt-3-5ad6a3#assessment-marcus-noam-chomsky-and-gpt-3-5ad6a3-a1"}],"claim":{"claim_id":"marcus-noam-chomsky-and-gpt-3-5ad6a3","subject_id":"gary-marcus","revision":1,"content_hash":"dd8cc80f9af38bb4d36447c6c0b2a65ff000cb10a1218155fc49817ef78b92d1","normalized_proposition":"New AI systems with a firmer innate, Kantian basis might eventually tell us more about the human mind.","kind":"forecast","target":"nativist AI systems","conditions":[],"original_horizon_text":"eventually","topic_tags":["capabilities"],"concerns_providers":[],"selection_stratum":"declared-lead","occurrences":[{"id":"marcus-noam-chomsky-and-gpt-3-5ad6a3-o1","statement_date":"2022-05-21","statement_date_precision":"day","excerpt":"A new breed of AI systems, with a firmer innate basis, closer to the Kantian model, but out of step with current Zeitgeist, might eventually tell us more.","locator":"paragraph 26","document":{"url":"https://garymarcus.substack.com/p/noam-chomsky-and-gpt-3","title":"Noam Chomsky and GPT-3","publisher":"Marcus on AI","published_at":"2022-05-21T17:45:07Z","date_uncertain":false,"date_provenance":"JSON-LD datePublished"},"observed_at":"2026-10-29T00:05:23Z","observed_via":"live","archive_url":null,"access_state":"full_public","snapshot_hash":"07d5a897e1c3fa996767148162ac6fcb20d0ba1228f4392f98d810f139f1b769","receipts":1}],"first_statement_date":"2022-05-21","related":[],"current":{"assessment_id":"marcus-noam-chomsky-and-gpt-3-5ad6a3-a1","payload":{"claim_id":"marcus-noam-chomsky-and-gpt-3-5ad6a3","subject_id":"gary-marcus","normalized_proposition":"New AI systems with a firmer innate, Kantian basis might eventually tell us more about the human mind.","kind":"forecast","conditions":[],"original_horizon_text":"eventually","statement":{"excerpt":"A new breed of AI systems, with a firmer innate basis, closer to the Kantian model, but out of step with current Zeitgeist, might eventually tell us more.","locator":"paragraph 26","url":"https://garymarcus.substack.com/p/noam-chomsky-and-gpt-3","date":"2022-05-21","document_title":"Noam Chomsky and GPT-3","observed_at":"2026-10-29T00:05:23Z"},"criteria":{"from_speaker":["eventually"],"our_interpretation":[],"support_condition":"Not checked against evidence: both screens found that public evidence could not settle this claim.","contradiction_condition":"Not checked against evidence: both screens found that public evidence could not settle this claim."},"axis":"not_applicable","outcome":"not_adjudicable","horizon_state":"not_applicable","evaluation_as_of":"2026-10-28","rationale":"Not checked against evidence. Two independent screens, one from each provider, both found this stated as a chance or a possibility, or at medium or low confidence.","limitations":[],"evidence":[],"what_would_resolve":null,"retrospective":true,"uncheckable":["probability_language"]},"payload_hash":"231680ededea52772d34503f3cd4440c7ae819ac46a7e226cc8cd4c2b084753e","policy_version":"2026-10-28.7","recorded_at":"2026-10-29T02:32:05Z","drafter":{"provider":"anthropic","model":"claude-opus-5-5"},"reviewers":[{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-noam-chomsky-and-gpt-3-screen-anthropic-2"},{"provider":"openai","model":"gpt-6-astra","run_id":"marcus-noam-chomsky-and-gpt-3-screen-openai-1"}],"final_text":null,"status":"current","notice":null,"superseded_by":null,"evidence_meta":[],"public_from":null,"public_until":null},"history":[],"review":{"review_due":false,"triggers":[],"last_completed_review_at":"2026-10-29T02:32:05Z","next_review_due_at":null,"horizon_now":"not_applicable"},"event_ids":["marcus-noam-chomsky-and-gpt-3-5ad6a3-assessment-published-a1"],"last_substantive_change_at":"2026-10-29T17:00:18Z"}},{"url":"https://punditclaims.ai/claims/marcus-noam-chomsky-and-gpt-3-5c52cd","standing":"current","assessment_label":"Can’t be checked","versions":[{"assessment_id":"marcus-noam-chomsky-and-gpt-3-5c52cd-a1","status":"current","url":"https://punditclaims.ai/claims/marcus-noam-chomsky-and-gpt-3-5c52cd#assessment-marcus-noam-chomsky-and-gpt-3-5c52cd-a1"}],"claim":{"claim_id":"marcus-noam-chomsky-and-gpt-3-5c52cd","subject_id":"gary-marcus","revision":2,"content_hash":"dc8055bddb562c9ebc2ef75c90c77a60831c6b674ec430b86fca2e1e264a37a4","normalized_proposition":"Because large language models, unlike human minds, do not try to relate their language to what they know about the world, they just aren't telling us much about human minds.","kind":"causal","target":"large language models","conditions":[],"original_horizon_text":null,"topic_tags":["capabilities"],"concerns_providers":[],"selection_stratum":"declared-lead","occurrences":[{"id":"marcus-noam-chomsky-and-gpt-3-5c52cd-o1","statement_date":"2022-05-21","statement_date_precision":"day","excerpt":"Human minds try to relate their language to what they know about the world; large language models don’t. Which means the latter just aren’t telling us much about the former.","locator":"paragraph 12","document":{"url":"https://garymarcus.substack.com/p/noam-chomsky-and-gpt-3","title":"Noam Chomsky and GPT-3","publisher":"Marcus on AI","published_at":"2022-05-21T17:45:07Z","date_uncertain":false,"date_provenance":"JSON-LD datePublished"},"observed_at":"2026-10-29T00:05:23Z","observed_via":"live","archive_url":null,"access_state":"full_public","snapshot_hash":"07d5a897e1c3fa996767148162ac6fcb20d0ba1228f4392f98d810f139f1b769","receipts":1}],"first_statement_date":"2022-05-21","related":[],"current":{"assessment_id":"marcus-noam-chomsky-and-gpt-3-5c52cd-a1","payload":{"claim_id":"marcus-noam-chomsky-and-gpt-3-5c52cd","subject_id":"gary-marcus","normalized_proposition":"Because large language models, unlike human minds, do not try to relate their language to what they know about the world, they just aren't telling us much about human minds.","kind":"causal","conditions":[],"original_horizon_text":null,"statement":{"excerpt":"Human minds try to relate their language to what they know about the world; large language models don’t. Which means the latter just aren’t telling us much about the former.","locator":"paragraph 12","url":"https://garymarcus.substack.com/p/noam-chomsky-and-gpt-3","date":"2022-05-21","document_title":"Noam Chomsky and GPT-3","observed_at":"2026-10-29T00:05:23Z"},"criteria":{"from_speaker":[],"our_interpretation":[],"support_condition":"Not checked against evidence: both screens found that public evidence could not settle this claim.","contradiction_condition":"Not checked against evidence: both screens found that public evidence could not settle this claim."},"axis":"not_applicable","outcome":"not_adjudicable","horizon_state":"not_applicable","evaluation_as_of":"2026-10-28","rationale":"Not checked against evidence. Two independent screens, one from each provider, both found this too vague to test: informed readers would test it in different ways and reach different answers.","limitations":[],"evidence":[],"what_would_resolve":null,"retrospective":true,"uncheckable":["no_plain_reading"]},"payload_hash":"b4c6e67710e1252551002e635f54da58f815c6d20d10614514329bd7ee457cdc","policy_version":"2026-10-28.7","recorded_at":"2026-10-29T02:52:48Z","drafter":{"provider":"anthropic","model":"claude-opus-5-5"},"reviewers":[{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-noam-chomsky-and-gpt-3-screen-anthropic-3"},{"provider":"openai","model":"gpt-6-astra","run_id":"marcus-noam-chomsky-and-gpt-3-screen-openai-2"}],"final_text":null,"status":"current","notice":null,"superseded_by":null,"evidence_meta":[],"public_from":null,"public_until":null},"history":[],"review":{"review_due":false,"triggers":[],"last_completed_review_at":"2026-10-29T02:52:48Z","next_review_due_at":null,"horizon_now":"not_applicable"},"event_ids":["marcus-noam-chomsky-and-gpt-3-5c52cd-assessment-published-a1"],"last_substantive_change_at":"2026-10-29T17:00:18Z"}},{"url":"https://punditclaims.ai/claims/marcus-noam-chomsky-and-gpt-3-61b596","standing":"current","assessment_label":"Contradicted","versions":[{"assessment_id":"marcus-noam-chomsky-and-gpt-3-61b596-a1","status":"current","url":"https://punditclaims.ai/claims/marcus-noam-chomsky-and-gpt-3-61b596#assessment-marcus-noam-chomsky-and-gpt-3-61b596-a1"}],"claim":{"claim_id":"marcus-noam-chomsky-and-gpt-3-61b596","subject_id":"gary-marcus","revision":2,"content_hash":"2c62e82ea82e0354d1456456eaa175ad25b25fcde312d1491e49dc7af6edd975","normalized_proposition":"Systems like GPT-3 do not gravitate towards human language, and they never develop a cohesive understanding of the world, no matter how much experience they have.","kind":"forecast","target":"GPT-3","conditions":[{"text":"systems like GPT-3","type":"scope","from_source":true},{"text":"no matter how much experience they have","type":"qualifier","from_source":true}],"original_horizon_text":null,"topic_tags":["capabilities","scaling"],"concerns_providers":["openai"],"selection_stratum":"declared-lead","occurrences":[{"id":"marcus-noam-chomsky-and-gpt-3-61b596-o1","statement_date":"2022-05-21","statement_date_precision":"day","excerpt":"They do not gravitate towards human language, and they never develop a cohesive undestanding of the world, no matter how much experience they have.","locator":"paragraph 23","document":{"url":"https://garymarcus.substack.com/p/noam-chomsky-and-gpt-3","title":"Noam Chomsky and GPT-3","publisher":"Marcus on AI","published_at":"2022-05-21T17:45:07Z","date_uncertain":false,"date_provenance":"JSON-LD datePublished"},"observed_at":"2026-10-29T00:05:23Z","observed_via":"live","archive_url":null,"access_state":"full_public","snapshot_hash":"07d5a897e1c3fa996767148162ac6fcb20d0ba1228f4392f98d810f139f1b769","receipts":1}],"first_statement_date":"2022-05-21","related":[],"current":{"assessment_id":"marcus-noam-chomsky-and-gpt-3-61b596-a1","payload":{"claim_id":"marcus-noam-chomsky-and-gpt-3-61b596","subject_id":"gary-marcus","normalized_proposition":"Systems like GPT-3 do not gravitate towards human language, and they never develop a cohesive understanding of the world, no matter how much experience they have.","kind":"forecast","conditions":["systems like GPT-3","no matter how much experience they have"],"original_horizon_text":null,"statement":{"excerpt":"They do not gravitate towards human language, and they never develop a cohesive undestanding of the world, no matter how much experience they have.","locator":"paragraph 23","url":"https://garymarcus.substack.com/p/noam-chomsky-and-gpt-3","date":"2022-05-21","document_title":"Noam Chomsky and GPT-3","observed_at":"2026-10-29T00:05:23Z"},"criteria":{"from_speaker":["Systems like GPT-3","do not gravitate towards human language","never develop a cohesive undestanding of the world","no matter how much experience they have"],"our_interpretation":["I read this as two categorical claims about a model family, including a permanent limitation, without a deadline.","I interpret systems like GPT-3 to include GPT-style autoregressive transformer language models, including experimentally trained GPT-2 models. This does not restrict the claim to the released GPT-3 checkpoint.","The ordinary reading of gravitate towards human language is a learning preference for human-language patterns over appropriate artificial-language controls. It does not require an exclusively human learning capacity.","Cohesive understanding means reliable, generalizable tracking of objects, events and their spatial, temporal and causal relationships. Success in one restricted environment is insufficient.","A demonstrated counterexample to either categorical component contradicts the combined assertion. It does not establish that the other component is false.","No matter how much experience is a universal qualification, not a conditional antecedent requiring a particular training run."],"support_condition":"Evidence establishing both limitations throughout the stated model family and across additional training experience. Finite examples of failure can support present limitations but cannot establish never.","contradiction_condition":"Replicated controlled evaluations demonstrate a learning preference for human-language patterns in the stated model family, or demonstrate broad, consistent and generalizable world understanding without supplying a separate symbolic world model."},"axis":"forecast_outcome","outcome":"contradicted","horizon_state":"conclusive_event","evaluation_as_of":"2026-10-28","rationale":"The language-learning component has a controlled counterexample and independent corroboration. GPT-2 experiments show some preference for human-language patterns over altered alternatives. That contradicts the categorical absence of such a tendency under the stated reading of systems like GPT-3. The studies do not show that models learn exactly like humans. Hunter's criticism limits the stronger grammatical interpretation. The separate claim about never developing cohesive world understanding remains unsettled: finite failures cannot prove never, and restricted simulation successes do not establish broad understanding.","limitations":["The contradiction applies to the language-learning component of the combined assertion.","Including GPT-2 experiments within systems like GPT-3 is an explicit scope interpretation.","Preference for human-language patterns is weaker than a complete human-like language faculty.","Experimental dates are reported at year precision; availability dates identify the cited versions."],"evidence":[{"id":"e1","relation":"contradicts","excerpt":"struggles to learn","locator":"Abstract, core finding","url":"https://arxiv.org/abs/2401.06416","publisher":"arXiv","event_date":"2024","available_at":"2024-08-02","explanation":"Kallini and colleagues compared GPT-2 learning across English and systematically altered languages, including different training stages. The observed preference conflicts with the language-learning component under the stated family-level interpretation."},{"id":"e2","relation":"contradicts","excerpt":"human-like inductive biases","locator":"Abstract, final sentence","url":"https://arxiv.org/abs/2502.18795v2","publisher":"arXiv","event_date":"2025","available_at":"2025-05-31","explanation":"Yang and colleagues extended testing to 12 languages from four families and found partial discrimination between attested and impossible patterns. This independently corroborates a preference, while explicitly rejecting equivalence with human learning."},{"id":"e3","relation":"context","excerpt":"does not actually test","locator":"Discussion of the WordHop–NoHop comparison","url":"https://arxiv.org/html/2410.12271v1","publisher":"arXiv","event_date":"2024","available_at":"2024-10-16","explanation":"Hunter identifies a confound between counting rules and non-adjacent dependencies. This limits claims that the initial experiment demonstrates specifically human grammatical constraints, although that stronger conclusion is not required by the ordinary preference criterion."},{"id":"e4","relation":"context","excerpt":"incoherence creates fragility","locator":"Abstract, sentence on generalization failures","url":"https://arxiv.org/abs/2406.03689","publisher":"arXiv","event_date":"2024","available_at":"2024-11-10","explanation":"Vafa and colleagues find weaknesses in world-model coherence across game, logic and navigation settings. These findings support concern about tested systems but cannot establish failure at every future training scale."},{"id":"e5","relation":"context","excerpt":"implicit text-based world models.","locator":"Section 8, Conclusion","url":"https://arxiv.org/html/2512.18832v2","publisher":"arXiv","event_date":"2026","available_at":"2026-03-05","explanation":"Li and colleagues report coherent state tracking and useful simulation in five text environments after targeted training. The restricted environments and dependence on training coverage prevent treating this as decisive evidence of broad world understanding."}],"what_would_resolve":"Broad controlled evaluations of state tracking, causal reasoning and transfer across unfamiliar domains would address the remaining world-understanding component.","retrospective":true,"plain_reading":true},"payload_hash":"365fe04438ec3e9d356ef0854d5b342d6823d57d0bacd877ec6901cdc6f247a2","policy_version":"2026-10-28.7","recorded_at":"2026-10-29T08:01:38Z","drafter":{"provider":"openai","model":"gpt-6-astra"},"reviewers":[{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-noam-chomsky-and-gpt-3-61b596-4x-review-anthropic-1"},{"provider":"openai","model":"gpt-6-astra","run_id":"marcus-noam-chomsky-and-gpt-3-61b596-4x-review-openai-1"}],"final_text":{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-noam-chomsky-and-gpt-3-61b596-4x-final-anthropic-1"},"status":"current","notice":null,"superseded_by":null,"evidence_meta":[{"id":"e1","source_type":"preprint","audited":null,"derived":null,"metric":null},{"id":"e2","source_type":"preprint","audited":null,"derived":null,"metric":null},{"id":"e3","source_type":"preprint","audited":null,"derived":null,"metric":null},{"id":"e4","source_type":"preprint","audited":null,"derived":null,"metric":null},{"id":"e5","source_type":"preprint","audited":null,"derived":null,"metric":null}],"public_from":null,"public_until":null},"history":[],"review":{"review_due":false,"triggers":[],"last_completed_review_at":"2026-10-29T08:01:38Z","next_review_due_at":null,"horizon_now":"conclusive_event"},"event_ids":["marcus-noam-chomsky-and-gpt-3-61b596-assessment-published-a1"],"last_substantive_change_at":"2026-10-29T17:00:18Z"}},{"url":"https://punditclaims.ai/claims/marcus-noam-chomsky-and-gpt-3-960ed6","standing":"current","assessment_label":"Can’t be checked","versions":[{"assessment_id":"marcus-noam-chomsky-and-gpt-3-960ed6-a1","status":"current","url":"https://punditclaims.ai/claims/marcus-noam-chomsky-and-gpt-3-960ed6#assessment-marcus-noam-chomsky-and-gpt-3-960ed6-a1"}],"claim":{"claim_id":"marcus-noam-chomsky-and-gpt-3-960ed6","subject_id":"gary-marcus","revision":2,"content_hash":"87e6b01a01fd61b140b9c6faeeba2f4dda8f721ce61c36cc510ce8ace03b9892","normalized_proposition":"Because large language models could glom onto any other data set (e.g. programming languages) just as well as human data, they tell us far too little about how humans have come to be what they are.","kind":"causal","target":"large language models","conditions":[],"original_horizon_text":null,"topic_tags":["capabilities"],"concerns_providers":[],"selection_stratum":"declared-lead","occurrences":[{"id":"marcus-noam-chomsky-and-gpt-3-960ed6-o1","statement_date":"2022-05-21","statement_date_precision":"day","excerpt":"Large language models can, parasitically, glom onto human data, but they could glom onto any other data set just as well (e.g. programming languages); as such, they tell us far too little about how humans have come to be what they are.","locator":"paragraph 42","document":{"url":"https://garymarcus.substack.com/p/noam-chomsky-and-gpt-3","title":"Noam Chomsky and GPT-3","publisher":"Marcus on AI","published_at":"2022-05-21T17:45:07Z","date_uncertain":false,"date_provenance":"JSON-LD datePublished"},"observed_at":"2026-10-29T00:05:23Z","observed_via":"live","archive_url":null,"access_state":"full_public","snapshot_hash":"07d5a897e1c3fa996767148162ac6fcb20d0ba1228f4392f98d810f139f1b769","receipts":1}],"first_statement_date":"2022-05-21","related":[],"current":{"assessment_id":"marcus-noam-chomsky-and-gpt-3-960ed6-a1","payload":{"claim_id":"marcus-noam-chomsky-and-gpt-3-960ed6","subject_id":"gary-marcus","normalized_proposition":"Because large language models could glom onto any other data set (e.g. programming languages) just as well as human data, they tell us far too little about how humans have come to be what they are.","kind":"causal","conditions":[],"original_horizon_text":null,"statement":{"excerpt":"Large language models can, parasitically, glom onto human data, but they could glom onto any other data set just as well (e.g. programming languages); as such, they tell us far too little about how humans have come to be what they are.","locator":"paragraph 42","url":"https://garymarcus.substack.com/p/noam-chomsky-and-gpt-3","date":"2022-05-21","document_title":"Noam Chomsky and GPT-3","observed_at":"2026-10-29T00:05:23Z"},"criteria":{"from_speaker":[],"our_interpretation":[],"support_condition":"Not checked against evidence: both screens found that public evidence could not settle this claim.","contradiction_condition":"Not checked against evidence: both screens found that public evidence could not settle this claim."},"axis":"not_applicable","outcome":"not_adjudicable","horizon_state":"not_applicable","evaluation_as_of":"2026-10-28","rationale":"Not checked against evidence. Two independent screens, one from each provider, both found this too vague to test: informed readers would test it in different ways and reach different answers.","limitations":[],"evidence":[],"what_would_resolve":null,"retrospective":true,"uncheckable":["no_plain_reading"]},"payload_hash":"2ad5af021aeeb8e1eb42e3bb83d0fcaf8219d9fd631f9396bce2f0ce350567c2","policy_version":"2026-10-28.7","recorded_at":"2026-10-29T02:52:48Z","drafter":{"provider":"anthropic","model":"claude-opus-5-5"},"reviewers":[{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-noam-chomsky-and-gpt-3-screen-anthropic-3"},{"provider":"openai","model":"gpt-6-astra","run_id":"marcus-noam-chomsky-and-gpt-3-screen-openai-2"}],"final_text":null,"status":"current","notice":null,"superseded_by":null,"evidence_meta":[],"public_from":null,"public_until":null},"history":[],"review":{"review_due":false,"triggers":[],"last_completed_review_at":"2026-10-29T02:52:48Z","next_review_due_at":null,"horizon_now":"not_applicable"},"event_ids":["marcus-noam-chomsky-and-gpt-3-960ed6-assessment-published-a1"],"last_substantive_change_at":"2026-10-29T17:00:18Z"}},{"url":"https://punditclaims.ai/claims/marcus-noam-chomsky-and-gpt-3-c476d0","standing":"current","assessment_label":"Can’t be checked","versions":[{"assessment_id":"marcus-noam-chomsky-and-gpt-3-c476d0-a1","status":"current","url":"https://punditclaims.ai/claims/marcus-noam-chomsky-and-gpt-3-c476d0#assessment-marcus-noam-chomsky-and-gpt-3-c476d0-a1"}],"claim":{"claim_id":"marcus-noam-chomsky-and-gpt-3-c476d0","subject_id":"gary-marcus","revision":2,"content_hash":"61183b508574b75b13ea285a234c790f3c94fd6b95c5d32966e0ec948cf9a974","normalized_proposition":"Because systems like GPT-3 don't tell us why human languages have the special character that they do, they have little explanatory value.","kind":"causal","target":"GPT-3","conditions":[{"text":"systems like GPT-3","type":"scope","from_source":true}],"original_horizon_text":null,"topic_tags":["capabilities"],"concerns_providers":["openai"],"selection_stratum":"declared-lead","occurrences":[{"id":"marcus-noam-chomsky-and-gpt-3-c476d0-o1","statement_date":"2022-05-21","statement_date_precision":"day","excerpt":"Systems like GPT-3 don’t tell us why human languages have the special character that they do. As such, there is little explanatory value.","locator":"paragraph 15","document":{"url":"https://garymarcus.substack.com/p/noam-chomsky-and-gpt-3","title":"Noam Chomsky and GPT-3","publisher":"Marcus on AI","published_at":"2022-05-21T17:45:07Z","date_uncertain":false,"date_provenance":"JSON-LD datePublished"},"observed_at":"2026-10-29T00:05:23Z","observed_via":"live","archive_url":null,"access_state":"full_public","snapshot_hash":"07d5a897e1c3fa996767148162ac6fcb20d0ba1228f4392f98d810f139f1b769","receipts":1}],"first_statement_date":"2022-05-21","related":[],"current":{"assessment_id":"marcus-noam-chomsky-and-gpt-3-c476d0-a1","payload":{"claim_id":"marcus-noam-chomsky-and-gpt-3-c476d0","subject_id":"gary-marcus","normalized_proposition":"Because systems like GPT-3 don't tell us why human languages have the special character that they do, they have little explanatory value.","kind":"causal","conditions":["systems like GPT-3"],"original_horizon_text":null,"statement":{"excerpt":"Systems like GPT-3 don’t tell us why human languages have the special character that they do. As such, there is little explanatory value.","locator":"paragraph 15","url":"https://garymarcus.substack.com/p/noam-chomsky-and-gpt-3","date":"2022-05-21","document_title":"Noam Chomsky and GPT-3","observed_at":"2026-10-29T00:05:23Z"},"criteria":{"from_speaker":["systems like GPT-3"],"our_interpretation":[],"support_condition":"Not checked against evidence: both screens found that public evidence could not settle this claim.","contradiction_condition":"Not checked against evidence: both screens found that public evidence could not settle this claim."},"axis":"not_applicable","outcome":"not_adjudicable","horizon_state":"not_applicable","evaluation_as_of":"2026-10-28","rationale":"Not checked against evidence. Two independent screens, one from each provider, both found this too vague to test: informed readers would test it in different ways and reach different answers.","limitations":[],"evidence":[],"what_would_resolve":null,"retrospective":true,"uncheckable":["no_plain_reading"]},"payload_hash":"c343c5dbcdd80b359ffa2adba74783defaedc946bc5a0a092595649f10ecef18","policy_version":"2026-10-28.7","recorded_at":"2026-10-29T02:52:48Z","drafter":{"provider":"anthropic","model":"claude-opus-5-5"},"reviewers":[{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-noam-chomsky-and-gpt-3-screen-anthropic-3"},{"provider":"openai","model":"gpt-6-astra","run_id":"marcus-noam-chomsky-and-gpt-3-screen-openai-2"}],"final_text":null,"status":"current","notice":null,"superseded_by":null,"evidence_meta":[],"public_from":null,"public_until":null},"history":[],"review":{"review_due":false,"triggers":[],"last_completed_review_at":"2026-10-29T02:52:48Z","next_review_due_at":null,"horizon_now":"not_applicable"},"event_ids":["marcus-noam-chomsky-and-gpt-3-c476d0-assessment-published-a1"],"last_substantive_change_at":"2026-10-29T17:00:18Z"}},{"url":"https://punditclaims.ai/claims/marcus-noam-chomsky-and-gpt-3-c49391","standing":"current","assessment_label":"Can’t be checked","versions":[{"assessment_id":"marcus-noam-chomsky-and-gpt-3-c49391-a1","status":"current","url":"https://punditclaims.ai/claims/marcus-noam-chomsky-and-gpt-3-c49391#assessment-marcus-noam-chomsky-and-gpt-3-c49391-a1"}],"claim":{"claim_id":"marcus-noam-chomsky-and-gpt-3-c49391","subject_id":"gary-marcus","revision":1,"content_hash":"0ad4fb8b12a36125fdc9578b19b042852e13c615c618fa3d16a91c6f214531d2","normalized_proposition":"For AI to understand language, it should process language through built-in commonsense understanding instead of starting as a blank slate and learning statistics.","kind":"normative","target":"AI systems","conditions":[{"text":"if AI's are to understand language","type":"qualifier","from_source":true}],"original_horizon_text":null,"topic_tags":["capabilities"],"concerns_providers":[],"selection_stratum":"declared-lead","occurrences":[{"id":"marcus-noam-chomsky-and-gpt-3-c49391-o1","statement_date":"2022-05-21","statement_date_precision":"day","excerpt":"Humans know these things; if AI’s are to understand language, they should refract language through this sort basic understanding of common sense, rather than needlessly starting as blank slates and acquiring statistics without concepts.","locator":"paragraph 25","document":{"url":"https://garymarcus.substack.com/p/noam-chomsky-and-gpt-3","title":"Noam Chomsky and GPT-3","publisher":"Marcus on AI","published_at":"2022-05-21T17:45:07Z","date_uncertain":false,"date_provenance":"JSON-LD datePublished"},"observed_at":"2026-10-29T00:05:23Z","observed_via":"live","archive_url":null,"access_state":"full_public","snapshot_hash":"07d5a897e1c3fa996767148162ac6fcb20d0ba1228f4392f98d810f139f1b769","receipts":1}],"first_statement_date":"2022-05-21","related":[],"current":{"assessment_id":"marcus-noam-chomsky-and-gpt-3-c49391-a1","payload":{"claim_id":"marcus-noam-chomsky-and-gpt-3-c49391","subject_id":"gary-marcus","normalized_proposition":"For AI to understand language, it should process language through built-in commonsense understanding instead of starting as a blank slate and learning statistics.","kind":"normative","conditions":["if AI's are to understand language"],"original_horizon_text":null,"statement":{"excerpt":"Humans know these things; if AI’s are to understand language, they should refract language through this sort basic understanding of common sense, rather than needlessly starting as blank slates and acquiring statistics without concepts.","locator":"paragraph 25","url":"https://garymarcus.substack.com/p/noam-chomsky-and-gpt-3","date":"2022-05-21","document_title":"Noam Chomsky and GPT-3","observed_at":"2026-10-29T00:05:23Z"},"criteria":{"from_speaker":["if AI's are to understand language"],"our_interpretation":[],"support_condition":"Not checked against evidence: both screens found that public evidence could not settle this claim.","contradiction_condition":"Not checked against evidence: both screens found that public evidence could not settle this claim."},"axis":"not_applicable","outcome":"not_adjudicable","horizon_state":"not_applicable","evaluation_as_of":"2026-10-28","rationale":"Not checked against evidence. Two independent screens, one from each provider, both found this a recommendation or a value judgment.","limitations":[],"evidence":[],"what_would_resolve":null,"retrospective":true,"uncheckable":["value_judgment"]},"payload_hash":"1b9c1b324bf2ee9ee53454040cc49733a43822d9266d9de02d23116d69219728","policy_version":"2026-10-28.7","recorded_at":"2026-10-29T02:32:05Z","drafter":{"provider":"anthropic","model":"claude-opus-5-5"},"reviewers":[{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-noam-chomsky-and-gpt-3-screen-anthropic-2"},{"provider":"openai","model":"gpt-6-astra","run_id":"marcus-noam-chomsky-and-gpt-3-screen-openai-1"}],"final_text":null,"status":"current","notice":null,"superseded_by":null,"evidence_meta":[],"public_from":null,"public_until":null},"history":[],"review":{"review_due":false,"triggers":[],"last_completed_review_at":"2026-10-29T02:32:05Z","next_review_due_at":null,"horizon_now":"not_applicable"},"event_ids":["marcus-noam-chomsky-and-gpt-3-c49391-assessment-published-a1"],"last_substantive_change_at":"2026-10-29T17:00:18Z"}},{"url":"https://punditclaims.ai/claims/marcus-noam-chomsky-and-gpt-3-d8d192","standing":"current","assessment_label":"Can’t be checked","versions":[{"assessment_id":"marcus-noam-chomsky-and-gpt-3-d8d192-a1","status":"current","url":"https://punditclaims.ai/claims/marcus-noam-chomsky-and-gpt-3-d8d192#assessment-marcus-noam-chomsky-and-gpt-3-d8d192-a1"}],"claim":{"claim_id":"marcus-noam-chomsky-and-gpt-3-d8d192","subject_id":"gary-marcus","revision":1,"content_hash":"43c5ebfa6ccdb545d5bf3a45defb6dc15e05251d7639900cd7c5e1e30d6cc37b","normalized_proposition":"LLMs' failures argue strongly for exploring Kantian AI systems that start with built-in representations of space, time, causality and some universal grammar.","kind":"normative","target":"AI research","conditions":[],"original_horizon_text":null,"topic_tags":["capabilities"],"concerns_providers":[],"selection_stratum":"declared-lead","occurrences":[{"id":"marcus-noam-chomsky-and-gpt-3-d8d192-o1","statement_date":"2022-05-21","statement_date_precision":"day","excerpt":"They are an embarrassment to Locke’s conjecture, and a strong argument for exploring their very antithesis, which would be Kantian systems that start with representations of space and time and causality, and at least some kernels of universal grammar.","locator":"paragraph 23","document":{"url":"https://garymarcus.substack.com/p/noam-chomsky-and-gpt-3","title":"Noam Chomsky and GPT-3","publisher":"Marcus on AI","published_at":"2022-05-21T17:45:07Z","date_uncertain":false,"date_provenance":"JSON-LD datePublished"},"observed_at":"2026-10-29T00:05:23Z","observed_via":"live","archive_url":null,"access_state":"full_public","snapshot_hash":"07d5a897e1c3fa996767148162ac6fcb20d0ba1228f4392f98d810f139f1b769","receipts":1}],"first_statement_date":"2022-05-21","related":[],"current":{"assessment_id":"marcus-noam-chomsky-and-gpt-3-d8d192-a1","payload":{"claim_id":"marcus-noam-chomsky-and-gpt-3-d8d192","subject_id":"gary-marcus","normalized_proposition":"LLMs' failures argue strongly for exploring Kantian AI systems that start with built-in representations of space, time, causality and some universal grammar.","kind":"normative","conditions":[],"original_horizon_text":null,"statement":{"excerpt":"They are an embarrassment to Locke’s conjecture, and a strong argument for exploring their very antithesis, which would be Kantian systems that start with representations of space and time and causality, and at least some kernels of universal grammar.","locator":"paragraph 23","url":"https://garymarcus.substack.com/p/noam-chomsky-and-gpt-3","date":"2022-05-21","document_title":"Noam Chomsky and GPT-3","observed_at":"2026-10-29T00:05:23Z"},"criteria":{"from_speaker":[],"our_interpretation":[],"support_condition":"Not checked against evidence: both screens found that public evidence could not settle this claim.","contradiction_condition":"Not checked against evidence: both screens found that public evidence could not settle this claim."},"axis":"not_applicable","outcome":"not_adjudicable","horizon_state":"not_applicable","evaluation_as_of":"2026-10-28","rationale":"Not checked against evidence. Two independent screens, one from each provider, both found this a recommendation or a value judgment.","limitations":[],"evidence":[],"what_would_resolve":null,"retrospective":true,"uncheckable":["value_judgment"]},"payload_hash":"d2d97e20f21ab48907d94bbae1a1ad953625e7fbe209bbe10c1ac8566f94671f","policy_version":"2026-10-28.7","recorded_at":"2026-10-29T02:32:05Z","drafter":{"provider":"anthropic","model":"claude-opus-5-5"},"reviewers":[{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-noam-chomsky-and-gpt-3-screen-anthropic-2"},{"provider":"openai","model":"gpt-6-astra","run_id":"marcus-noam-chomsky-and-gpt-3-screen-openai-1"}],"final_text":null,"status":"current","notice":null,"superseded_by":null,"evidence_meta":[],"public_from":null,"public_until":null},"history":[],"review":{"review_due":false,"triggers":[],"last_completed_review_at":"2026-10-29T02:32:05Z","next_review_due_at":null,"horizon_now":"not_applicable"},"event_ids":["marcus-noam-chomsky-and-gpt-3-d8d192-assessment-published-a1"],"last_substantive_change_at":"2026-10-29T17:00:18Z"}},{"url":"https://punditclaims.ai/claims/marcus-noam-chomsky-and-gpt-3-f648bc","standing":"current","assessment_label":"Unresolved","versions":[{"assessment_id":"marcus-noam-chomsky-and-gpt-3-f648bc-a1","status":"current","url":"https://punditclaims.ai/claims/marcus-noam-chomsky-and-gpt-3-f648bc#assessment-marcus-noam-chomsky-and-gpt-3-f648bc-a1"}],"claim":{"claim_id":"marcus-noam-chomsky-and-gpt-3-f648bc","subject_id":"gary-marcus","revision":1,"content_hash":"ee5e9456c1bae3baea7524b27fadd8094ae0eb790781414d8348c839ad8ed9a6","normalized_proposition":"GPT-3 cannot avoid producing misinformation because it models how words relate to each other, not how language relates to the world.","kind":"causal","target":"GPT-3","conditions":[],"original_horizon_text":null,"topic_tags":["reliability","safety"],"concerns_providers":["openai"],"selection_stratum":"declared-lead","occurrences":[{"id":"marcus-noam-chomsky-and-gpt-3-f648bc-o1","statement_date":"2022-05-21","statement_date_precision":"day","excerpt":"There is no intention to create misinformation—but also no capacity to avoid it, because fundamentally, GPT-3 is a model of how words relate to one another, not a model of how language might relate to the perceived world.","locator":"paragraph 12","document":{"url":"https://garymarcus.substack.com/p/noam-chomsky-and-gpt-3","title":"Noam Chomsky and GPT-3","publisher":"Marcus on AI","published_at":"2022-05-21T17:45:07Z","date_uncertain":false,"date_provenance":"JSON-LD datePublished"},"observed_at":"2026-10-29T00:05:23Z","observed_via":"live","archive_url":null,"access_state":"full_public","snapshot_hash":"07d5a897e1c3fa996767148162ac6fcb20d0ba1228f4392f98d810f139f1b769","receipts":1}],"first_statement_date":"2022-05-21","related":[],"current":{"assessment_id":"marcus-noam-chomsky-and-gpt-3-f648bc-a1","payload":{"claim_id":"marcus-noam-chomsky-and-gpt-3-f648bc","subject_id":"gary-marcus","normalized_proposition":"GPT-3 cannot avoid producing misinformation because it models how words relate to each other, not how language relates to the world.","kind":"causal","conditions":[],"original_horizon_text":null,"statement":{"excerpt":"There is no intention to create misinformation—but also no capacity to avoid it, because fundamentally, GPT-3 is a model of how words relate to one another, not a model of how language might relate to the perceived world.","locator":"paragraph 12","url":"https://garymarcus.substack.com/p/noam-chomsky-and-gpt-3","date":"2022-05-21","document_title":"Noam Chomsky and GPT-3","observed_at":"2026-10-29T00:05:23Z"},"criteria":{"from_speaker":["no intention to create misinformation","but also no capacity to avoid it","because fundamentally, GPT-3 is a model of how words relate to one another","not a model of how language might relate to the perceived world"],"our_interpretation":["Two parts. (a) GPT-3 had no reliable built-in way to avoid producing false content. (b) The cause is that it is trained only on relations among words, without grounding in the perceived world.","Read fairly, 'no capacity to avoid it' means it cannot reliably avoid misinformation, not that it never says anything true."],"support_condition":"Controlled evidence that GPT-3-era models' misinformation stems from text-only training, and that grounding in the world is what would be needed to fix it.","contradiction_condition":"Evidence that GPT-3-era models without world grounding could reliably avoid misinformation, or that the errors had a different main cause."},"axis":"causal_assessment","outcome":"unresolved","horizon_state":"not_applicable","evaluation_as_of":"2026-10-28","rationale":"The claim has an empirical part and a causal part. The evidence fits the empirical part read fairly. GPT-3 and InstructGPT kept making up information, at 41% and 21% on closed-domain tasks (e2). But fine-tuning without any grounding in the world halved that rate, which shows some capacity to reduce misinformation. TruthfulQA's authors tie false answers to imitating training text (e1), which partly fits Marcus's story. No cited study isolates the cause he names, being a model of word relations rather than of the world, from other factors. The causal part therefore stays open on the available evidence.","limitations":["The evidence is about error rates, not about mechanism.","Settling this would need a controlled grounded-versus-ungrounded comparison, which no source here provides."],"evidence":[{"id":"e1","relation":"context","excerpt":"However, this result is expected if false answers are learned from the training distribution.","locator":"Abstract","url":"https://arxiv.org/abs/2109.07958","publisher":"ACL 2022 (Lin, Hilton, Evans), via arXiv","event_date":"2021-09","available_at":"2021-09-08","explanation":"Links falsehoods to imitating training text, which partly fits Marcus's word-statistics story. It does not test whether grounding in the world is the missing factor."},{"id":"e2","relation":"context","excerpt":"InstructGPT models make up information not present in the input about half as often as GPT-3 (a\n21% vs. 41% hallucination rate, respectively).","locator":"Section 1 Introduction, summary of findings","url":"https://arxiv.org/pdf/2203.02155","publisher":"arXiv (OpenAI authors)","event_date":"2022-03","available_at":"2022-03-04","explanation":"Human-feedback fine-tuning, with no grounding in the world, halved fabrication but left it at 21%. This fits 'cannot reliably avoid' and also shows some capacity to reduce errors, so the causal story stays unsettled."}],"what_would_resolve":"Controlled studies comparing text-only and world-grounded models of similar scale on misinformation rates, or mechanistic work showing why GPT-3-era models fabricate.","retrospective":true,"plain_reading":true},"payload_hash":"0806e439dcf4d61d7983acdaccc30f4b9a89a7470ec219625c4c92fd69166df7","policy_version":"2026-10-28.7","recorded_at":"2026-10-29T08:01:38Z","drafter":{"provider":"anthropic","model":"claude-opus-5-5"},"reviewers":[{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-noam-chomsky-and-gpt-3-61b596-4x-review-anthropic-1"},{"provider":"openai","model":"gpt-6-astra","run_id":"marcus-noam-chomsky-and-gpt-3-61b596-4x-review-openai-1"}],"final_text":{"provider":"openai","model":"gpt-6-astra","run_id":"marcus-noam-chomsky-and-gpt-3-61b596-4x-final-openai-1"},"status":"current","notice":null,"superseded_by":null,"evidence_meta":[{"id":"e1","source_type":"peer_reviewed","audited":null,"derived":null,"metric":null},{"id":"e2","source_type":"preprint","audited":null,"derived":null,"metric":null}],"public_from":null,"public_until":null},"history":[],"review":{"review_due":false,"triggers":[],"last_completed_review_at":"2026-10-29T08:01:38Z","next_review_due_at":null,"horizon_now":"not_applicable"},"event_ids":["marcus-noam-chomsky-and-gpt-3-f648bc-assessment-published-a1"],"last_substantive_change_at":"2026-10-29T17:00:18Z"}},{"url":"https://punditclaims.ai/claims/marcus-the-new-science-of-alt-intelligence-11b850","standing":"current","assessment_label":"Unresolved","versions":[{"assessment_id":"marcus-the-new-science-of-alt-intelligence-11b850-a2","status":"current","url":"https://punditclaims.ai/claims/marcus-the-new-science-of-alt-intelligence-11b850#assessment-marcus-the-new-science-of-alt-intelligence-11b850-a2"}],"claim":{"claim_id":"marcus-the-new-science-of-alt-intelligence-11b850","subject_id":"gary-marcus","revision":2,"content_hash":"0a921f013d37c361ccddd49271142df8cdfe121826a4edd09ca397ce4c6c9e6f","normalized_proposition":"If we are to build AGI, we will need to learn something from humans: how they reason and understand the physical world, and how they represent and acquire language and complex concepts.","kind":"conditional_forecast","target":"AGI","conditions":[{"text":"if we are to build AGI","type":"antecedent","from_source":true},{"text":"learn something from humans","type":"qualifier","from_source":true}],"original_horizon_text":null,"topic_tags":["agi-timelines"],"concerns_providers":[],"selection_stratum":"declared-lead","occurrences":[{"id":"marcus-the-new-science-of-alt-intelligence-11b850-o1","statement_date":"2022-05-14","statement_date_precision":"day","excerpt":"The bottom line is this, something that AI once cherished but has now forgotten: If we are to build AGI, we are going to need to learn something from humans, how they reason and understand the physical world, and how they represent and acquire language and complex concepts.","locator":"paragraph 72","document":{"url":"https://garymarcus.substack.com/p/the-new-science-of-alt-intelligence","title":"The New Science of Alt Intelligence","publisher":"Marcus on AI","published_at":"2022-05-14T17:02:35Z","date_uncertain":false,"date_provenance":"JSON-LD datePublished"},"observed_at":"2026-10-29T00:05:24Z","observed_via":"live","archive_url":null,"access_state":"full_public","snapshot_hash":"b5456ff99a853d55856ae13c7d7571ff3970a73d8fefc55e7d34fd90527b6f81","receipts":1}],"first_statement_date":"2022-05-14","related":[],"current":{"assessment_id":"marcus-the-new-science-of-alt-intelligence-11b850-a2","payload":{"claim_id":"marcus-the-new-science-of-alt-intelligence-11b850","subject_id":"gary-marcus","normalized_proposition":"If we are to build AGI, we will need to learn something from humans: how they reason and understand the physical world, and how they represent and acquire language and complex concepts.","kind":"conditional_forecast","conditions":["if we are to build AGI","learn something from humans"],"original_horizon_text":null,"statement":{"excerpt":"The bottom line is this, something that AI once cherished but has now forgotten: If we are to build AGI, we are going to need to learn something from humans, how they reason and understand the physical world, and how they represent and acquire language and complex concepts.","locator":"paragraph 72","url":"https://garymarcus.substack.com/p/the-new-science-of-alt-intelligence","date":"2022-05-14","document_title":"The New Science of Alt Intelligence","observed_at":"2026-10-29T00:05:24Z"},"criteria":{"from_speaker":["If we are to build AGI","we are going to need to learn something from humans","how they reason and understand the physical world","how they represent and acquire language and complex concepts"],"our_interpretation":["Read this as a necessary-condition prediction about building AGI, rather than merely advice.","The antecedent is an established AGI achievement. Learning from humans is the proposed necessary requirement, not an additional antecedent that must be assumed.","Use broadly human-level, general cognitive competence as the ordinary meaning of AGI; a company declaration or one benchmark score alone does not establish it.","Interpret learning from humans as substantive design insights from human cognition. Exposure to human-written training data alone does not establish that requirement.","A successful system using cognitive insights would show compatibility with the claim, but would not by itself prove those insights were necessary."],"support_condition":"An independently established AGI achievement is accompanied by credible evidence that substantive insights from human reasoning, world understanding, language, or concept acquisition were necessary to its development.","contradiction_condition":"An independently established AGI system is developed without those substantive cognitive insights, with sufficiently complete development records to establish the counterexample."},"axis":"forecast_outcome","outcome":"unresolved","horizon_state":"open_horizon","evaluation_as_of":"2026-10-28","rationale":"ARC Prize's assessment, available September 3, 2026, describes Astra as progress toward generalization but explicitly declines to call it AGI [e7]. The AGI Society's September 14, 2026 update says there is no generally accepted empirical test for AGI and identifies gaps in the capability evidence [e8]. These assessments establish neither a qualifying AGI achievement nor the necessity of human cognitive insights, leaving the claim unresolved [e7, e8].","limitations":["These assessments do not establish that AGI has not been achieved elsewhere [e7, e8].","The evidence provides no qualifying system with an auditable development history sufficient to test the asserted necessity [e7, e8]."],"evidence":[{"id":"e7","relation":"context","excerpt":"not claiming","locator":"Section ARC-AGI Series","url":"https://arcprize.org/blog/astra","publisher":"ARC Prize","event_date":"2026-09","available_at":"2026-09-03","explanation":"The benchmark evaluator reports substantial progress but explicitly declines to treat it as an AGI achievement. This leaves the antecedent unestablished; it does not prove that AGI has not been built elsewhere."},{"id":"e8","relation":"context","excerpt":"There is no generally accepted empirical test for AGI.","locator":"Section Proposed tests of general intelligence","url":"https://agi-society.org/publications/evaluating-agi-claims","publisher":"AGI Society","event_date":"2026-09","available_at":"2026-09-14","explanation":"This assessment considers public declarations that AGI has arrived but concludes that the published evidence does not establish the achievement. September 14 is the disclosed update date of the version read; the assessment does not establish Marcus's proposed necessary development method."}],"what_would_resolve":"An independently established AGI achievement with auditable development records. A system developed without substantive insights from human cognition would contradict the claim. Support would require strong evidence that those insights were necessary, beyond showing that they were used.","retrospective":true,"plain_reading":true},"payload_hash":"b4bb480654e91dce796562a345c531175e82469a4f3416b91f8ce6887d754ab6","policy_version":"2026-10-28.7","recorded_at":"2026-10-29T12:42:08Z","drafter":{"provider":"openai","model":"gpt-6-astra"},"reviewers":[{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-the-new-science-of-alt-intelligence-11b850-2x-review-anthropic-1"},{"provider":"openai","model":"gpt-6-astra","run_id":"marcus-the-new-science-of-alt-intelligence-11b850-2x-review-openai-1"}],"final_text":{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-the-new-science-of-alt-intelligence-11b850-2x-final-anthropic-2"},"status":"current","notice":null,"superseded_by":null,"evidence_meta":[{"id":"e7","source_type":"benchmark","audited":null,"derived":null,"metric":null},{"id":"e8","source_type":"secondary_summary","audited":null,"derived":null,"metric":null}],"public_from":null,"public_until":null},"history":[],"review":{"review_due":false,"triggers":[],"last_completed_review_at":"2026-10-29T12:42:08Z","next_review_due_at":null,"horizon_now":"open_horizon"},"event_ids":["marcus-the-new-science-of-alt-intelligence-11b850-assessment-published-a2"],"last_substantive_change_at":"2026-10-29T17:00:18Z"}},{"url":"https://punditclaims.ai/claims/marcus-the-new-science-of-alt-intelligence-174e93","standing":"current","assessment_label":"Can’t be checked","versions":[{"assessment_id":"marcus-the-new-science-of-alt-intelligence-174e93-a1","status":"current","url":"https://punditclaims.ai/claims/marcus-the-new-science-of-alt-intelligence-174e93#assessment-marcus-the-new-science-of-alt-intelligence-174e93-a1"}],"claim":{"claim_id":"marcus-the-new-science-of-alt-intelligence-174e93","subject_id":"gary-marcus","revision":1,"content_hash":"4f4445964373ece4edcd98724ce93f273b7cebababaaba07dbf440f4cec53113","normalized_proposition":"AI is still at an early stage of development.","kind":"observation","target":"AI field","conditions":[],"original_horizon_text":null,"topic_tags":["agi-timelines"],"concerns_providers":[],"selection_stratum":"declared-lead","occurrences":[{"id":"marcus-the-new-science-of-alt-intelligence-174e93-o1","statement_date":"2022-05-14","statement_date_precision":"day","excerpt":"We are in early days in AI.","locator":"paragraph 57","document":{"url":"https://garymarcus.substack.com/p/the-new-science-of-alt-intelligence","title":"The New Science of Alt Intelligence","publisher":"Marcus on AI","published_at":"2022-05-14T17:02:35Z","date_uncertain":false,"date_provenance":"JSON-LD datePublished"},"observed_at":"2026-10-29T00:05:24Z","observed_via":"live","archive_url":null,"access_state":"full_public","snapshot_hash":"b5456ff99a853d55856ae13c7d7571ff3970a73d8fefc55e7d34fd90527b6f81","receipts":1}],"first_statement_date":"2022-05-14","related":[],"current":{"assessment_id":"marcus-the-new-science-of-alt-intelligence-174e93-a1","payload":{"claim_id":"marcus-the-new-science-of-alt-intelligence-174e93","subject_id":"gary-marcus","normalized_proposition":"AI is still at an early stage of development.","kind":"observation","conditions":[],"original_horizon_text":null,"statement":{"excerpt":"We are in early days in AI.","locator":"paragraph 57","url":"https://garymarcus.substack.com/p/the-new-science-of-alt-intelligence","date":"2022-05-14","document_title":"The New Science of Alt Intelligence","observed_at":"2026-10-29T00:05:24Z"},"criteria":{"from_speaker":[],"our_interpretation":[],"support_condition":"Not checked against evidence: both screens found that public evidence could not settle this claim.","contradiction_condition":"Not checked against evidence: both screens found that public evidence could not settle this claim."},"axis":"not_applicable","outcome":"not_adjudicable","horizon_state":"not_applicable","evaluation_as_of":"2026-10-28","rationale":"Not checked against evidence. Two independent screens, one from each provider, both found this too vague to test: informed readers would test it in different ways and reach different answers.","limitations":[],"evidence":[],"what_would_resolve":null,"retrospective":true,"uncheckable":["no_plain_reading"]},"payload_hash":"24a5e02c159b07b841c3f5ed52722306f4a69889d3bc4ef559bab9e583fe30d2","policy_version":"2026-10-28.7","recorded_at":"2026-10-29T02:32:05Z","drafter":{"provider":"anthropic","model":"claude-opus-5-5"},"reviewers":[{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-the-new-science-of-alt-intelligence-screen-anthropic-2"},{"provider":"openai","model":"gpt-6-astra","run_id":"marcus-the-new-science-of-alt-intelligence-screen-openai-2"}],"final_text":null,"status":"current","notice":null,"superseded_by":null,"evidence_meta":[],"public_from":null,"public_until":null},"history":[],"review":{"review_due":false,"triggers":[],"last_completed_review_at":"2026-10-29T02:32:05Z","next_review_due_at":null,"horizon_now":"not_applicable"},"event_ids":["marcus-the-new-science-of-alt-intelligence-174e93-assessment-published-a1"],"last_substantive_change_at":"2026-10-29T17:00:18Z"}},{"url":"https://punditclaims.ai/claims/marcus-the-new-science-of-alt-intelligence-1ae1e2","standing":"current","assessment_label":"Can’t be checked","versions":[{"assessment_id":"marcus-the-new-science-of-alt-intelligence-1ae1e2-a1","status":"current","url":"https://punditclaims.ai/claims/marcus-the-new-science-of-alt-intelligence-1ae1e2#assessment-marcus-the-new-science-of-alt-intelligence-1ae1e2-a1"}],"claim":{"claim_id":"marcus-the-new-science-of-alt-intelligence-1ae1e2","subject_id":"gary-marcus","revision":2,"content_hash":"2a6a9bc7f8c0688a12618297db9e510947d3334d631467b598652335045ade4a","normalized_proposition":"Anyone who is candid will recognise that these kinds of errors by current AI systems reveal that something is, for now, deeply amiss.","kind":"observation","target":"current AI systems","conditions":[{"text":"for now","type":"qualifier","from_source":true},{"text":"anyone who is candid","type":"scope","from_source":true}],"original_horizon_text":null,"topic_tags":["reliability"],"concerns_providers":[],"selection_stratum":"declared-lead","occurrences":[{"id":"marcus-the-new-science-of-alt-intelligence-1ae1e2-o1","statement_date":"2022-05-14","statement_date_precision":"day","excerpt":"But anyone who is candid will recognize that these kinds of errors reveal that something is, for now, deeply amiss.","locator":"paragraph 34","document":{"url":"https://garymarcus.substack.com/p/the-new-science-of-alt-intelligence","title":"The New Science of Alt Intelligence","publisher":"Marcus on AI","published_at":"2022-05-14T17:02:35Z","date_uncertain":false,"date_provenance":"JSON-LD datePublished"},"observed_at":"2026-10-29T00:05:24Z","observed_via":"live","archive_url":null,"access_state":"full_public","snapshot_hash":"b5456ff99a853d55856ae13c7d7571ff3970a73d8fefc55e7d34fd90527b6f81","receipts":1}],"first_statement_date":"2022-05-14","related":[],"current":{"assessment_id":"marcus-the-new-science-of-alt-intelligence-1ae1e2-a1","payload":{"claim_id":"marcus-the-new-science-of-alt-intelligence-1ae1e2","subject_id":"gary-marcus","normalized_proposition":"Anyone who is candid will recognise that these kinds of errors by current AI systems reveal that something is, for now, deeply amiss.","kind":"observation","conditions":["for now","anyone who is candid"],"original_horizon_text":null,"statement":{"excerpt":"But anyone who is candid will recognize that these kinds of errors reveal that something is, for now, deeply amiss.","locator":"paragraph 34","url":"https://garymarcus.substack.com/p/the-new-science-of-alt-intelligence","date":"2022-05-14","document_title":"The New Science of Alt Intelligence","observed_at":"2026-10-29T00:05:24Z"},"criteria":{"from_speaker":["for now","anyone who is candid"],"our_interpretation":[],"support_condition":"Not checked against evidence: both screens found that public evidence could not settle this claim.","contradiction_condition":"Not checked against evidence: both screens found that public evidence could not settle this claim."},"axis":"not_applicable","outcome":"not_adjudicable","horizon_state":"not_applicable","evaluation_as_of":"2026-10-28","rationale":"Not checked against evidence. Two independent screens, one from each provider, both found this too vague to test: informed readers would test it in different ways and reach different answers.","limitations":[],"evidence":[],"what_would_resolve":null,"retrospective":true,"uncheckable":["no_plain_reading"]},"payload_hash":"210c4e46cb24283972851d2dd940ea48ef6e665e771d4e57518121d43f395e78","policy_version":"2026-10-28.7","recorded_at":"2026-10-29T03:08:15Z","drafter":{"provider":"anthropic","model":"claude-opus-5-5"},"reviewers":[{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-the-new-science-of-alt-intelligence-screen-anthropic-3"},{"provider":"openai","model":"gpt-6-astra","run_id":"marcus-the-new-science-of-alt-intelligence-screen-openai-3"}],"final_text":null,"status":"current","notice":null,"superseded_by":null,"evidence_meta":[],"public_from":null,"public_until":null},"history":[],"review":{"review_due":false,"triggers":[],"last_completed_review_at":"2026-10-29T03:08:15Z","next_review_due_at":null,"horizon_now":"not_applicable"},"event_ids":["marcus-the-new-science-of-alt-intelligence-1ae1e2-assessment-published-a1"],"last_substantive_change_at":"2026-10-29T17:00:18Z"}},{"url":"https://punditclaims.ai/claims/marcus-the-new-science-of-alt-intelligence-1cffad","standing":"current","assessment_label":"Can’t be checked","versions":[{"assessment_id":"marcus-the-new-science-of-alt-intelligence-1cffad-a1","status":"current","url":"https://punditclaims.ai/claims/marcus-the-new-science-of-alt-intelligence-1cffad#assessment-marcus-the-new-science-of-alt-intelligence-1cffad-a1"}],"claim":{"claim_id":"marcus-the-new-science-of-alt-intelligence-1cffad","subject_id":"gary-marcus","revision":1,"content_hash":"a9cd46a760a67bf811719ca0b73d717d95cf6bd6898ec802db5ef358582799dc","normalized_proposition":"The AI field should pursue multiple research directions rather than dismissing undeveloped ideas prematurely.","kind":"normative","target":"AI field","conditions":[],"original_horizon_text":null,"topic_tags":["other"],"concerns_providers":[],"selection_stratum":"declared-lead","occurrences":[{"id":"marcus-the-new-science-of-alt-intelligence-1cffad-o1","statement_date":"2022-05-14","statement_date_precision":"day","excerpt":"Let us all encourage a field that is open-minded enough to work in multiple directions, without prematurely dismissing ideas that happen to be not yet fully developed.","locator":"paragraph 68","document":{"url":"https://garymarcus.substack.com/p/the-new-science-of-alt-intelligence","title":"The New Science of Alt Intelligence","publisher":"Marcus on AI","published_at":"2022-05-14T17:02:35Z","date_uncertain":false,"date_provenance":"JSON-LD datePublished"},"observed_at":"2026-10-29T00:05:24Z","observed_via":"live","archive_url":null,"access_state":"full_public","snapshot_hash":"b5456ff99a853d55856ae13c7d7571ff3970a73d8fefc55e7d34fd90527b6f81","receipts":1}],"first_statement_date":"2022-05-14","related":[],"current":{"assessment_id":"marcus-the-new-science-of-alt-intelligence-1cffad-a1","payload":{"claim_id":"marcus-the-new-science-of-alt-intelligence-1cffad","subject_id":"gary-marcus","normalized_proposition":"The AI field should pursue multiple research directions rather than dismissing undeveloped ideas prematurely.","kind":"normative","conditions":[],"original_horizon_text":null,"statement":{"excerpt":"Let us all encourage a field that is open-minded enough to work in multiple directions, without prematurely dismissing ideas that happen to be not yet fully developed.","locator":"paragraph 68","url":"https://garymarcus.substack.com/p/the-new-science-of-alt-intelligence","date":"2022-05-14","document_title":"The New Science of Alt Intelligence","observed_at":"2026-10-29T00:05:24Z"},"criteria":{"from_speaker":[],"our_interpretation":[],"support_condition":"Not checked against evidence: both screens found that public evidence could not settle this claim.","contradiction_condition":"Not checked against evidence: both screens found that public evidence could not settle this claim."},"axis":"not_applicable","outcome":"not_adjudicable","horizon_state":"not_applicable","evaluation_as_of":"2026-10-28","rationale":"Not checked against evidence. Two independent screens, one from each provider, both found this a recommendation or a value judgment.","limitations":[],"evidence":[],"what_would_resolve":null,"retrospective":true,"uncheckable":["value_judgment"]},"payload_hash":"276fc3861db494854edead76950255ac0e8aeb4b3a654d2d62297ed1e5dbd484","policy_version":"2026-10-28.7","recorded_at":"2026-10-29T02:32:05Z","drafter":{"provider":"anthropic","model":"claude-opus-5-5"},"reviewers":[{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-the-new-science-of-alt-intelligence-screen-anthropic-2"},{"provider":"openai","model":"gpt-6-astra","run_id":"marcus-the-new-science-of-alt-intelligence-screen-openai-2"}],"final_text":null,"status":"current","notice":null,"superseded_by":null,"evidence_meta":[],"public_from":null,"public_until":null},"history":[],"review":{"review_due":false,"triggers":[],"last_completed_review_at":"2026-10-29T02:32:05Z","next_review_due_at":null,"horizon_now":"not_applicable"},"event_ids":["marcus-the-new-science-of-alt-intelligence-1cffad-assessment-published-a1"],"last_substantive_change_at":"2026-10-29T17:00:18Z"}},{"url":"https://punditclaims.ai/claims/marcus-the-new-science-of-alt-intelligence-2b01c1","standing":"current","assessment_label":"Can’t be checked","versions":[{"assessment_id":"marcus-the-new-science-of-alt-intelligence-2b01c1-a1","status":"current","url":"https://punditclaims.ai/claims/marcus-the-new-science-of-alt-intelligence-2b01c1#assessment-marcus-the-new-science-of-alt-intelligence-2b01c1-a1"}],"claim":{"claim_id":"marcus-the-new-science-of-alt-intelligence-2b01c1","subject_id":"gary-marcus","revision":1,"content_hash":"05d90b035989c190070f445517028c6938ee8ccf459c0f5f57a64bb95c12561a","normalized_proposition":"The best path to AGI may not be through data-driven Alt Intelligence approaches.","kind":"forecast","target":"Alt Intelligence / scaling","conditions":[],"original_horizon_text":null,"topic_tags":["agi-timelines","scaling"],"concerns_providers":[],"selection_stratum":"declared-lead","occurrences":[{"id":"marcus-the-new-science-of-alt-intelligence-2b01c1-o1","statement_date":"2022-05-14","statement_date_precision":"day","excerpt":"It may just be that the best path to artificial (general) intelligence isn’t through Alt Intelligence, after all.","locator":"paragraph 68","document":{"url":"https://garymarcus.substack.com/p/the-new-science-of-alt-intelligence","title":"The New Science of Alt Intelligence","publisher":"Marcus on AI","published_at":"2022-05-14T17:02:35Z","date_uncertain":false,"date_provenance":"JSON-LD datePublished"},"observed_at":"2026-10-29T00:05:24Z","observed_via":"live","archive_url":null,"access_state":"full_public","snapshot_hash":"b5456ff99a853d55856ae13c7d7571ff3970a73d8fefc55e7d34fd90527b6f81","receipts":1}],"first_statement_date":"2022-05-14","related":[],"current":{"assessment_id":"marcus-the-new-science-of-alt-intelligence-2b01c1-a1","payload":{"claim_id":"marcus-the-new-science-of-alt-intelligence-2b01c1","subject_id":"gary-marcus","normalized_proposition":"The best path to AGI may not be through data-driven Alt Intelligence approaches.","kind":"forecast","conditions":[],"original_horizon_text":null,"statement":{"excerpt":"It may just be that the best path to artificial (general) intelligence isn’t through Alt Intelligence, after all.","locator":"paragraph 68","url":"https://garymarcus.substack.com/p/the-new-science-of-alt-intelligence","date":"2022-05-14","document_title":"The New Science of Alt Intelligence","observed_at":"2026-10-29T00:05:24Z"},"criteria":{"from_speaker":[],"our_interpretation":[],"support_condition":"Not checked against evidence: both screens found that public evidence could not settle this claim.","contradiction_condition":"Not checked against evidence: both screens found that public evidence could not settle this claim."},"axis":"not_applicable","outcome":"not_adjudicable","horizon_state":"not_applicable","evaluation_as_of":"2026-10-28","rationale":"Not checked against evidence. Two independent screens, one from each provider, both found this stated as a chance or a possibility, or at medium or low confidence.","limitations":[],"evidence":[],"what_would_resolve":null,"retrospective":true,"uncheckable":["probability_language"]},"payload_hash":"bd85d32899637cbc350ed090a48c7e0430195da265a45bc58235912b7b7b583d","policy_version":"2026-10-28.7","recorded_at":"2026-10-29T02:32:05Z","drafter":{"provider":"anthropic","model":"claude-opus-5-5"},"reviewers":[{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-the-new-science-of-alt-intelligence-screen-anthropic-2"},{"provider":"openai","model":"gpt-6-astra","run_id":"marcus-the-new-science-of-alt-intelligence-screen-openai-2"}],"final_text":null,"status":"current","notice":null,"superseded_by":null,"evidence_meta":[],"public_from":null,"public_until":null},"history":[],"review":{"review_due":false,"triggers":[],"last_completed_review_at":"2026-10-29T02:32:05Z","next_review_due_at":null,"horizon_now":"not_applicable"},"event_ids":["marcus-the-new-science-of-alt-intelligence-2b01c1-assessment-published-a1"],"last_substantive_change_at":"2026-10-29T17:00:18Z"}},{"url":"https://punditclaims.ai/claims/marcus-the-new-science-of-alt-intelligence-405ffd","standing":"current","assessment_label":"Can’t be checked","versions":[{"assessment_id":"marcus-the-new-science-of-alt-intelligence-405ffd-a1","status":"current","url":"https://punditclaims.ai/claims/marcus-the-new-science-of-alt-intelligence-405ffd#assessment-marcus-the-new-science-of-alt-intelligence-405ffd-a1"}],"claim":{"claim_id":"marcus-the-new-science-of-alt-intelligence-405ffd","subject_id":"gary-marcus","revision":2,"content_hash":"e6e59160ab138065a30a95134acc5d65dfe51db2183c739127cfbecaf1ad071d","normalized_proposition":"A large fraction of current AI researchers, or at least those currently in power, do not, so far as Marcus can tell, care about AI taking inspiration from natural intelligence.","kind":"observation","target":"current AI researchers","conditions":[{"text":"a large fraction of current AI researchers, or at least those currently in power","type":"scope","from_source":true},{"text":"so far as I can tell","type":"qualifier","from_source":true}],"original_horizon_text":null,"topic_tags":["other"],"concerns_providers":[],"selection_stratum":"declared-lead","occurrences":[{"id":"marcus-the-new-science-of-alt-intelligence-405ffd-o1","statement_date":"2022-05-14","statement_date_precision":"day","excerpt":"A large fraction of current AI researchers, or at least those currently in power, don’t (so far as I can tell) give a damn about any of this.","locator":"paragraph 2","document":{"url":"https://garymarcus.substack.com/p/the-new-science-of-alt-intelligence","title":"The New Science of Alt Intelligence","publisher":"Marcus on AI","published_at":"2022-05-14T17:02:35Z","date_uncertain":false,"date_provenance":"JSON-LD datePublished"},"observed_at":"2026-10-29T00:05:24Z","observed_via":"live","archive_url":null,"access_state":"full_public","snapshot_hash":"b5456ff99a853d55856ae13c7d7571ff3970a73d8fefc55e7d34fd90527b6f81","receipts":1}],"first_statement_date":"2022-05-14","related":[],"current":{"assessment_id":"marcus-the-new-science-of-alt-intelligence-405ffd-a1","payload":{"claim_id":"marcus-the-new-science-of-alt-intelligence-405ffd","subject_id":"gary-marcus","normalized_proposition":"A large fraction of current AI researchers, or at least those currently in power, do not, so far as Marcus can tell, care about AI taking inspiration from natural intelligence.","kind":"observation","conditions":["a large fraction of current AI researchers, or at least those currently in power","so far as I can tell"],"original_horizon_text":null,"statement":{"excerpt":"A large fraction of current AI researchers, or at least those currently in power, don’t (so far as I can tell) give a damn about any of this.","locator":"paragraph 2","url":"https://garymarcus.substack.com/p/the-new-science-of-alt-intelligence","date":"2022-05-14","document_title":"The New Science of Alt Intelligence","observed_at":"2026-10-29T00:05:24Z"},"criteria":{"from_speaker":["a large fraction of current AI researchers, or at least those currently in power","so far as I can tell"],"our_interpretation":[],"support_condition":"Not checked against evidence: both screens found that public evidence could not settle this claim.","contradiction_condition":"Not checked against evidence: both screens found that public evidence could not settle this claim."},"axis":"not_applicable","outcome":"not_adjudicable","horizon_state":"not_applicable","evaluation_as_of":"2026-10-28","rationale":"Not checked against evidence. Two independent screens, one from each provider, both found that evidence could not settle it: one found it too vague to test: informed readers would test it in different ways and reach different answers; the other found it about something no public evidence could show, such as a private intention or a counterfactual.","limitations":[],"evidence":[],"what_would_resolve":null,"retrospective":true,"uncheckable":["no_plain_reading","no_public_evidence"]},"payload_hash":"7d69840d595d1ee9a121322ec0184fbff0a946eed2a422b3d973379d12b41911","policy_version":"2026-10-28.7","recorded_at":"2026-10-29T03:08:15Z","drafter":{"provider":"anthropic","model":"claude-opus-5-5"},"reviewers":[{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-the-new-science-of-alt-intelligence-screen-anthropic-3"},{"provider":"openai","model":"gpt-6-astra","run_id":"marcus-the-new-science-of-alt-intelligence-screen-openai-3"}],"final_text":null,"status":"current","notice":null,"superseded_by":null,"evidence_meta":[],"public_from":null,"public_until":null},"history":[],"review":{"review_due":false,"triggers":[],"last_completed_review_at":"2026-10-29T03:08:15Z","next_review_due_at":null,"horizon_now":"not_applicable"},"event_ids":["marcus-the-new-science-of-alt-intelligence-405ffd-assessment-published-a1"],"last_substantive_change_at":"2026-10-29T17:00:18Z"}},{"url":"https://punditclaims.ai/claims/marcus-the-new-science-of-alt-intelligence-4a9f0c","standing":"current","assessment_label":"Can’t be checked","versions":[{"assessment_id":"marcus-the-new-science-of-alt-intelligence-4a9f0c-a1","status":"current","url":"https://punditclaims.ai/claims/marcus-the-new-science-of-alt-intelligence-4a9f0c#assessment-marcus-the-new-science-of-alt-intelligence-4a9f0c-a1"}],"claim":{"claim_id":"marcus-the-new-science-of-alt-intelligence-4a9f0c","subject_id":"gary-marcus","revision":1,"content_hash":"324921b7f57a24d23a49c87a58523634b005de74bc3fda058d26e96178d4c189","normalized_proposition":"AI research should draw clues from human and animal cognition without slavishly replicating human intelligence.","kind":"normative","target":"AI research","conditions":[],"original_horizon_text":null,"topic_tags":["other"],"concerns_providers":[],"selection_stratum":"declared-lead","occurrences":[{"id":"marcus-the-new-science-of-alt-intelligence-4a9f0c-o1","statement_date":"2022-05-14","statement_date_precision":"day","excerpt":"AI should certainly not be a slavish replica of human intelligence (which after all is flawed in its own ways, saddled with lousy memory and cognitive bias). But it should look to human (and animal cognition) for clues.","locator":"paragraph 71","document":{"url":"https://garymarcus.substack.com/p/the-new-science-of-alt-intelligence","title":"The New Science of Alt Intelligence","publisher":"Marcus on AI","published_at":"2022-05-14T17:02:35Z","date_uncertain":false,"date_provenance":"JSON-LD datePublished"},"observed_at":"2026-10-29T00:05:24Z","observed_via":"live","archive_url":null,"access_state":"full_public","snapshot_hash":"b5456ff99a853d55856ae13c7d7571ff3970a73d8fefc55e7d34fd90527b6f81","receipts":1}],"first_statement_date":"2022-05-14","related":[],"current":{"assessment_id":"marcus-the-new-science-of-alt-intelligence-4a9f0c-a1","payload":{"claim_id":"marcus-the-new-science-of-alt-intelligence-4a9f0c","subject_id":"gary-marcus","normalized_proposition":"AI research should draw clues from human and animal cognition without slavishly replicating human intelligence.","kind":"normative","conditions":[],"original_horizon_text":null,"statement":{"excerpt":"AI should certainly not be a slavish replica of human intelligence (which after all is flawed in its own ways, saddled with lousy memory and cognitive bias). But it should look to human (and animal cognition) for clues.","locator":"paragraph 71","url":"https://garymarcus.substack.com/p/the-new-science-of-alt-intelligence","date":"2022-05-14","document_title":"The New Science of Alt Intelligence","observed_at":"2026-10-29T00:05:24Z"},"criteria":{"from_speaker":[],"our_interpretation":[],"support_condition":"Not checked against evidence: both screens found that public evidence could not settle this claim.","contradiction_condition":"Not checked against evidence: both screens found that public evidence could not settle this claim."},"axis":"not_applicable","outcome":"not_adjudicable","horizon_state":"not_applicable","evaluation_as_of":"2026-10-28","rationale":"Not checked against evidence. Two independent screens, one from each provider, both found this a recommendation or a value judgment.","limitations":[],"evidence":[],"what_would_resolve":null,"retrospective":true,"uncheckable":["value_judgment"]},"payload_hash":"155fd845c4fad12d45d651afa5dbc87fb71ac1f614516f45fa081872d9b7c3a3","policy_version":"2026-10-28.7","recorded_at":"2026-10-29T02:32:05Z","drafter":{"provider":"anthropic","model":"claude-opus-5-5"},"reviewers":[{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-the-new-science-of-alt-intelligence-screen-anthropic-2"},{"provider":"openai","model":"gpt-6-astra","run_id":"marcus-the-new-science-of-alt-intelligence-screen-openai-2"}],"final_text":null,"status":"current","notice":null,"superseded_by":null,"evidence_meta":[],"public_from":null,"public_until":null},"history":[],"review":{"review_due":false,"triggers":[],"last_completed_review_at":"2026-10-29T02:32:05Z","next_review_due_at":null,"horizon_now":"not_applicable"},"event_ids":["marcus-the-new-science-of-alt-intelligence-4a9f0c-assessment-published-a1"],"last_substantive_change_at":"2026-10-29T17:00:18Z"}},{"url":"https://punditclaims.ai/claims/marcus-the-new-science-of-alt-intelligence-854ec6","standing":"current","assessment_label":"Can’t be checked","versions":[{"assessment_id":"marcus-the-new-science-of-alt-intelligence-854ec6-a1","status":"current","url":"https://punditclaims.ai/claims/marcus-the-new-science-of-alt-intelligence-854ec6#assessment-marcus-the-new-science-of-alt-intelligence-854ec6-a1"}],"claim":{"claim_id":"marcus-the-new-science-of-alt-intelligence-854ec6","subject_id":"gary-marcus","revision":2,"content_hash":"dff38d1dee454a75df072ddde1212ebe57e2bb6e12f545bdd1414cadc0570334","normalized_proposition":"Gato may well be better than anything else we currently have, but the fact that it still doesn't really work, even after immense investment, should give us pause.","kind":"normative","target":"Gato / Alt Intelligence approach","conditions":[{"text":"may well be better than anything else we currently have","type":"qualifier","from_source":true},{"text":"even after all the immense investments","type":"qualifier","from_source":true}],"original_horizon_text":null,"topic_tags":["scaling","business-economics"],"concerns_providers":[],"selection_stratum":"declared-lead","occurrences":[{"id":"marcus-the-new-science-of-alt-intelligence-854ec6-o1","statement_date":"2022-05-14","statement_date_precision":"day","excerpt":"It may well be better than anything else we currently have, but the fact that it still doesn’t really work, even after all the immense investments that have been made in it, should give us pause.","locator":"paragraph 70","document":{"url":"https://garymarcus.substack.com/p/the-new-science-of-alt-intelligence","title":"The New Science of Alt Intelligence","publisher":"Marcus on AI","published_at":"2022-05-14T17:02:35Z","date_uncertain":false,"date_provenance":"JSON-LD datePublished"},"observed_at":"2026-10-29T00:05:24Z","observed_via":"live","archive_url":null,"access_state":"full_public","snapshot_hash":"b5456ff99a853d55856ae13c7d7571ff3970a73d8fefc55e7d34fd90527b6f81","receipts":1}],"first_statement_date":"2022-05-14","related":[],"current":{"assessment_id":"marcus-the-new-science-of-alt-intelligence-854ec6-a1","payload":{"claim_id":"marcus-the-new-science-of-alt-intelligence-854ec6","subject_id":"gary-marcus","normalized_proposition":"Gato may well be better than anything else we currently have, but the fact that it still doesn't really work, even after immense investment, should give us pause.","kind":"normative","conditions":["may well be better than anything else we currently have","even after all the immense investments"],"original_horizon_text":null,"statement":{"excerpt":"It may well be better than anything else we currently have, but the fact that it still doesn’t really work, even after all the immense investments that have been made in it, should give us pause.","locator":"paragraph 70","url":"https://garymarcus.substack.com/p/the-new-science-of-alt-intelligence","date":"2022-05-14","document_title":"The New Science of Alt Intelligence","observed_at":"2026-10-29T00:05:24Z"},"criteria":{"from_speaker":["may well be better than anything else we currently have","even after all the immense investments"],"our_interpretation":[],"support_condition":"Not checked against evidence: both screens found that public evidence could not settle this claim.","contradiction_condition":"Not checked against evidence: both screens found that public evidence could not settle this claim."},"axis":"not_applicable","outcome":"not_adjudicable","horizon_state":"not_applicable","evaluation_as_of":"2026-10-28","rationale":"Not checked against evidence. Two independent screens, one from each provider, both found this a recommendation or a value judgment.","limitations":[],"evidence":[],"what_would_resolve":null,"retrospective":true,"uncheckable":["value_judgment"]},"payload_hash":"7320403da910a6c7adcfc351bfb7155d9ba037c22e21d514eb2752044a41bd7e","policy_version":"2026-10-28.7","recorded_at":"2026-10-29T03:08:15Z","drafter":{"provider":"anthropic","model":"claude-opus-5-5"},"reviewers":[{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-the-new-science-of-alt-intelligence-screen-anthropic-3"},{"provider":"openai","model":"gpt-6-astra","run_id":"marcus-the-new-science-of-alt-intelligence-screen-openai-3"}],"final_text":null,"status":"current","notice":null,"superseded_by":null,"evidence_meta":[],"public_from":null,"public_until":null},"history":[],"review":{"review_due":false,"triggers":[],"last_completed_review_at":"2026-10-29T03:08:15Z","next_review_due_at":null,"horizon_now":"not_applicable"},"event_ids":["marcus-the-new-science-of-alt-intelligence-854ec6-assessment-published-a1"],"last_substantive_change_at":"2026-10-29T17:00:18Z"}},{"url":"https://punditclaims.ai/claims/marcus-the-new-science-of-alt-intelligence-9e7096","standing":"current","assessment_label":"Unresolved","versions":[{"assessment_id":"marcus-the-new-science-of-alt-intelligence-9e7096-a2","status":"current","url":"https://punditclaims.ai/claims/marcus-the-new-science-of-alt-intelligence-9e7096#assessment-marcus-the-new-science-of-alt-intelligence-9e7096-a2"}],"claim":{"claim_id":"marcus-the-new-science-of-alt-intelligence-9e7096","subject_id":"gary-marcus","revision":1,"content_hash":"ee2b37c26bc69269d9691f68e54d75ed2eb4f1d071af9ecfc8b361205a3502bf","normalized_proposition":"Gato does not work like the brain, learn like a child, understand language, or align with human values, and cannot be trusted with mission-critical tasks.","kind":"observation","target":"Gato","conditions":[],"original_horizon_text":null,"topic_tags":["capabilities","reliability","safety"],"concerns_providers":[],"selection_stratum":"declared-lead","occurrences":[{"id":"marcus-the-new-science-of-alt-intelligence-9e7096-o1","statement_date":"2022-05-14","statement_date_precision":"day","excerpt":"As I have written, I am fine with thinking of Gato as an “Alt Intelligence”— an interesting exploration in alternative ways to build intelligence—but we need to take it in context: it doesn’t work like the brain, it doesn’t learn like a child, it doesn’t understand language, it doesn’t align with human values, and it can’t be trusted with mission-critical tasks.","locator":"paragraph 69","document":{"url":"https://garymarcus.substack.com/p/the-new-science-of-alt-intelligence","title":"The New Science of Alt Intelligence","publisher":"Marcus on AI","published_at":"2022-05-14T17:02:35Z","date_uncertain":false,"date_provenance":"JSON-LD datePublished"},"observed_at":"2026-10-29T00:05:24Z","observed_via":"live","archive_url":null,"access_state":"full_public","snapshot_hash":"b5456ff99a853d55856ae13c7d7571ff3970a73d8fefc55e7d34fd90527b6f81","receipts":1}],"first_statement_date":"2022-05-14","related":[],"current":{"assessment_id":"marcus-the-new-science-of-alt-intelligence-9e7096-a2","payload":{"claim_id":"marcus-the-new-science-of-alt-intelligence-9e7096","subject_id":"gary-marcus","normalized_proposition":"Gato does not work like the brain, learn like a child, understand language, or align with human values, and cannot be trusted with mission-critical tasks.","kind":"observation","conditions":[],"original_horizon_text":null,"statement":{"excerpt":"As I have written, I am fine with thinking of Gato as an “Alt Intelligence”— an interesting exploration in alternative ways to build intelligence—but we need to take it in context: it doesn’t work like the brain, it doesn’t learn like a child, it doesn’t understand language, it doesn’t align with human values, and it can’t be trusted with mission-critical tasks.","locator":"paragraph 69","url":"https://garymarcus.substack.com/p/the-new-science-of-alt-intelligence","date":"2022-05-14","document_title":"The New Science of Alt Intelligence","observed_at":"2026-10-29T00:05:24Z"},"criteria":{"from_speaker":["Gato","doesn’t work like the brain","doesn’t learn like a child","doesn’t understand language","doesn’t align with human values","can’t be trusted with mission-critical tasks"],"our_interpretation":["Assess the original Gato described immediately before the post, not subsequent models.","Use May 12–14, 2022: the initial public report through the post date is the latest reportable state.","Read the brain and child comparisons as comparisons of operating and learning methods, not a denial of every biological inspiration.","Read language understanding as dependable comprehension beyond selected demonstrations; fluent output alone does not settle it.","Read alignment and mission-critical trust as dependable behavior consistent with intended human goals and adequate reliability where failure has serious consequences.","Support for the complete claim requires evidence for all five assertions. An untested capability is not automatically an absent capability."],"support_condition":"Architecture and learning evidence establish the asserted differences, and suitable language, alignment, and reliability evaluations substantiate the remaining categorical limitations.","contradiction_condition":"Direct evidence establishes a capability or correspondence that clearly conflicts with at least one assertion under the stated reading. Selected language examples alone do not establish dependable general comprehension."},"axis":"historical_observation","outcome":"unresolved","horizon_state":"not_applicable","evaluation_as_of":"2026-10-28","rationale":"The original Gato report, available May 12, 2022, describes a transformer trained offline with supervised learning and says harm-mitigation tools require further research before deployment [e6]. It also reports rudimentary dialogue, captioning, and instruction-following abilities [e6]. These demonstrations do not establish dependable language comprehension, and the deployment warnings do not establish categorical misalignment or failure against a specified mission-critical reliability requirement [e6]. The report therefore neither substantiates all five assertions nor supplies a clear contradiction under the stated reading [e6].","limitations":["The evidence is the developers' original report, rather than independent validation of all five assertions [e6].","The architecture and training descriptions do not settle every interpretation of the brain and child comparisons [e6].","The report supplies neither comprehensive comprehension and alignment tests nor an application-specific mission-critical reliability assessment; missing tests do not establish absent capabilities [e6]."],"evidence":[{"id":"e6","relation":"context","excerpt":"require further research","locator":"Section 7, Broader Impact; architecture in Sections 1–2 and language demonstrations in Section 4.3","url":"https://arxiv.org/html/2205.06175v1","publisher":"arXiv; DeepMind","event_date":"2022-05","available_at":"2022-05-12","explanation":"The report describes a transformer trained offline with supervised learning and cautions that deployment safeguards need further work. It also reports rudimentary dialogue, captioning, and instruction-following abilities, which qualify a literal denial of all language capability without establishing dependable comprehension or value alignment."}],"what_would_resolve":"Defined comparison criteria and evaluations of the original Gato as released in May 2022, covering operating and learning methods, dependable language comprehension, human-value alignment, and reliability in a specified mission-critical application. These could substantiate all five assertions or clearly contradict at least one.","retrospective":true,"plain_reading":true},"payload_hash":"9fe8591513374c52a9e7d1df2c4e7933bd4c54d9f5ba6394f1cb92ff59985bcc","policy_version":"2026-10-28.7","recorded_at":"2026-10-29T12:42:08Z","drafter":{"provider":"openai","model":"gpt-6-astra"},"reviewers":[{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-the-new-science-of-alt-intelligence-11b850-2x-review-anthropic-1"},{"provider":"openai","model":"gpt-6-astra","run_id":"marcus-the-new-science-of-alt-intelligence-11b850-2x-review-openai-1"}],"final_text":{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-the-new-science-of-alt-intelligence-11b850-2x-final-anthropic-2"},"status":"current","notice":null,"superseded_by":null,"evidence_meta":[{"id":"e6","source_type":"preprint","audited":null,"derived":null,"metric":null}],"public_from":null,"public_until":null},"history":[],"review":{"review_due":false,"triggers":[],"last_completed_review_at":"2026-10-29T12:42:08Z","next_review_due_at":null,"horizon_now":"not_applicable"},"event_ids":["marcus-the-new-science-of-alt-intelligence-9e7096-assessment-published-a2"],"last_substantive_change_at":"2026-10-29T17:00:18Z"}},{"url":"https://punditclaims.ai/claims/marcus-the-new-science-of-alt-intelligence-a01724","standing":"current","assessment_label":"Can’t be checked","versions":[{"assessment_id":"marcus-the-new-science-of-alt-intelligence-a01724-a1","status":"current","url":"https://punditclaims.ai/claims/marcus-the-new-science-of-alt-intelligence-a01724#assessment-marcus-the-new-science-of-alt-intelligence-a01724-a1"}],"claim":{"claim_id":"marcus-the-new-science-of-alt-intelligence-a01724","subject_id":"gary-marcus","revision":2,"content_hash":"d79b6f951d5fce891e26ffbad8479fbe774a2ca6af35f83d24ce77e15cad0bda","normalized_proposition":"It is sheer hubris to believe that AGI can be built without learning from how humans reason, understand the world, and represent and acquire language and concepts.","kind":"normative","target":"scaling-only view of AGI","conditions":[],"original_horizon_text":null,"topic_tags":["agi-timelines","scaling"],"concerns_providers":[],"selection_stratum":"declared-lead","occurrences":[{"id":"marcus-the-new-science-of-alt-intelligence-a01724-o1","statement_date":"2022-05-14","statement_date_precision":"day","excerpt":"It is sheer hubris to believe otherwise.","locator":"paragraph 73","document":{"url":"https://garymarcus.substack.com/p/the-new-science-of-alt-intelligence","title":"The New Science of Alt Intelligence","publisher":"Marcus on AI","published_at":"2022-05-14T17:02:35Z","date_uncertain":false,"date_provenance":"JSON-LD datePublished"},"observed_at":"2026-10-29T00:05:24Z","observed_via":"live","archive_url":null,"access_state":"full_public","snapshot_hash":"b5456ff99a853d55856ae13c7d7571ff3970a73d8fefc55e7d34fd90527b6f81","receipts":1}],"first_statement_date":"2022-05-14","related":[],"current":{"assessment_id":"marcus-the-new-science-of-alt-intelligence-a01724-a1","payload":{"claim_id":"marcus-the-new-science-of-alt-intelligence-a01724","subject_id":"gary-marcus","normalized_proposition":"It is sheer hubris to believe that AGI can be built without learning from how humans reason, understand the world, and represent and acquire language and concepts.","kind":"normative","conditions":[],"original_horizon_text":null,"statement":{"excerpt":"It is sheer hubris to believe otherwise.","locator":"paragraph 73","url":"https://garymarcus.substack.com/p/the-new-science-of-alt-intelligence","date":"2022-05-14","document_title":"The New Science of Alt Intelligence","observed_at":"2026-10-29T00:05:24Z"},"criteria":{"from_speaker":[],"our_interpretation":[],"support_condition":"Not checked against evidence: both screens found that public evidence could not settle this claim.","contradiction_condition":"Not checked against evidence: both screens found that public evidence could not settle this claim."},"axis":"not_applicable","outcome":"not_adjudicable","horizon_state":"not_applicable","evaluation_as_of":"2026-10-28","rationale":"Not checked against evidence. Two independent screens, one from each provider, both found this a recommendation or a value judgment.","limitations":[],"evidence":[],"what_would_resolve":null,"retrospective":true,"uncheckable":["value_judgment"]},"payload_hash":"f4ced30be66a6a1d5dc14fb24e42bce0d9af949564f40f280dc25f9fbedef853","policy_version":"2026-10-28.7","recorded_at":"2026-10-29T03:08:15Z","drafter":{"provider":"anthropic","model":"claude-opus-5-5"},"reviewers":[{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-the-new-science-of-alt-intelligence-screen-anthropic-3"},{"provider":"openai","model":"gpt-6-astra","run_id":"marcus-the-new-science-of-alt-intelligence-screen-openai-3"}],"final_text":null,"status":"current","notice":null,"superseded_by":null,"evidence_meta":[],"public_from":null,"public_until":null},"history":[],"review":{"review_due":false,"triggers":[],"last_completed_review_at":"2026-10-29T03:08:15Z","next_review_due_at":null,"horizon_now":"not_applicable"},"event_ids":["marcus-the-new-science-of-alt-intelligence-a01724-assessment-published-a1"],"last_substantive_change_at":"2026-10-29T17:00:18Z"}},{"url":"https://punditclaims.ai/claims/marcus-the-new-science-of-alt-intelligence-f493ea","standing":"current","assessment_label":"Can’t be checked","versions":[{"assessment_id":"marcus-the-new-science-of-alt-intelligence-f493ea-a1","status":"current","url":"https://punditclaims.ai/claims/marcus-the-new-science-of-alt-intelligence-f493ea#assessment-marcus-the-new-science-of-alt-intelligence-f493ea-a1"}],"claim":{"claim_id":"marcus-the-new-science-of-alt-intelligence-f493ea","subject_id":"gary-marcus","revision":1,"content_hash":"00bc6f4726dcfca7d29f059ec0fdc311f92e79eb8c4ad3bccd6bc9dffa394ff1","normalized_proposition":"The persistent problem of discomprehension may signal that AI needs a fundamental rethink.","kind":"normative","target":"AI field","conditions":[],"original_horizon_text":null,"topic_tags":["scaling","capabilities"],"concerns_providers":[],"selection_stratum":"declared-lead","occurrences":[{"id":"marcus-the-new-science-of-alt-intelligence-f493ea-o1","statement_date":"2022-05-14","statement_date_precision":"day","excerpt":"To some (such as myself), the abiding problem of discomprehension might—despite the immense progress—signal the need for a fundamental rethink, such as the one that Davis and I offered in our book Rebooting AI.","locator":"paragraph 38","document":{"url":"https://garymarcus.substack.com/p/the-new-science-of-alt-intelligence","title":"The New Science of Alt Intelligence","publisher":"Marcus on AI","published_at":"2022-05-14T17:02:35Z","date_uncertain":false,"date_provenance":"JSON-LD datePublished"},"observed_at":"2026-10-29T00:05:24Z","observed_via":"live","archive_url":null,"access_state":"full_public","snapshot_hash":"b5456ff99a853d55856ae13c7d7571ff3970a73d8fefc55e7d34fd90527b6f81","receipts":1}],"first_statement_date":"2022-05-14","related":[],"current":{"assessment_id":"marcus-the-new-science-of-alt-intelligence-f493ea-a1","payload":{"claim_id":"marcus-the-new-science-of-alt-intelligence-f493ea","subject_id":"gary-marcus","normalized_proposition":"The persistent problem of discomprehension may signal that AI needs a fundamental rethink.","kind":"normative","conditions":[],"original_horizon_text":null,"statement":{"excerpt":"To some (such as myself), the abiding problem of discomprehension might—despite the immense progress—signal the need for a fundamental rethink, such as the one that Davis and I offered in our book Rebooting AI.","locator":"paragraph 38","url":"https://garymarcus.substack.com/p/the-new-science-of-alt-intelligence","date":"2022-05-14","document_title":"The New Science of Alt Intelligence","observed_at":"2026-10-29T00:05:24Z"},"criteria":{"from_speaker":[],"our_interpretation":[],"support_condition":"Not checked against evidence: both screens found that public evidence could not settle this claim.","contradiction_condition":"Not checked against evidence: both screens found that public evidence could not settle this claim."},"axis":"not_applicable","outcome":"not_adjudicable","horizon_state":"not_applicable","evaluation_as_of":"2026-10-28","rationale":"Not checked against evidence. Two independent screens, one from each provider, both found this stated as a chance or a possibility, or at medium or low confidence.","limitations":[],"evidence":[],"what_would_resolve":null,"retrospective":true,"uncheckable":["probability_language"]},"payload_hash":"1c4e3fe4bd9fd836e33ca98932cbf2a9f3312c104a42fc92afc8e63db0d4e3f3","policy_version":"2026-10-28.7","recorded_at":"2026-10-29T02:32:05Z","drafter":{"provider":"anthropic","model":"claude-opus-5-5"},"reviewers":[{"provider":"anthropic","model":"claude-opus-5-5","run_id":"marcus-the-new-science-of-alt-intelligence-screen-anthropic-2"},{"provider":"openai","model":"gpt-6-astra","run_id":"marcus-the-new-science-of-alt-intelligence-screen-openai-2"}],"final_text":null,"status":"current","notice":null,"superseded_by":null,"evidence_meta":[],"public_from":null,"public_until":null},"history":[],"review":{"review_due":false,"triggers":[],"last_completed_review_at":"2026-10-29T02:32:05Z","next_review_due_at":null,"horizon_now":"not_applicable"},"event_ids":["marcus-the-new-science-of-alt-intelligence-f493ea-assessment-published-a1"],"last_substantive_change_at":"2026-10-29T17:00:18Z"}}]}