2024

• Certified Deductive Reasoning with Language Models. [PDF] [BibTeX]
  G. Poesia, K. Gandhi, E. Zelikman, & N. D. Goodman. (2024). Transactions on Machine Learning Research.

  BibTeX: @article{poesia2024certified, title = {Certified Deductive Reasoning with Language Models}, author = {Poesia, Gabriel and Gandhi, Kanishk and Zelikman, Eric and Goodman, Noah D.}, year = {2024}, journal = {Transactions on Machine Learning Research}, website = {https://arxiv.org/abs/2306.04031} }

• Hypothesis Search: Inductive Reasoning with Language Models. [PDF] [BibTeX]
  R. Wang, E. Zelikman, G. Poesia, Y. Pu, N. Haber, & N. D. Goodman. (2024). The Twelfth International Conference on Learning Representations (ICLR).

  BibTeX: @article{wang2024hypothesis, title = {Hypothesis Search: Inductive Reasoning with Language Models}, author = {Wang, Ruocheng and Zelikman, Eric and Poesia, Gabriel and Pu, Yewen and Haber, Nick and Goodman, Noah D}, journal = {The Twelfth International Conference on Learning Representations (ICLR)}, website = {https://arxiv.org/abs/2309.05660}, year = {2024} }

• Bayesian reinforcement learning with limited cognitive load. [PDF] [BibTeX]
  D. Arumugam, M. K. Ho, N. D. Goodman, & B. Van Roy. (2024). Open Mind.

  BibTeX: @article{arumugam2024bayesian, title = {Bayesian reinforcement learning with limited cognitive load}, author = {Arumugam, Dilip and Ho, Mark K and Goodman, Noah D and Van Roy, Benjamin}, journal = {Open Mind}, volume = {8}, pages = {395--438}, year = {2024}, website = {https://arxiv.org/abs/2305.03263}, publisher = {MIT Press One Broadway, 12th Floor, Cambridge, Massachusetts 02142, USA~…} }

• Finding Alignments Between Interpretable Causal Variables and Distributed Neural Representations. [PDF] [BibTeX]
  A. Geiger, Z. Wu, C. Potts, T. Icard, & N. D. Goodman. (2024). In Causal Learning and Reasoning (CLEAR).

  BibTeX: @inproceedings{geiger2024finding, title = {Finding Alignments Between Interpretable Causal Variables and Distributed Neural Representations}, author = {Geiger, Atticus and Wu, Zhengxuan and Potts, Christopher and Icard, Thomas and Goodman, Noah D}, year = {2024}, website = {https://arxiv.org/abs/2303.02536}, booktitle = {Causal Learning and Reasoning (CLEAR)} }

• Backtracing: Retrieving the Cause of the Query. [PDF] [BibTeX]
  R. E. Wang, P. Wirawarn, O. Khattab, N. Goodman, & D. Demszky. (2024). In EACL 2024, Long Paper Findings.

  BibTeX: @inproceedings{wang2024backtracing, title = {Backtracing: Retrieving the Cause of the Query}, author = {Wang, Rose E and Wirawarn, Pawan and Khattab, Omar and Goodman, Noah and Demszky, Dorottya}, website = {https://arxiv.org/abs/2403.03956}, booktitle = {EACL 2024, Long Paper Findings}, year = {2024} }

• pyvene: A library for understanding and improving PyTorch models via interventions. [PDF] [BibTeX]
  Z. Wu, A. Geiger, A. Arora, J. Huang, Z. Wang, N. D. Goodman, C. D. Manning, & C. Potts. (2024). In NAACL Demo.

  BibTeX: @inproceedings{wu2024pyvene, title = {pyvene: A library for understanding and improving PyTorch models via interventions}, author = {Wu, Zhengxuan and Geiger, Atticus and Arora, Aryaman and Huang, Jing and Wang, Zheng and Goodman, Noah D and Manning, Christopher D and Potts, Christopher}, website = {https://arxiv.org/abs/2403.07809}, booktitle = {NAACL Demo}, year = {2024} }

• Procedural Dilemma Generation for Evaluating Moral Reasoning in Humans and Language Models. [PDF] [BibTeX]
  J.-P. Fränken, K. Gandhi, T. Qiu, A. Khawaja, N. D. Goodman, & T. Gerstenberg. (2024). In Proceedings of the 46th Annual Conference of the Cognitive Science Society (Oral).

  BibTeX: @inproceedings{franken2024procedural, title = {Procedural Dilemma Generation for Evaluating Moral Reasoning in Humans and Language Models}, author = {Fr{\"a}nken, Jan-Philipp and Gandhi, Kanishk and Qiu, Tori and Khawaja, Ayesha and Goodman, Noah D and Gerstenberg, Tobias}, website = {https://arxiv.org/pdf/2404.10975}, booktitle = {Proceedings of the 46th Annual Conference of the Cognitive Science Society (Oral)}, year = {2024} }

• Automated Statistical Model Discovery with Language Models. [PDF] [BibTeX]
  M. Y. Li, E. B. Fox, & N. D. Goodman. (2024). In International Conference on Machine Learning (ICML).

  BibTeX: @inproceedings{li2024automated, title = {Automated Statistical Model Discovery with Language Models}, author = {Li, Michael Y and Fox, Emily B and Goodman, Noah D}, booktitle = {International Conference on Machine Learning (ICML)}, website = {https://arxiv.org/abs/2402.17879}, year = {2024} }

• Codebook features: Sparse and discrete interpretability for neural networks. [PDF] [BibTeX]
  A. Tamkin, M. Taufeeque, & N. D. Goodman. (2024). In International Conference on Machine Learning (ICML).

  BibTeX: @inproceedings{tamkin2024codebook, title = {Codebook features: Sparse and discrete interpretability for neural networks}, author = {Tamkin, Alex and Taufeeque, Mohammad and Goodman, Noah D}, booktitle = {International Conference on Machine Learning (ICML)}, website = {https://arxiv.org/abs/2310.17230}, year = {2024} }

• Evaluating and Optimizing Educational Content with Large Language Model Judgments. [PDF] [BibTeX]
  J. He-Yueya, N. D. Goodman, & E. Brunskill. (2024). In International Educational Data Mining Society.

  BibTeX: @inproceedings{he2024evaluating, title = {Evaluating and Optimizing Educational Content with Large Language Model Judgments}, author = {He-Yueya, Joy and Goodman, Noah D and Brunskill, Emma}, booktitle = {International Educational Data Mining Society}, website = {https://arxiv.org/pdf/2403.02795}, year = {2024} }

Preprints

• Quiet-star: Language models can teach themselves to think before speaking. [PDF] [BibTeX]
  E. Zelikman, G. Harik, Y. Shao, V. Jayasiri, N. Haber, & N. D. Goodman. 2024-Preprint ArXiv Preprint ArXiv:2403.09629.

  BibTeX: @article{zelikman2024quiet, title = {Quiet-star: Language models can teach themselves to think before speaking}, author = {Zelikman, Eric and Harik, Georges and Shao, Yijia and Jayasiri, Varuna and Haber, Nick and Goodman, Noah D}, journal = {arXiv preprint arXiv:2403.09629}, website = {https://arxiv.org/abs/2403.09629}, year = {2024-Preprint} }

• Stream of Search (SoS): Learning to Search in Language. [PDF] [BibTeX]
  K. Gandhi, D. Lee, G. Grand, M. Liu, W. Cheng, A. Sharma, & N. D. Goodman. 2024-Preprint ArXiv Preprint ArXiv:2404.03683.

  BibTeX: @article{gandhi2024stream, title = {Stream of Search (SoS): Learning to Search in Language}, author = {Gandhi, Kanishk and Lee, Denise and Grand, Gabriel and Liu, Muxin and Cheng, Winson and Sharma, Archit and Goodman, Noah D}, journal = {arXiv preprint arXiv:2404.03683}, website = {https://arxiv.org/abs/2404.03683}, year = {2024-Preprint} }

• Self-Supervised Alignment with Mutual Information: Learning to Follow Principles without Preference Labels. [PDF] [BibTeX]
  J.-P. Fränken, E. Zelikman, R. Rafailov, K. Gandhi, T. Gerstenberg, & N. D. Goodman. 2024-Preprint ArXiv Preprint ArXiv:2404.14313.

  BibTeX: @article{franken2024self, title = {Self-Supervised Alignment with Mutual Information: Learning to Follow Principles without Preference Labels}, author = {Fr{\"a}nken, Jan-Philipp and Zelikman, Eric and Rafailov, Rafael and Gandhi, Kanishk and Gerstenberg, Tobias and Goodman, Noah D}, journal = {arXiv preprint arXiv:2404.14313}, website = {https://arxiv.org/abs/2404.14313}, year = {2024-Preprint} }

• Star-gate: Teaching language models to ask clarifying questions. [PDF] [BibTeX]
  C. Andukuri, J.-P. Fränken, T. Gerstenberg, & N. D. Goodman. 2024-Preprint ArXiv Preprint ArXiv:2403.19154.

  BibTeX: @article{andukuri2024star, title = {Star-gate: Teaching language models to ask clarifying questions}, author = {Andukuri, Chinmaya and Fr{\"a}nken, Jan-Philipp and Gerstenberg, Tobias and Goodman, Noah D}, journal = {arXiv preprint arXiv:2403.19154}, year = {2024-Preprint}, website = {https://arxiv.org/abs/2403.19154} }

2023

• Task Ambiguity in Humans and Language Models. [PDF] [BibTeX]
  A. Tamkin, K. Handa, A. Shrestha, & N. Goodman. (2023). In The Eleventh International Conference on Learning Representations (ICLR).

  BibTeX: @inproceedings{tamkin2023task, title = {Task Ambiguity in Humans and Language Models}, author = {Tamkin, Alex and Handa, Kunal and Shrestha, Avash and Goodman, Noah}, booktitle = {The Eleventh International Conference on Learning Representations (ICLR)}, year = {2023}, website = {https://arxiv.org/abs/2212.10711} }

• Peano: Learning Formal Mathematical Reasoning. [PDF] [BibTeX]
  G. Poesia, & N. D. Goodman. (2023). Phil. Trans. of the Royal Society A.

  BibTeX: @article{poesia2022peano, title = {Peano: Learning Formal Mathematical Reasoning}, author = {Poesia, Gabriel and Goodman, Noah D}, journal = {Phil. Trans. of the Royal Society A}, year = {2023}, website = {https://arxiv.org/abs/2211.15864} }

• Flexible social inference facilitates targeted social learning when rewards are not observable. [PDF] [BibTeX]
  R. D. Hawkins, A. Berdahl, A. S. Pentland, N. D. Goodman, J. B. Tenenbaum, & P. M. Krafft. (2023). Nature Human Behavior.

  BibTeX: @article{hawkins2023flexible, title = {Flexible social inference facilitates targeted social learning when rewards are not observable}, author = {Hawkins, RD and Berdahl, A and Pentland, AS and Goodman, ND and Tenenbaum, JB and Krafft, PM}, journal = {Nature Human Behavior}, year = {2023}, publisher = {Nature Publishing Group}, website = {https://arxiv.org/abs/2212.00869} }

• Visual resemblance and interaction history jointly constrain pictorial meaning. [PDF] [BibTeX]
  R. D. Hawkins, M. Sano, N. D. Goodman, & J. E. Fan. (2023). Nature Communications.

  BibTeX: @article{hawkins2023visual, title = {Visual resemblance and interaction history jointly constrain pictorial meaning}, author = {Hawkins, Robert D and Sano, Megumi and Goodman, Noah D and Fan, Judith E}, journal = {Nature Communications}, volume = {14}, number = {1}, pages = {2199}, year = {2023}, publisher = {Nature Publishing Group UK London}, website = {https://www.nature.com/articles/s41467-023-37737-w} }

• Psychologically-informed chain-of-thought prompts for metaphor understanding in large language models. [PDF] [BibTeX]
  B. Prystawski, P. Thibodeau, C. Potts, & N. D. Goodman. (2023). In Proceedings of the 45th Annual Conference of the Cognitive Science Society (CogSci 2023).

  BibTeX: @inproceedings{prystawski2023psychologically, title = {Psychologically-informed chain-of-thought prompts for metaphor understanding in large language models}, author = {Prystawski, Ben and Thibodeau, Paul and Potts, Christopher and Goodman, Noah D}, booktitle = {Proceedings of the 45th Annual Conference of the Cognitive Science Society (CogSci 2023)}, website = {https://arxiv.org/abs/2209.08141}, year = {2023} }

• Cultural reinforcement learning: a framework for modeling cumulative culture on a limited channel. [PDF] [BibTeX]
  B. Prystawski, D. Arumugam, & N. D. Goodman. (2023). In Proceedings of the 45th Annual Conference of the Cognitive Science Society (CogSci 2023).

  BibTeX: @inproceedings{prystawski2023cultural, title = {Cultural reinforcement learning: a framework for modeling cumulative culture on a limited channel}, author = {Prystawski, Ben and Arumugam, Dilip and Goodman, Noah D}, booktitle = {Proceedings of the 45th Annual Conference of the Cognitive Science Society (CogSci 2023)}, website = {https://psyarxiv.com/q4tz8/}, year = {2023} }

• Generating Language Corrections for Teaching Physical Control Tasks. [BibTeX]
  M. Srivastava, N. D. Goodman, & D. Sadigh. (2023). In 40th International Conference on Machine Learning (ICML).

  BibTeX: @inproceedings{srivastava2023generating, title = {Generating Language Corrections for Teaching Physical Control Tasks}, author = {Srivastava, Megha and Goodman, Noah D and Sadigh, Dorsa}, booktitle = {40th International Conference on Machine Learning (ICML)}, year = {2023} }

• Characterizing tradeoffs between teaching via language and demonstrations in multi-agent systems. [PDF] [BibTeX]
  D. Yu, N. D. Goodman, & J. Mu. (2023). In Proceedings of the 45th Annual Conference of the Cognitive Science Society (CogSci 2023).

  BibTeX: @inproceedings{yu2023characterizing, title = {Characterizing tradeoffs between teaching via language and demonstrations in multi-agent systems}, author = {Yu, Dhara and Goodman, Noah D and Mu, Jesse}, website = {https://arxiv.org/abs/2305.11374}, booktitle = {Proceedings of the 45th Annual Conference of the Cognitive Science Society (CogSci 2023)}, year = {2023} }

• Why think step-by-step? Reasoning emerges from the locality of experience. [PDF] [BibTeX]
  B. Prystawski, M. Y. Li, & N. D. Goodman. (2023). In Advances in Neural Information Processing Systems (Oral).

  BibTeX: @inproceedings{prystawski2023think, title = {Why think step-by-step? Reasoning emerges from the locality of experience}, author = {Prystawski, Ben and Li, Michael Y. and Goodman, Noah D}, booktitle = {Advances in Neural Information Processing Systems (Oral)}, year = {2023}, website = {https://arxiv.org/abs/2304.03843} }

• Strategic Reasoning with Language Models. [PDF] [BibTeX]
  K. Gandhi, D. Sadigh, & N. D. Goodman. (2023). NeurIPS Workshop on Foundation Models and Decision Making.

  BibTeX: @article{gandhi2023strategic, title = {Strategic Reasoning with Language Models}, author = {Gandhi, Kanishk and Sadigh, Dorsa and Goodman, Noah D.}, year = {2023}, website = {https://arxiv.org/abs/2305.19165}, journal = {NeurIPS Workshop on Foundation Models and Decision Making} }

• Understanding social reasoning in language models with language models. [PDF] [BibTeX]
  K. Gandhi, J.-P. Fränken, T. Gerstenberg, & N. D. Goodman. (2023). In Advances in Neural Information Processing Systems (Spotlight).

  BibTeX: @inproceedings{gandhi2023understanding, title = {Understanding social reasoning in language models with language models}, author = {Gandhi, Kanishk and Fr{\"a}nken, Jan-Philipp and Gerstenberg, Tobias and Goodman, Noah D}, year = {2023}, booktitle = {Advances in Neural Information Processing Systems (Spotlight)}, website = {https://arxiv.org/abs/2306.15448} }

• Overinformative Question Answering by Humans and Machines. [PDF] [BibTeX]
  P. Tsvilodub, M. Franke, R. D. Hawkins, & N. D. Goodman. (2023). In Proceedings of the 45th Annual Conference of the Cognitive Science Society (2023).

  BibTeX: @inproceedings{tsvilodub2023overinformative, title = {Overinformative Question Answering by Humans and Machines}, author = {Tsvilodub, Polina and Franke, Michael and Hawkins, Robert D and Goodman, Noah D}, year = {2023}, website = {https://arxiv.org/abs/2305.07151}, booktitle = {Proceedings of the 45th Annual Conference of the Cognitive Science Society (2023)} }

• Learning to compress prompts with gist tokens. [PDF] [BibTeX]
  J. Mu, X. L. Li, & N. Goodman. (2023). In Advances in Neural Information Processing Systems.

  BibTeX: @inproceedings{mu2023learning, title = {Learning to compress prompts with gist tokens}, author = {Mu, Jesse and Li, Xiang Lisa and Goodman, Noah}, booktitle = {Advances in Neural Information Processing Systems}, year = {2023}, website = {https://arxiv.org/abs/2304.08467} }

• Solving math word problems by combining language models with symbolic solvers. [PDF] [BibTeX]
  J. He-Yueya, G. Poesia, R. E. Wang, & N. D. Goodman. (2023). In NeurIPS Workshop on Math-AI.

  BibTeX: @inproceedings{he2023solving, title = {Solving math word problems by combining language models with symbolic solvers}, author = {He-Yueya, Joy and Poesia, Gabriel and Wang, Rose E and Goodman, Noah D}, year = {2023}, website = {https://arxiv.org/abs/2304.09102}, booktitle = {NeurIPS Workshop on Math-AI} }

• Multispectral Self-Supervised Learning with Viewmaker Networks. [PDF] [BibTeX]
  J. Bayrooti, N. Goodman, & A. Tamkin. (2023). In CVPR-PBVS 2023.

  BibTeX: @inproceedings{bayrooti2023multispectral, title = {Multispectral Self-Supervised Learning with Viewmaker Networks}, author = {Bayrooti, Jasmine and Goodman, Noah and Tamkin, Alex}, year = {2023}, website = {https://arxiv.org/abs/2302.05757}, booktitle = {CVPR-PBVS 2023} }

• Parsel: A (de-) compositional framework for algorithmic reasoning with language models. [PDF] [BibTeX]
  E. Zelikman, Q. Huang, G. Poesia, N. D. Goodman, & N. Haber. (2023). In Advances in Neural Information Processing Systems (Spotlight).

  BibTeX: @inproceedings{zelikman2023parsel, title = {Parsel: A (de-) compositional framework for algorithmic reasoning with language models}, author = {Zelikman, Eric and Huang, Qian and Poesia, Gabriel and Goodman, Noah D and Haber, Nick}, booktitle = {Advances in Neural Information Processing Systems (Spotlight)}, year = {2023}, website = {https://arxiv.org/abs/2212.10561} }

• Feature Dropout: Revisiting the Role of Augmentations in Contrastive Learning. [PDF] [BibTeX]
  A. Tamkin, M. Glasgow, X. He, & N. Goodman. (2023). In Advances in Neural Information Processing Systems.

  BibTeX: @inproceedings{tamkin2022feature, title = {Feature Dropout: Revisiting the Role of Augmentations in Contrastive Learning}, author = {Tamkin, Alex and Glasgow, Margalit and He, Xiluo and Goodman, Noah}, booktitle = {Advances in Neural Information Processing Systems}, year = {2023}, website = {https://arxiv.org/abs/2212.08378} }

• Interpretability at Scale: Identifying Causal Mechanisms in Alpaca. [PDF] [BibTeX]
  Z. Wu, A. Geiger, C. Potts, & N. D. Goodman. (2023). In Advances in Neural Information Processing Systems.

  BibTeX: @inproceedings{wu2023interpretability, title = {Interpretability at Scale: Identifying Causal Mechanisms in Alpaca}, author = {Wu, Zhengxuan and Geiger, Atticus and Potts, Christopher and Goodman, Noah D}, booktitle = {Advances in Neural Information Processing Systems}, year = {2023}, website = {https://arxiv.org/abs/2305.08809} }

• Sight: A large annotated dataset on student insights gathered from higher education transcripts. [PDF] [BibTeX]
  R. E. Wang, P. Wirawarn, N. Goodman, & D. Demszky. (2023). In BEA 2023.

  BibTeX: @inproceedings{wang2023sight, title = {Sight: A large annotated dataset on student insights gathered from higher education transcripts}, author = {Wang, Rose E and Wirawarn, Pawan and Goodman, Noah and Demszky, Dorottya}, booktitle = {BEA 2023}, website = {https://arxiv.org/abs/2306.09343}, year = {2023} }

• Social Contract AI: Aligning AI Assistants with Implicit Group Norms. [PDF] [BibTeX]
  J.-P. Fränken, S. Kwok, P. Ye, K. Gandhi, D. Arumugam, J. Moore, A. Tamkin, T. Gerstenberg, & N. D. Goodman. (2023). In NeurIPS Workshop in Socially Respobsible Language Modeling Research, Oral.

  BibTeX: @inproceedings{franken2023social, title = {Social Contract AI: Aligning AI Assistants with Implicit Group Norms}, author = {Fr{\"a}nken, Jan-Philipp and Kwok, Sam and Ye, Peixuan and Gandhi, Kanishk and Arumugam, Dilip and Moore, Jared and Tamkin, Alex and Gerstenberg, Tobias and Goodman, Noah D}, website = {https://arxiv.org/abs/2310.17769}, booktitle = {NeurIPS Workshop in Socially Respobsible Language Modeling Research, Oral}, year = {2023} }

• Can Visual Scratchpads With Diagrammatic Abstractions Augment LLM Reasoning?. [BibTeX]
  J. Hsu, G. Poesia, J. Wu, & N. Goodman. (2023). In NeurIPS Workshop on MathAI, and NeurIPS Workshop on I Can’t Believe It’s Not Better (ICBINB).

  BibTeX: @inproceedings{hsu2023can, title = {Can Visual Scratchpads With Diagrammatic Abstractions Augment LLM Reasoning?}, author = {Hsu, Joy and Poesia, Gabriel and Wu, Jiajun and Goodman, Noah}, booktitle = {NeurIPS Workshop on MathAI, and NeurIPS Workshop on I Can't Believe It's Not Better (ICBINB)}, pages = {21--28}, year = {2023} }

Preprints

2022

• Mixed-effects transformers for hierarchical adaptation. [PDF] [BibTeX]
  J. White, N. Goodman, & R. Hawkins. (2022). In Proceedings of the 2022 Conference on Empirical Methods in Natural Language Processing (EMNLP). Association for Computational Linguistics.

  BibTeX: @inproceedings{white2022mixed, title = {Mixed-effects transformers for hierarchical adaptation}, author = {White, Julia and Goodman, Noah and Hawkins, Robert}, year = {2022}, booktitle = {Proceedings of the 2022 Conference on Empirical Methods in Natural Language Processing (EMNLP). Association for Computational Linguistics}, website = {https://arxiv.org/pdf/2205.01749.pdf} }

• Concadia: Towards image-based text generation with a purpose. [PDF] [BibTeX]
  E. Kreiss, F. Fang, N. D. Goodman, & C. Potts. (2022). In Proceedings of the 2022 Conference on Empirical Methods in Natural Language Processing (EMNLP). Association for Computational Linguistics.

  BibTeX: @inproceedings{kreiss2022concadia, title = {Concadia: Towards image-based text generation with a purpose}, author = {Kreiss, Elisa and Fang, Fei and Goodman, Noah D and Potts, Christopher}, booktitle = {Proceedings of the 2022 Conference on Empirical Methods in Natural Language Processing (EMNLP). Association for Computational Linguistics}, year = {2022}, website = {https://arxiv.org/pdf/2104.08376.pdf} }

• Inducing causal structure for interpretable neural networks. [PDF] [BibTeX]
  A. Geiger, Z. Wu, H. Lu, J. Rozner, E. Kreiss, T. Icard, N. Goodman, & C. Potts. (2022). In International Conference on Machine Learning.

  BibTeX: @inproceedings{geiger2022inducing, title = {Inducing causal structure for interpretable neural networks}, author = {Geiger, Atticus and Wu, Zhengxuan and Lu, Hanson and Rozner, Josh and Kreiss, Elisa and Icard, Thomas and Goodman, Noah and Potts, Christopher}, booktitle = {International Conference on Machine Learning}, pages = {7324--7338}, year = {2022}, organization = {PMLR}, website = {https://arxiv.org/pdf/2112.00826.pdf} }

• Two’s company but six is a crowd: emergence of conventions in multiparty communication games. [PDF] [BibTeX]
  V. Boyce, R. Hawkins, N. Goodman, & M. C. Frank. (2022). In Proceedings of the Annual Meeting of the Cognitive Science Society.

  BibTeX: @inproceedings{boyce2022two, title = {Two's company but six is a crowd: emergence of conventions in multiparty communication games}, author = {Boyce, Veronica and Hawkins, Robert and Goodman, Noah and Frank, Michael C}, booktitle = {Proceedings of the Annual Meeting of the Cognitive Science Society}, volume = {44}, number = {44}, year = {2022}, website = {https://psyarxiv.com/xd7ep/} }

• Automated generation of sentence reading fluency test items. [PDF] [BibTeX]
  J. White, A. Burkhardt, J. Yeatman, & N. Goodman. (2022). In Proceedings of the Annual Meeting of the Cognitive Science Society.

  BibTeX: @inproceedings{white2022automated, title = {Automated generation of sentence reading fluency test items}, author = {White, Julia and Burkhardt, Amy and Yeatman, Jason and Goodman, Noah}, booktitle = {Proceedings of the Annual Meeting of the Cognitive Science Society}, volume = {44}, number = {44}, year = {2022}, website = {https://escholarship.org/uc/item/3804p0ff} }

• Left to the Reader: Abstracting Solutions in Mathematical Reasoning. [Abstract] [PDF] [BibTeX]
  G. Poesia, & N. Goodman. (2022). In Proceedings of the Annual Meeting of the Cognitive Science Society.

  Abstract: Formal mathematical reasoning is unique in its precision: any valid conclusion can be justified by a sequence of base axioms. But human-written proofs or solutions rarely operate at that level. Instead, obvious steps are skipped to provide a simple, lucid argument. This is especially important in an educational setting, where too many details in an example solution, or too few, can confuse a student. What are the key steps for humans in a given formal solution? We investigate several computational hypotheses in the context of equation solving. Specifically, we take a reinforcement learning agent that solves equations using low-level axioms, and propose a series of methods for abstracting its solutions by selecting key steps. We consider methods based on the semantic distance between subsequent steps, based on the steps with the highest uncertainty for the agent, and based on transitions between latent “high-level skills” learned from a large number of agent-produced solutions. In a human evaluation we find that skill-base simplifications were judged most useful. These results suggest new directions for understanding human mathematical reasoning.

  BibTeX: @inproceedings{poesia2022left, author = {Poesia, Gabriel and Goodman, Noah}, booktitle = {Proceedings of the Annual Meeting of the Cognitive Science Society}, volume = {44}, number = {44}, title = {Left to the Reader: Abstracting Solutions in Mathematical Reasoning}, website = {https://gpoesia.com/papers/poesia2022abstracting.pdf}, year = {2022} }

• Color Overmodification Emerges from Data-Driven Learning and Pragmatic Reasoning. [PDF] [BibTeX]
  F. Fang, K. Sinha, N. D. Goodman, C. Potts, & E. Kreiss. (2022). In Proceedings of the Annual Meeting of the Cognitive Science Society.

  BibTeX: @inproceedings{fang2022color, author = {Fang, Fei and Sinha, Kunal and Goodman, Noah D and Potts, Christopher and Kreiss, Elisa}, title = {Color Overmodification Emerges from Data-Driven Learning and Pragmatic Reasoning}, year = {2022}, booktitle = {Proceedings of the Annual Meeting of the Cognitive Science Society}, volume = {44}, number = {44}, website = {https://arxiv.org/pdf/2205.09172.pdf} }

• Active Learning Helps Pretrained Models Learn the Intended Task. [PDF] [BibTeX]
  A. Tamkin, D. Nguyen, S. Deshpande, J. Mu, & N. Goodman. (2022). In Advances in Neural Information Processing Systems , A. H. Oh, A. Agarwal, D. Belgrave, & K. Cho (Eds.).

  BibTeX: @inproceedings{tamkin2022active, title = {Active Learning Helps Pretrained Models Learn the Intended Task}, author = {Tamkin, Alex and Nguyen, Dat and Deshpande, Salil and Mu, Jesse and Goodman, Noah}, booktitle = {Advances in Neural Information Processing Systems}, editor = {Oh, Alice H. and Agarwal, Alekh and Belgrave, Danielle and Cho, Kyunghyun}, year = {2022}, website = {https://arxiv.org/pdf/2204.08491.pdf} }

• Foundation Posteriors for Approximate Probabilistic Inference. [PDF] [BibTeX]
  M. Wu, & N. Goodman. (2022). In Advances in Neural Information Processing Systems.

  BibTeX: @inproceedings{wu2022foundation, title = {Foundation Posteriors for Approximate Probabilistic Inference}, author = {Wu, Mike and Goodman, Noah}, booktitle = {Advances in Neural Information Processing Systems}, year = {2022}, website = {https://openreview.net/forum?id=DbEVhhuNjr} }

• Geoclidean: Few-Shot Generalization in Euclidean Geometry. [PDF] [BibTeX]
  J. Hsu, J. Wu, & N. Goodman. (2022). In Thirty-sixth Conference on Neural Information Processing Systems Datasets and Benchmarks Track.

  BibTeX: @inproceedings{hsu2022geoclidean, title = {Geoclidean: Few-Shot Generalization in Euclidean Geometry}, author = {Hsu, Joy and Wu, Jiajun and Goodman, Noah}, booktitle = {Thirty-sixth Conference on Neural Information Processing Systems Datasets and Benchmarks Track}, year = {2022}, website = {https://openreview.net/forum?id=3lk54yE2tYJ} }

• CLEVRER-Humans: Describing Physical and Causal Events the Human Way. [PDF] [BibTeX]
  J. Mao, X. Yang, X. Zhang, N. Goodman, & J. Wu. (2022). In Thirty-sixth Conference on Neural Information Processing Systems Datasets and Benchmarks Track.

  BibTeX: @inproceedings{mao2022clevrerhumans, title = {{CLEVRER}-Humans: Describing Physical and Causal Events the Human Way}, author = {Mao, Jiayuan and Yang, Xuelin and Zhang, Xikun and Goodman, Noah and Wu, Jiajun}, booktitle = {Thirty-sixth Conference on Neural Information Processing Systems Datasets and Benchmarks Track}, year = {2022}, website = {https://openreview.net/forum?id=sd1fv0g3UO1} }

• DABS 2.0: Improved Datasets and Algorithms for Universal Self-Supervision. [PDF] [BibTeX]
  A. Tamkin, G. Banerjee, M. Owda, V. Liu, S. Rammoorthy, & N. Goodman. (2022). In Thirty-sixth Conference on Neural Information Processing Systems Datasets and Benchmarks Track.

  BibTeX: @inproceedings{tamkin2022dabs, title = {{DABS} 2.0: Improved Datasets and Algorithms for Universal Self-Supervision}, author = {Tamkin, Alex and Banerjee, Gaurab and Owda, Mohamed and Liu, Vincent and Rammoorthy, Shashank and Goodman, Noah}, booktitle = {Thirty-sixth Conference on Neural Information Processing Systems Datasets and Benchmarks Track}, year = {2022}, website = {https://openreview.net/forum?id=ChWf1E43l4} }

• Improving intrinsic exploration with language abstractions. [PDF] [BibTeX]
  J. Mu, V. Zhong, R. Raileanu, M. Jiang, N. Goodman, T. Rocktäschel, & E. Grefenstette. (2022). In Advances in Neural Information Processing Systems.

  BibTeX: @inproceedings{mu2022improving, title = {Improving intrinsic exploration with language abstractions}, author = {Mu, Jesse and Zhong, Victor and Raileanu, Roberta and Jiang, Minqi and Goodman, Noah and Rockt{\"a}schel, Tim and Grefenstette, Edward}, booktitle = {Advances in Neural Information Processing Systems}, year = {2022}, website = {https://arxiv.org/pdf/2202.08938.pdf} }

• Assistive Teaching of Motor Control Tasks to Humans. [PDF] [BibTeX]
  M. Srivastava, E. Biyik, S. Mirchandani, N. Goodman, & D. Sadigh. (2022). In Advances in Neural Information Processing Systems.

  BibTeX: @inproceedings{srivastava2022assistive, title = {Assistive Teaching of Motor Control Tasks to Humans}, author = {Srivastava, Megha and Biyik, Erdem and Mirchandani, Suvir and Goodman, Noah and Sadigh, Dorsa}, booktitle = {Advances in Neural Information Processing Systems}, year = {2022}, website = {https://iliad.stanford.edu/pdfs/publications/srivastava2022assistive.pdf} }

• STaR: Bootstrapping Reasoning With Reasoning. [PDF] [BibTeX]
  E. Zelikman, Y. Wu, J. Mu, & N. Goodman. (2022). In Advances in Neural Information Processing Systems.

  BibTeX: @inproceedings{zelikman2022star, title = {{ST}aR: Bootstrapping Reasoning With Reasoning}, author = {Zelikman, Eric and Wu, Yuhuai and Mu, Jesse and Goodman, Noah}, booktitle = {Advances in Neural Information Processing Systems}, year = {2022}, website = {https://arxiv.org/abs/2203.14465} }

• Language modeling via stochastic processes. [Abstract] [PDF] [BibTeX]
  R. E. Wang, E. Durmus, N. Goodman, & T. Hashimoto. (2022). In International Conference on Learning Representations.

  Abstract: Modern language models can generate high-quality short texts. However, they often meander or are incoherent when generating longer texts. These issues arise from the next-token-only language modeling objective. To address these issues, we introduce Time Control (TC), a language model that implicitly plans via a latent stochastic process. TC does this by learning a representation which maps the dynamics of how text changes in a document to the dynamics of a stochastic process of interest. Using this representation, the language model can generate text by first implicitly generating a document plan via a stochastic process, and then generating text that is consistent with this latent plan. Compared to domain-specific methods and fine-tuning GPT2 across a variety of text domains, TC improves performance on text infilling and discourse coherence. On long text generation settings, TC preserves the text structure both in terms of ordering (up to +40% better) and text length consistency (up to +17% better). Human evaluators also prefer TC’s output 28.6% more than the baselines.

  BibTeX: @inproceedings{wang2022language, author = {Wang, Rose E and Durmus, Esin and Goodman, Noah and Hashimoto, Tatsunori}, booktitle = {International Conference on Learning Representations}, title = {Language modeling via stochastic processes}, website = {https://arxiv.org/abs/2203.11370}, year = {2022} }

• From partners to populations: A hierarchical Bayesian account of coordination and convention. [Abstract] [PDF] [BibTeX]
  R. D. Hawkins, M. Franke, M. C. Frank, A. E. Goldberg, K. Smith, T. L. Griffiths, & N. D. Goodman. (2022). Psychological Review.

  Abstract: Languages are powerful solutions to coordination problems: they provide stable, shared expectations about how the words we say correspond to the beliefs and intentions in our heads. Yet language use in a variable and non-stationary social environment requires linguistic representations to be flexible: old words acquire new ad hoc or partner-specific meanings on the fly. In this paper, we introduce CHAI (Continual Hierarchical Adaptation through Inference), a hierarchical Bayesian theory of coordination and convention formation that aims to reconcile the long-standing tension between these two basic observations. We argue that the central computational problem of communication is not simply transmission, as in classical formulations, but continual learning and adaptation over multiple timescales. Partner-specific common ground quickly emerges from social inferences within dyadic interactions, while community-wide social conventions are stable priors that have been abstracted away from interactions with multiple partners. We present new empirical data alongside simulations showing how our model provides a computational foundation for several phenomena that have posed a challenge for previous accounts: (1) the convergence to more efficient referring expressions across repeated interaction with the same partner, (2) the gradual transfer of partner-specific common ground to strangers, and (3) the influence of communicative context on which conventions eventually form.

  BibTeX: @article{hawkins2022from, author = {Hawkins, Robert D. and Franke, Michael and Frank, Michael C. and Goldberg, Adele E. and Smith, Kenny and Griffiths, Thomas L. and Goodman, Noah D.}, journal = {Psychological Review}, publisher = {American Psychological Association}, title = {From partners to populations: A hierarchical Bayesian account of coordination and convention}, website = {https://arxiv.org/abs/2104.05857}, year = {2022} }

• Warm (for Winter): Inferring comparison classes in communication. [Abstract] [PDF] [BibTeX]
  M. H. Tessler, & N. D. Goodman. (2022). Cognitive Science.

  Abstract: The meanings of natural language utterances depend heavily on context. Yet, what counts as context is often only implicit in conversation. The utteranceit’s warm outsidesignals that the temperature outside is relatively high, but the temperature could be high relative to a number of different comparison classes: other days of the year, other weeks, other seasons, etc. Theories of context-sensitivity in language agree that the comparison classis a crucial variable for understanding meaning, but little is known abouthow a listener decides upon the comparison class. Using the case study ofgradable adjectives (e.g., warm), we extend a Bayesian model of pragmaticinference to reason flexibly about the comparison class and test its qualitative predictions in a large-scale free-production experiment. We find that human listeners infer the comparison class by reasoning about the kinds ofobservations that would be remarkable enough for a speaker to mention, given the speaker and listener’s shared knowledge of the world. Further, we quantitatively synthesize the model and data using Bayesian data analysis, which reveals that usage frequency and a preference for basic-level categories are two main factors in comparison class inference. This work presents new data and reveals the mechanisms by which human listeners recover the rel-evant aspects of context when understanding language.

  BibTeX: @article{tessler2022warm, author = {Tessler, Michael Henry and Goodman, Noah D.}, journal = {Cognitive Science}, title = {Warm (for Winter): Inferring comparison classes in communication}, website = {https://psyarxiv.com/8bfna}, year = {2022} }

• Logic, Probability, and Pragmatics in Syllogistic Reasoning. [Abstract] [PDF] [BibTeX]
  M. H. Tessler, J. Tenenbaum, & N. D. Goodman. (2022). Topics in Cognitive Science.

  Abstract: Syllogistic reasoning lies at the intriguing intersection of natural and formal reasoning of language and logic. Syllogisms comprise a formal system of reasoning yet make use of natural language quantifiers (e.g., all, some) and invite natural language conclusions. The conclusions people tend to draw from syllogisms, however, deviate substantially from the purely logical system. Are principles of natural language understanding to blame? We introduce a probabilistic pragmatic perspective on syllogistic reasoning: We decompose reasoning with natural language arguments into two subproblems: language comprehension and language production. We formalize models of these processes within the Rational Speech Act framework and explore the pressures that pragmatic reasoning places on the production of conclusions. We test our models on a recent, large data set of syllogistic reasoning and find that the selection process of conclusions from syllogisms are best modeled as a pragmatic speaker who has the goal of aligning the beliefs of a naive listener with those of their own. We compare our model to previously published models that implement two alternative theories—Mental Models and Probability Heuristics—finding that our model quantitatively predicts the full distributions of responses as well as or better than previous accounts, but with far fewer parameters. Our results suggest that human syllogistic reasoning may be best understood not as a poor approximation to ideal logical reasoning, but rather as rational probabilistic inference in support of natural communication.

  BibTeX: @article{tessler2022logic, author = {Tessler, Michael Henry and Tenenbaum, Joshua and Goodman, Noah D.}, journal = {Topics in Cognitive Science}, publisher = {Wiley Online Library}, title = {Logic, Probability, and Pragmatics in Syllogistic Reasoning}, website = {https://onlinelibrary.wiley.com/doi/pdf/10.1111/tops.12593}, year = {2022} }

• The effects of information utility and teachers’ knowledge on evaluations of under-informative pedagogy across development. [Abstract] [PDF] [BibTeX]
  I. Bass, E. Bonawitz, D. Hawthorne-Madell, W. Vong, N. Goodman, & H. Gweon. (2022). Cognition.

  Abstract: Teaching is a powerful way to transmit knowledge, but with this power comes a hazard: When teachers fail to select the best set of evidence for the learner, learners can be misled to draw inaccurate inferences. Evaluating others’ failures as teachers, however, is a nontrivial problem; people may fail to be informative for different reasons, and not all failures are equally blameworthy. How do learners evaluate the quality of teachers, and what factors influence such evaluations? Here, we present a Bayesian model of teacher evaluation that considers the utility of a teacher’s pedagogical sampling given their prior knowledge. In Experiment 1 (N=1168), we test the model predictions against adults’ evaluations of a teacher who demonstrated all or a subset of the functions on a novel device. Consistent with the model predictions, participants’ ratings integrated information about the number of functions taught, their values, as well as how much the teacher knew. Using a modified paradigm for children, Experiments 2 (N=48) and 3 (N=40) found that preschool-aged children (2a, 3) and adults (2b) make nuanced judgments of teacher quality that are well predicted by the model. However, after an unsuccessful attempt to replicate the results with preschoolers (Experiment 4, N=24), in Experiment 5 (N=24) we further investigate the development of teacher evaluation in a sample of seven- and eight-year-olds. These older children successfully distinguished teachers based on the amount and value of what was demonstrated, and their ability to evaluate omissions relative to the teacher’s knowledge state was related to their tendency to spontaneously reference the teacher’s knowledge when explaining their evaluations. In sum, our work illustrates how the human ability to learn from others supports not just learning about the world but also learning about the teachers themselves. By reasoning about others’ informativeness, learners can evaluate others’ teaching and make better learning decisions.

  BibTeX: @article{bass2022the, author = {Bass, Ilona and Bonawitz, Elizabeth and Hawthorne-Madell, Daniel and Vong, Wai and Goodman, Noah and Gweon, Hyowon}, year = {2022}, month = may, pages = {104999}, title = {The effects of information utility and teachers’ knowledge on evaluations of under-informative pedagogy across development}, volume = {222}, journal = {Cognition}, website = {https://www.researchgate.net/publication/357787304_The_effects_of_information_utility_and_teachers'_knowledge_on_evaluations_of_under-informative_pedagogy_across_development} }

Preprints

2021

• Applying Probabilistic Programming to Affective Computing. [Abstract] [PDF] [BibTeX]
  D. Ong, H. Soh, J. Zaki, & N. Goodman. (2021). IEEE Transactions on Affective Computing. [Best of IEEE Transactions on Affective Computing 2021 Paper Collection.]

  Abstract: Affective Computing is a rapidly growing field spurred by advancements in artificial intelligence, but often, held back by the inability to translate psychological theories of emotion into tractable computational models. To address this, we propose a probabilistic programming approach to affective computing, which models psychological-grounded theories as generative models of emotion, and implements them as stochastic, executable computer programs. We first review probabilistic approaches that integrate reasoning about emotions with reasoning about other latent mental states (e.g., beliefs, desires) in context. Recently-developed probabilistic programming languages offer several key desidarata over previous approaches, such as: (i) flexibility in representing emotions and emotional processes; (ii) modularity and compositionality; (iii) integration with deep learning libraries that facilitate efficient inference and learning from large, naturalistic data; and (iv) ease of adoption. Furthermore, using a probabilistic programming framework allows a standardized platform for theory-building and experimentation: Competing theories (e.g., of appraisal or other emotional processes) can be easily compared via modular substitution of code followed by model comparison. To jumpstart adoption, we illustrate our points with executable code that researchers can easily modify for their own models. We end with a discussion of applications and future directions of the probabilistic programming approach

  BibTeX: @article{OngEtAl19, annote = {[Best of IEEE Transactions on Affective Computing 2021 Paper Collection.]}, author = {{Ong}, D. and {Soh}, H. and {Zaki}, J. and {Goodman}, N.}, journal = {IEEE Transactions on Affective Computing}, title = {Applying Probabilistic Programming to Affective Computing}, website = {https://arxiv.org/abs/1903.06445}, year = {2021} }

• The division of labor in communication: Speakers help listeners account for asymmetries in visual perspective. [PDF] [BibTeX]
  R. D. Hawkins, H. Gweon, & N. D. Goodman. (2021). Cognitive Science.

  BibTeX: @article{hawkins2020division, author = {Hawkins, Robert D and Gweon, Hyowon and Goodman, Noah D}, journal = {Cognitive Science}, number = {3}, pages = {e12926}, title = {The division of labor in communication: Speakers help listeners account for asymmetries in visual perspective}, volume = {45}, website = {https://arxiv.org/abs/1807.09000}, year = {2021} }

• Conditional Negative Sampling for Contrastive Learning of Visual Representations. [PDF] [BibTeX]
  M. Wu, M. Mosse, C. Zhuang, D. Yamins, & N. Goodman. (2021). In International Conference on Learning Representations.

  BibTeX: @inproceedings{wu2021conditional, author = {Wu, Mike and Mosse, Milan and Zhuang, Chengxu and Yamins, Daniel and Goodman, Noah}, booktitle = {International Conference on Learning Representations}, title = {Conditional Negative Sampling for Contrastive Learning of Visual Representations}, website = {https://arxiv.org/abs/2010.02037}, year = {2021} }

• Viewmaker Networks: Learning Views for Unsupervised Representation Learning. [PDF] [BibTeX]
  A. Tamkin, M. Wu, & N. Goodman. (2021). In International Conference on Learning Representations.

  BibTeX: @inproceedings{tamkin2021viewmaker, author = {Tamkin, Alex and Wu, Mike and Goodman, Noah}, booktitle = {International Conference on Learning Representations}, title = {Viewmaker Networks: Learning Views for Unsupervised Representation Learning}, website = {https://arxiv.org/abs/2010.07432}, year = {2021} }

• Pragmatic Code Autocomplete. [PDF] [BibTeX]
  G. Poesia, & N. Goodman. (2021). In Proceedings of the AAAI Conference on Artificial Intelligence.

  BibTeX: @inproceedings{poesia2021pragmatic, author = {Poesia, Gabriel and Goodman, Noah}, booktitle = {Proceedings of the AAAI Conference on Artificial Intelligence}, title = {Pragmatic Code Autocomplete}, volume = {35}, website = {https://gpoesia.com/papers/poesia2021pragmatic.pdf}, year = {2021} }

• Question Generation for Adaptive Education. [PDF] [BibTeX]
  M. Srivastava, & N. Goodman. (2021). In Association for Computational Linguistics (ACL).

  BibTeX: @inproceedings{srivastava2021question, author = {Srivastava, Megha and Goodman, Noah}, booktitle = {Association for Computational Linguistics (ACL)}, title = {Question Generation for Adaptive Education}, website = {https://arxiv.org/abs/2106.04262}, year = {2021} }

• Emergent Communication of Generalizations. [Abstract] [PDF] [BibTeX]
  J. Mu, & N. Goodman. (2021). In Advances in Neural Information Processing Systems (NeurIPS).

  Abstract: To build agents that can collaborate effectively with others, recent research has trained artificial agents to communicate with each other in Lewis-style referential games. However, this often leads to successful but uninterpretable communication. We argue that this is due to the game objective: communicating about a single object in a shared visual context is prone to overfitting and does not encourage language useful beyond concrete reference. In contrast, human language conveys a rich variety of abstract ideas. To promote such skills, we propose games that require communicating generalizations over sets of objects representing abstract visual concepts, optionally with separate contexts for each agent. We find that these games greatly improve systematicity and interpretability of the learned languages, according to several metrics in the literature. Finally, we propose a method for identifying logical operations embedded in the emergent languages by learning an approximate compositional reconstruction of the language.

  BibTeX: @inproceedings{mu2021emergent, author = {Mu, Jesse and Goodman, Noah}, booktitle = {Advances in Neural Information Processing Systems (NeurIPS)}, title = {Emergent Communication of Generalizations}, website = {https://arxiv.org/abs/2106.02668}, year = {2021} }

• Contrastive Reinforcement Learning of Symbolic Reasoning Domains. [Abstract] [PDF] [BibTeX]
  G. Poesia, W. X. Dong, & N. Goodman. (2021). In Advances in Neural Information Processing Systems (NeurIPS).

  Abstract: Abstract symbolic reasoning, as required in domains such as mathematics and logic, is a key component of human intelligence. Solvers for these domains have important applications, especially to computer-assisted education. But learning to solve symbolic problems is challenging for machine learning algorithms. Existing models either learn from human solutions or use hand-engineered features, making them expensive to apply in new domains. In this paper, we instead consider symbolic domains as simple environments where states and actions are given as unstructured text, and binary rewards indicate whether a problem is solved. This flexible setup makes it easy to specify new domains, but search and planning become challenging. We introduce four environments inspired by the Mathematics Common Core Curriculum, and observe that existing Reinforcement Learning baselines perform poorly. We then present a novel learning algorithm, Contrastive Policy Learning (ConPoLe) that explicitly optimizes the InfoNCE loss, which lower bounds the mutual information between the current state and next states that continue on a path to the solution. ConPoLe successfully solves all four domains. Moreover, problem representations learned by ConPoLe enable accurate prediction of the categories of problems in a real mathematics curriculum. Our results suggest new directions for reinforcement learning in symbolic domains, as well as applications to mathematics education.

  BibTeX: @inproceedings{poesia2021contrastive, author = {Poesia, Gabriel and Dong, WenXin and Goodman, Noah}, booktitle = {Advances in Neural Information Processing Systems (NeurIPS)}, title = {Contrastive Reinforcement Learning of Symbolic Reasoning Domains}, website = {https://arxiv.org/abs/2106.09146}, year = {2021} }

• DABS: A Domain-Agnostic Benchmark for Self-Supervised Learning. [Abstract] [PDF] [BibTeX]
  A. Tamkin, V. Liu, R. Lu, D. Fein, C. Schultz, & N. Goodman. (2021). In Advances in Neural Information Processing Systems (NeurIPS) Datasets and Benchmarks Track.

  Abstract: Self-supervised learning algorithms, including BERT and SimCLR, have enabled significant strides in fields like natural language processing, computer vision, and speech processing. However, the domain-specificity of these algorithms means that solutions must be handcrafted for each new setting, including myriad healthcare, scientific, and multimodal domains. To catalyze progress towards more domain6 agnostic methods, we introduce DABS: a Domain-Agnostic Benchmark for Self7 supervised learning. To perform well on DABS, an algorithm must be pretrained on six unlabeled datasets from diverse domains: natural images, text, speech recordings, medical imaging, multichannel sensor data, and paired text and images, and then perform well on a set of labeled tasks in each domain. We also present e-Mix and ShED: two baseline domain-agnostic algorithms; their relatively modest performance demonstrates that significant progress is needed before self-supervised learning is an out-of-the-box solution for arbitrary domains. Code for benchmark datasets and baseline algorithms is available at [redacted].

  BibTeX: @inproceedings{tamkin2021dabs, author = {Tamkin, Alex and Liu, Vincent and Lu, Rongfei and Fein, Daniel and Schultz, Colin and Goodman, Noah}, booktitle = {Advances in Neural Information Processing Systems (NeurIPS) Datasets and Benchmarks Track}, title = {DABS: A Domain-Agnostic Benchmark for Self-Supervised Learning}, website = {https://openreview.net/pdf?id=Uk2mymgn_LZ}, year = {2021} }

• Generative Grading: Near Human-level Accuracy for Automated Feedback on Richly Structured Problems. [Abstract] [PDF] [BibTeX]
  A. Malik, M. Wu, V. Vasavada, J. Song, M. Coots, J. Mitchell, N. Goodman, & C. Piech. (2021). In Proceedings of The 14th International Conference on Educational Data Mining (EDM).

  Abstract: Access to high-quality education at scale is limited by the difficulty of providing student feedback on open-ended assignments in structured domains like computer programming, graphics, and short response questions. This problem has proven to be exceptionally difficult: for humans, it requires large amounts of manual work, and for computers, until recently, achieving anything near human-level accuracy has been unattainable. In this paper, we present generative grading: a novel computational approach for providing feedback at scale that is capable of accurately grading student work and providing nuanced, interpretable feedback. Our approach uses generative descriptions of student cognition, written as probabilistic programs, to synthesise millions of labelled example solutions to a problem; we then learn to infer feedback for real student solutions based on this cognitive model. We apply our methods to three settings. In block-based coding, we achieve a 50% improvement upon the previous best results for feedback, achieving super-human accuracy. In two other widely different domains – graphical tasks and short text answers – we achieve major improvement over the previous state of the art by about 4x and 1.5x respectively, approaching human accuracy. In a real classroom, we ran an experiment where we used our system to augment human graders, yielding doubled grading accuracy while halving grading time.

  BibTeX: @inproceedings{malik2021generative, author = {Malik, Ali and Wu, Mike and Vasavada, Vrinda and Song, Jinpeng and Coots, Madison and Mitchell, John and Goodman, Noah and Piech, Chris}, booktitle = {Proceedings of The 14th International Conference on Educational Data Mining (EDM)}, title = {Generative Grading: Near Human-level Accuracy for Automated Feedback on Richly Structured Problems}, website = {https://arxiv.org/abs/1905.09916}, year = {2021} }

• Neural Event Semantics for Grounded Language Understanding. [Abstract] [PDF] [BibTeX]
  S. Buch, L. Fei-Fei, & N. Goodman. (2021). Transactions of the Association for Computational Linguistics (TACL).

  Abstract: We present a new conjunctivist framework, neural event semantics (NES), for compositional grounded language understanding. Our approach treats all words as classifiers that compose to form a sentence meaning by multiplying output scores. These classifiers apply to spatial regions (events) and NES derives its semantic structure from language by routing events to different classifier argument inputs via soft attention. NES is trainable end-to-end by gradient descent with minimal supervision. We evaluate our method on compositional grounded language tasks in controlled synthetic and real-world settings. NES offers stronger generalization capability than standard function-based compositional frameworks, while improving accuracy over state-of-the-art neural methods on real-world language tasks.

  BibTeX: @article{buch2021transactions, author = {Buch, Shyamal and Fei-Fei, Li and Goodman, Noah}, date-modified = {2021-12-14 16:11:25 -0800}, journal = {Transactions of the Association for Computational Linguistics (TACL)}, title = {Neural Event Semantics for Grounded Language Understanding}, website = {https://transacl.org/ojs/index.php/tacl/article/view/3007}, year = {2021} }

• Improving Compositionality of Neural Networks by Decoding Representations to Inputs. [Abstract] [PDF] [BibTeX]
  M. Wu, N. Goodman, & S. Ermon. (2021). In Advances in Neural Information Processing Systems (NeurIPS).

  Abstract: In traditional software programs, we take for granted how easy it is to debug code by tracing program logic from variables back to input, apply unit tests and assertion statements to block erroneous behavior, and compose programs together. But as the programs we write grow more complex, it becomes hard to apply traditional software to applications like computer vision or natural language. Although deep learning programs have demonstrated strong performance on these applications, they sacrifice many of the functionalities of traditional software programs. In this paper, we work towards bridging the benefits of traditional and deep learning programs by jointly training a generative model to constrain neural network activations to "decode" back to inputs. Doing so enables practitioners to probe and track information encoded in activation(s), apply assertion-like constraints on what information is encoded in an activation, and compose separate neural networks together in a plug-and-play fashion. In our experiments, we demonstrate applications of decodable representations to out-of-distribution detection, adversarial examples, calibration, and fairness – while matching standard neural networks in accuracy.

  BibTeX: @inproceedings{wu2021improving, author = {Wu, Mike and Goodman, Noah and Ermon, Stefano}, booktitle = {Advances in Neural Information Processing Systems (NeurIPS)}, title = {Improving Compositionality of Neural Networks by Decoding Representations to Inputs}, website = {https://arxiv.org/abs/2106.00769}, year = {2021} }

• Open-domain clarification question generation without question examples. [Abstract] [PDF] [BibTeX]
  J. White, G. Poesia, R. Hawkins, D. Sadigh, & N. Goodman. (2021). In Proceedings of the 2021 Conference on Empirical Methods on Natural Language Processing (EMNLP).

  Abstract: An overarching goal of natural language processing is to enable machines to communicate seamlessly with humans. However, natural language can be ambiguous or unclear. In cases of uncertainty, humans engage in an interactive process known as repair: asking questions and seeking clarification until their uncertainty is resolved. We propose a framework for building a visually grounded question-asking model capable of producing polar (yes-no) clarification questions to resolve misunderstandings in dialogue. Our model uses an expected information gain objective to derive informative questions from an off-the-shelf image captioner without requiring any supervised question-answer data. We demonstrate our model’s ability to pose questions that improve communicative success in a goal-oriented 20 questions game with synthetic and human answerers.

  BibTeX: @inproceedings{white2021open, author = {White, Julia and Poesia, Gabriel and Hawkins, Robert and Sadigh, Dorsa and Goodman, Noah}, booktitle = {Proceedings of the 2021 Conference on Empirical Methods on Natural Language Processing (EMNLP)}, title = {Open-domain clarification question generation without question examples}, website = {https://arxiv.org/abs/2110.09779}, year = {2021} }

• Calibrate your listeners! Robust communication-based training for pragmatic speakers. [Abstract] [PDF] [BibTeX]
  R. Wang, J. White, J. Mu, & N. Goodman. (2021). In Findings of the 2021 Conference on Empirical Methods on Natural Language Processing (Findings of EMNLP).

  Abstract: To be good conversational partners, natural language processing (NLP) systems should be trained to produce contextually useful utterances. Prior work has investigated training NLP systems with communication-based objectives, where a neural listener stands in as a communication partner. However, these systems commonly suffer from semantic drift where the learned language diverges radically from natural language. We propose a method that uses a population of neural listeners to regularize speaker training. We first show that language drift originates from the poor uncertainty calibration of a neural listener, which makes high-certainty predictions on novel sentences. We explore ensemble- and dropout-based populations of listeners and find that the former results in better uncertainty quantification. We evaluate both population-based objectives on reference games, and show that the ensemble method with better calibration enables the speaker to generate pragmatic utterances while scaling to a large vocabulary and generalizing to new games and listeners.

  BibTeX: @inproceedings{wang2021calibrate, author = {Wang, Rose and White, Julia and Mu, Jesse and Goodman, Noah}, booktitle = {Findings of the 2021 Conference on Empirical Methods on Natural Language Processing (Findings of EMNLP)}, title = {Calibrate your listeners! Robust communication-based training for pragmatic speakers}, website = {https://arxiv.org/abs/2110.05422}, year = {2021} }

• A counterfactual simulation model of causal judgments for physical events. [Abstract] [PDF] [BibTeX]
  T. Gerstenberg, N. Goodman, D. Lagnado, & J. Tenenbaum. (2021). Psychological Review.

  Abstract: How do people make causal judgments about physical events? We introduce the counterfactual simulation model (CSM) which predicts causal judgments in physical settings by comparing what actually happened with what would have happened in relevant counterfactual situations. The CSM postulates different aspects of causation that capture the extent to which a cause made a difference to whether and how the outcome occurred, and whether the cause was sufficient and robust. We test the CSM in several experiments in which participants make causal judgments about dynamic collision events. A preliminary study establishes a very close quantitative mapping between causal and counterfactual judgments. Experiment 1 demonstrates that counterfactuals are necessary for explaining causal judgments. Participants’ judgments differed dramatically between pairs of situations in which what actually happened was identical, but where what would have happened differed. Experiment 2 features multiple candidate causes and shows that participants’ judgments are sensitive to different aspects of causation. The CSM provides a better fit to participants’ judgments than a heuristic model which uses features based on what actually happened. We discuss how the CSM can be used to model the semantics of different causal verbs, how it captures related concepts such as physical support, and how its predictions extend beyond the physical domain.

  BibTeX: @article{tobias2021counterfactual, author = {Gerstenberg, Tobias and Goodman, Noah and Lagnado, David and Tenenbaum, Joshua}, journal = {Psychological Review}, publisher = {American Psychological Association}, title = {A counterfactual simulation model of causal judgments for physical events}, website = {https://psycnet.apa.org/record/2021-52648-001}, year = {2021} }

• Temperature as Uncertainty in Contrastive Learning. [Abstract] [PDF] [BibTeX]
  O. Zhang, M. Wu, J. Bayrooti, & N. Goodman. (2021). In NeurIPS Workshop on Self-Supervised Learning.

  Abstract: Contrastive learning has demonstrated great capability to learn representations without annotations, even outperforming supervised baselines. However, it still lacks important properties useful for real-world application, one of which is uncertainty. In this paper, we propose a simple way to generate uncertainty scores for many contrastive methods by re-purposing temperature, a mysterious hyperparameter used for scaling. By observing that temperature controls how sensitive the objective is to specific embedding locations, we aim to learn temperature as an input-dependent variable, treating it as a measure of embedding confidence. We call this approach "Temperature as Uncertainty", or TaU. Through experiments, we demonstrate that TaU is useful for out-of-distribution detection, while remaining competitive with benchmarks on linear evaluation. Moreover, we show that TaU can be learned on top of pretrained models, enabling uncertainty scores to be generated post-hoc with popular off-the-shelf models. In summary, TaU is a simple yet versatile method for generating uncertainties for contrastive learning.

  BibTeX: @inproceedings{zhang2021temperature, author = {Zhang, Oliver and Wu, Mike and Bayrooti, Jasmine and Goodman, Noah}, booktitle = {NeurIPS Workshop on Self-Supervised Learning}, title = {Temperature as Uncertainty in Contrastive Learning}, website = {https://arxiv.org/abs/2110.04403}, year = {2021} }

• Multi-party referential communication in complex strategic games. [Abstract] [PDF] [BibTeX]
  J. Mankewitz, V. Boyce, B. Waldon, G. Loukatou, D. Yu, J. Mu, N. D. Goodman, & M. C. Frank. (2021). In NeurIPS Workshop on Meaning in Context.

  Abstract: Verbal communication is an ubiquitous aspect of human interaction occurring in many contexts; however, it is primarily studied in the limited context of twopeople communicating information. Understanding communication in complex, multi-party interactions is both a scientific challenge for psycholinguistics and an engineering challenge for creating artificial agents who can participate in these richer contexts. We adapted the reference game paradigm to an online 3-player game where players refer to objects in order to coordinate selections based on the available utilities. We ran games with shared or individual payoffs and with or without access to language. Our paradigm can also be used for artificial agents; we trained reinforcement learning-based agents on the same task as a comparison. Our dataset shows the same patterns found in simpler reference games and contains rich language of reference and negotiation.

  BibTeX: @inproceedings{mankewitz2021multi, author = {Mankewitz, Jessica and Boyce, Veronica and Waldon, Brandon and Loukatou, Georgia and Yu, Dhara and Mu, Jesse and Goodman, Noah D and Frank, Michael C}, booktitle = {NeurIPS Workshop on Meaning in Context}, title = {Multi-party referential communication in complex strategic games}, website = {https://psyarxiv.com/tfb3d}, year = {2021} }

• Learning to solve complex tasks by growing knowledge culturally across generations. [Abstract] [PDF] [BibTeX]
  M. H. Tessler, J. Madeano, P. A. Tsividis, B. Harper, N. D. Goodman, & J. B. Tenenbaum. (2021). In NeurIPS Workshop on Cooperative AI. [Best paper award winner.]

  Abstract: Knowledge built culturally across generations allows humans to learn far more than an individual could glean from their own experience in a lifetime. Cultural knowledge in turn rests on language: language is the richest record of what previous generations believed, valued, and practiced, and how these evolved over time. The power and mechanisms of language as a means of cultural learning, however, are not well understood, and as a result, current AI systems do not leverage language as a means for cultural knowledge transmission. Here, we take a first step towards reverse-engineering cultural learning through language. We developed a suite of complex tasks in the form of minimalist-style video games, which we deployed in an iterated learning paradigm. Human participants were limited to only two attempts (two lives) to beat each game and were allowed to write a message to a future participant who read the message before playing. Knowledge accumulated gradually across generations, allowing later generations to advance further in the games and perform more efficient actions. Multigenerational learning followed a strikingly similar trajectory to individuals learning alone with an unlimited number of lives. Successive generations of learners were able to succeed by expressing distinct types of knowledge in natural language: the dynamics of the environment, valuable goals, dangerous risks, and strategies for success. The video game paradigm we pioneer here is thus a rich test bed for developing AI systems capable of acquiring and transmitting cultural knowledge.

  BibTeX: @inproceedings{tessler2021learning, annote = {[Best paper award winner.]}, author = {Tessler, Michael Henry and Madeano, Jason and Tsividis, Pedro A. and Harper, Brin and Goodman, Noah D. and Tenenbaum, Joshua B.}, booktitle = {NeurIPS Workshop on Cooperative AI}, title = {Learning to solve complex tasks by growing knowledge culturally across generations}, website = {https://arxiv.org/abs/2107.13377}, year = {2021} }

• Causal Distillation for Language Models. [Abstract] [PDF] [BibTeX]
  Z. Wu, A. Geiger, J. Rozner, E. Kreiss, H. Lu, T. Icard, C. Potts, & N. D. Goodman. (2021). In NAACL-HLT.

  Abstract: Distillation efforts have led to language models that are more compact and efficient without serious drops in performance. The standard approach to distillation trains a student model against two objectives: a task-specific objective (e.g., language modeling) and an imitation objective that encourages the hidden states of the student model to be similar to those of the larger teacher model. In this paper, we show that it is beneficial to augment distillation with a third objective that encourages the student to imitate the causal computation process of the teacher through interchange intervention training(IIT). IIT pushes the student model to become a causal abstraction of the teacher model - a simpler model with the same causal structure. IIT is fully differentiable, easily implemented, and combines flexibly with other objectives. Compared with standard distillation of BERT, distillation via IIT results in lower perplexity on Wikipedia (masked language modeling) and marked improvements on the GLUE benchmark (natural language understanding), SQuAD (question answering), and CoNLL-2003 (named entity recognition).

  BibTeX: @inproceedings{zhengxuan2021causal, title = {Causal Distillation for Language Models}, author = {Wu, Zhengxuan and Geiger, Atticus and Rozner, Josh and Kreiss, Elisa and Lu, Hanson and Icard, Thomas and Potts, Christopher and Goodman, Noah D.}, booktitle = {NAACL-HLT}, website = {https://arxiv.org/abs/2112.02505}, year = {2021} }

Preprints

2020

• Continual Adaptation for Efficient Machine Communication. [PDF] [BibTeX]
  R. X. D. Hawkins, M. Kwon, D. Sadigh, & N. D. Goodman. (2020). In Proceedings of the 24th Conference on Computational Natural Language Learning.

  BibTeX: @inproceedings{hawkins2020continual, author = {Hawkins, Robert X. D. and Kwon, Minae and Sadigh, Dorsa and Goodman, Noah D.}, booktitle = {Proceedings of the 24th Conference on Computational Natural Language Learning}, title = {Continual Adaptation for Efficient Machine Communication}, website = {https://arxiv.org/pdf/1911.09896.pdf}, year = {2020} }

• Shaping Visual Representations with Language for Few-shot Classification. [PDF] [BibTeX]
  J. Mu, P. Liang, & N. Goodman. (2020). In Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics.

  BibTeX: @inproceedings{mu2020shaping, author = {Mu, Jesse and Liang, Percy and Goodman, Noah}, booktitle = {Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics}, title = {Shaping Visual Representations with Language for Few-shot Classification}, website = {https://arxiv.org/abs/1911.02683}, year = {2020} }

• The interactions of rational, pragmatic agents lead to efficient language structure and use. [PDF] [BibTeX]
  B. N. Peloquin, N. D. Goodman, & M. C. Frank. (2020). Topics in Cognitive Science.

  BibTeX: @article{peloquin2020interactions, author = {Peloquin, Benjamin N and Goodman, Noah D and Frank, Michael C}, date-added = {2020-09-05 00:51:23 +0000}, date-modified = {2020-09-05 00:51:23 +0000}, journal = {Topics in Cognitive Science}, number = {1}, pages = {433--445}, publisher = {Wiley Online Library}, title = {The interactions of rational, pragmatic agents lead to efficient language structure and use}, volume = {12}, website = {https://cogsci.mindmodeling.org/2019/papers/0171/0171.pdf}, year = {2020} }

• Meta-Amortized Variational Inference and Learning. [PDF] [BibTeX]
  M. Wu, K. Choi, N. D. Goodman, & S. Ermon. (2020). In AAAI.

  BibTeX: @inproceedings{wu2020meta, author = {Wu, Mike and Choi, Kristy and Goodman, Noah D and Ermon, Stefano}, booktitle = {AAAI}, date-added = {2020-09-05 00:59:50 +0000}, date-modified = {2020-09-05 00:59:50 +0000}, pages = {6404--6412}, title = {Meta-Amortized Variational Inference and Learning}, website = {https://arxiv.org/abs/1902.01950}, year = {2020} }

• Variational Item Response Theory: Fast, Accurate, and Expressive. [PDF] [BibTeX]
  M. Wu, R. L. Davis, B. W. Domingue, C. Piech, & N. Goodman. (2020). In Proceedings of The 13th International Conference on Educational Data Mining (EDM 2020). [Best paper award winner.]

  BibTeX: @inproceedings{wu2020variational, annote = {{[Best paper award winner.]}}, author = {Wu, Mike and Davis, Richard L and Domingue, Benjamin W and Piech, Chris and Goodman, Noah}, booktitle = {Proceedings of The 13th International Conference on Educational Data Mining (EDM 2020)}, pages = {257--268}, title = {Variational Item Response Theory: Fast, Accurate, and Expressive}, website = {https://arxiv.org/abs/2002.00276}, year = {2020} }

• When redundancy is useful: A Bayesian approach to “overinformative” referring expressions. [Abstract] [PDF] [BibTeX]
  J. Degen, R. D. Hawkins, C. Graf, E. Kreiss, & N. D. Goodman. (2020). Psychological Review.

  Abstract: Referring is one of the most basic and prevalent uses of language. How do speakers choose from the wealth of referring expressions at their disposal? Rational theories of language use have come under attack for decades for not being able to account for the seemingly irrational overinformativeness ubiquitous in referring expressions. Here we present a novel production model of referring expressions within the Rational Speech Act framework that treats speakers as agents that rationally trade off cost and informativeness of utterances. Crucially, we relax the assumption that informativeness is computed with respect to a deterministic Boolean semantics, in favor of a non-deterministic continuous semantics. This innovation allows us to capture a large number of seemingly disparate phenomena within one unified framework: the basic asymmetry in speakers’ propensity to overmodify with color rather than size; the increase in overmodification in complex scenes; the increase in overmodification with atypical features; and the increase in specificity in nominal reference as a function of typicality. These findings cast a new light on the production of referring expressions: rather than being wastefully overinformative, reference is usefully redundant.

  BibTeX: @article{degen2020redundancy, author = {Degen, Judith and Hawkins, Robert D and Graf, Caroline and Kreiss, Elisa and Goodman, Noah D}, date-added = {2020-09-05 01:00:24 +0000}, date-modified = {2020-09-05 01:35:49 +0000}, journal = {Psychological Review}, publisher = {American Psychological Association}, title = {When redundancy is useful: A Bayesian approach to ``overinformative'' referring expressions}, website = {https://arxiv.org/abs/1903.08237}, year = {2020} }

• Generalizing meanings from partners to populations: Hierarchical inference supports convention formation on networks. [PDF] [BibTeX]
  R. D. Hawkins, N. D. Goodman, A. E. Goldberg, & T. L. Griffiths. (2020). In Proceedings of the 41st Annual Conference of the Cognitive Science Society. [Winner of the 2020 Cognitive Science Society computational modeling prize for Language.]

  BibTeX: @inproceedings{hawkins2020generalizing, annote = {[Winner of the 2020 Cognitive Science Society computational modeling prize for Language.]}, author = {Hawkins, Robert D and Goodman, Noah D and Goldberg, Adele E and Griffiths, Thomas L}, booktitle = {Proceedings of the 41st Annual Conference of the Cognitive Science Society}, date-added = {2020-09-05 01:12:17 +0000}, date-modified = {2020-09-05 01:12:23 +0000}, title = {Generalizing meanings from partners to populations: Hierarchical inference supports convention formation on networks}, website = {https://arxiv.org/abs/2002.01510}, year = {2020} }

• Characterizing the dynamics of learning in repeated reference games. [PDF] [BibTeX]
  R. D. Hawkins, M. C. Frank, & N. D. Goodman. (2020). Cognitive Science.

  BibTeX: @article{hawkins2020characterizing, author = {Hawkins, Robert D and Frank, Michael C and Goodman, Noah D}, date-added = {2020-09-05 01:13:50 +0000}, date-modified = {2020-09-05 01:13:50 +0000}, journal = {Cognitive Science}, number = {6}, pages = {e12845}, publisher = {Wiley Online Library}, title = {Characterizing the dynamics of learning in repeated reference games}, volume = {44}, website = {https://arxiv.org/abs/1912.07199}, year = {2020} }

• Learning to Refer Informatively by Amortizing Pragmatic Reasoning. [PDF] [BibTeX]
  J. White, J. Mu, & N. Goodman. (2020). In Proceedings of the 42nd Annual Meeting of the Cognitive Science Society.

  BibTeX: @inproceedings{white2020learning, author = {White, Julia and Mu, Jesse and Goodman, Noah}, booktitle = {Proceedings of the 42nd Annual Meeting of the Cognitive Science Society}, title = {Learning to Refer Informatively by Amortizing Pragmatic Reasoning}, website = {https://arxiv.org/abs/2006.00418}, year = {2020} }

• Language Through a Prism: A Spectral Approach for Multiscale Language Representations. [PDF] [BibTeX]
  A. Tamkin, D. Jurafsky, & N. Goodman. (2020). In Advances in Neural Information Processing Systems.

  BibTeX: @inproceedings{tamkin2020language, author = {Tamkin, Alex and Jurafsky, Dan and Goodman, Noah}, booktitle = {Advances in Neural Information Processing Systems}, title = {Language Through a Prism: A Spectral Approach for Multiscale Language Representations}, website = {https://proceedings.neurips.cc/paper/2020/hash/3acb2a202ae4bea8840224e6fce16fd0-Abstract.html}, year = {2020} }

• Investigating Transferability in Pretrained Language Models. [PDF] [BibTeX]
  A. Tamkin, T. Singh, D. Giovanardi, & N. Goodman. (2020). In Findings of the 2020 Conference on Empirical Methods on Natural Language Processing.

  BibTeX: @inproceedings{tamkin2020investigating, author = {Tamkin, Alex and Singh, Trisha and Giovanardi, Davide and Goodman, Noah}, booktitle = {Findings of the 2020 Conference on Empirical Methods on Natural Language Processing}, title = {Investigating Transferability in Pretrained Language Models}, website = {https://arxiv.org/abs/2004.14975}, year = {2020} }

• Polite speech emerges from competing social goals. [PDF] [BibTeX]
  E. J. Yoon, M. H. Tessler, N. D. Goodman, & M. C. Frank. (2020). Open Mind.

  BibTeX: @article{yoon2018polite, author = {Yoon, Erica J and Tessler, Michael Henry and Goodman, Noah D and Frank, Michael C}, date-modified = {2021-02-25 07:31:53 -0800}, journal = {Open Mind}, pages = {71--87}, publisher = {MIT Press}, title = {Polite speech emerges from competing social goals}, volume = {4}, website = {https://psyarxiv.com/67ne8/}, year = {2020} }

• Analyzing Machine-Learned Representations: A Natural Language Case Study. [PDF] [BibTeX]
  I. Dasgupta, D. Guo, S. Gershman, & N. Goodman. (2020). Cognitive Science.

  BibTeX: @article{dasgupta2020analyzing, author = {Dasgupta, Ishita and Guo, Demi and Gershman, Samuel and Goodman, Noah}, journal = {Cognitive Science}, pages = {e12925}, title = {Analyzing Machine-Learned Representations: A Natural Language Case Study}, website = {https://arxiv.org/abs/1909.05885}, year = {2020} }

Preprints

2019

• The Language of Generalization. [Abstract] [PDF] [BibTeX]
  M. H. Tessler, & N. D. Goodman. (2019). Psychological Review.

  Abstract: Language provides simple ways of communicating generalizable knowledge to each other (e.g., “Birds fly”, “John hikes”, “Fire makes smoke”). Though found in every language and emerging early in development, the language of generalization is philosphically puzzling and has resisted precise formalization. Here, we propose the first formal account of the language of generalization that makes quantitative predictions about human understanding. We test our model in three diverse domains: generalizations about categories (generic language), events (habitual language), and causes (causal language). The model explains the gradience in human endorsement through the interplay between a simple truth-conditional semantic theory and diverse beliefs about properties. This work opens the door to understanding precisely how abstract knowledge is learned from language.

  BibTeX: @article{tessler2019, author = {Tessler, Michael Henry and Goodman, Noah D.}, date-modified = {2019-03-29 15:16:45 +0000}, journal = {Psychological Review}, number = {3}, pages = {395-436}, title = {The Language of Generalization}, volume = {126}, website = {https://arxiv.org/abs/1608.02926}, year = {2019} }

• Reasoning about Social Sources to Learn from Actions and Outcomes. [Abstract] [PDF] [BibTeX]
  D. Hawthorne-Madell, & N. D. Goodman. (2019). Decision.

  Abstract: Learning what others know, especially experts, is a crucial shortcut to understanding the world. Other people’s actions and utterances are thus a powerful source of evidence. However, people do not simply copy others’ choices or stated beliefs; rather, they infer what others believe and integrate these beliefs with their own. In this paper, we present a computational account of the inference and integration process that underpins learning from a combination of social and direct evidence. This account formalizes the learner’s intuitive understanding of psychology—or theory of mind (ToM)—including attributes such as confidence, reliability, and knowledgeability. It then shows how ToM is the lens used to interpret another person’s choices, weighing them against the learner’s own direct evidence. To test this account, we develop an experimental paradigm that allows for graded manipulation of social and direct evidence, and for quantitative measurement of the learner’s resulting beliefs. Four experiments test the predictions of the model, manipulating knowledgeability, confidence, and reliability of the social source. Learners’ behavior is consistent with our quantitative and qualitative model predictions across all 4 experiments, demonstrating subtle interactions between evidence and the attributes of those learned from.

  BibTeX: @article{Hawthorne2017_SocialLearning, author = {Hawthorne-Madell, D. and Goodman, N. D.}, date-added = {2017-06-27 17:59:10 +0000}, date-modified = {2018-07-03 02:33:41 +0000}, journal = {Decision}, number = {1}, pages = {17---60}, title = {Reasoning about Social Sources to Learn from Actions and Outcomes}, volume = {6}, website = {papers/HawthorneGoodman2019-SocialSources.pdf}, year = {2019} }

• Computational models of emotion inference in Theory of Mind: A review and roadmap. [Abstract] [PDF] [BibTeX]
  D. C. Ong, J. Zaki, & N. D. Goodman. (2019). Topics in Cognitive Science.

  Abstract: Research on social cognition has fruitfully applied computational modeling approaches to explain how observers understand and reason about others’ mental states. By contrast, there has been less work on modeling observers’ understanding of emotional states. We propose an intuitive theory framework to studying affective cognition—how humans reason about emotions—and derive a taxonomy of inferences within affective cognition. Using this taxonomy, we review formal computational modeling work on such inferences, including: causal reasoning about how others react to events, reasoning about unseen causes of emotions, reasoning with multiple cues, as well as reasoning from emotions to other mental states. Additionally, we provide a roadmap for future research by charting out inferences—such as hypothetical and counterfactual reasoning about emotions—that are ripe for future computational modeling work. This framework proposes unifying these various types of reasoning as Bayesian inference within a common “intuitive Theory of Emotion”. Finally, we end with a discussion of important theoretical and methodological challenges that lie ahead in modeling affective cognition.

  BibTeX: @article{OngEtAl18_Topics, author = {Ong, Desmond C. and Zaki, Jamil and Goodman, Noah D.}, date-modified = {2019-09-06 16:01:25 +0000}, journal = {Topics in Cognitive Science}, title = {Computational models of emotion inference in Theory of Mind: A review and roadmap}, website = {papers/OngEtAl18_Topics.pdf}, year = {2019} }

• Zero Shot Learning for Code Education: Rubric Sampling with Deep Learning Inference. [PDF] [BibTeX]
  M. Wu, M. Mosse, N. Goodman, & C. Piech. (2019). In Association for the Advancement of Artificial Intelligence (AAAI). [Winner best student paper award.]

  BibTeX: @inproceedings{Wu2019-AAAI, annote = {[Winner best student paper award.]}, author = {Wu, Mike and Mosse, Milan and Goodman, Noah and Piech, Chris}, booktitle = {Association for the Advancement of Artificial Intelligence (AAAI)}, date-modified = {2019-03-29 15:10:21 +0000}, title = {Zero Shot Learning for Code Education: Rubric Sampling with Deep Learning Inference}, website = {https://arxiv.org/pdf/1809.01357.pdf}, year = {2019} }

• A thousand studies for the price of one: Accelerating psychological science with Pushkin. [PDF] [BibTeX]
  J. K. Hartshorne, J. R. de Leeuw, N. D. Goodman, M. Jennings, & T. J. O’Donnell. (2019). Behavior Research Methods.

  BibTeX: @article{hartshorne2019thousand, author = {Hartshorne, Joshua K and de Leeuw, Joshua R and Goodman, Noah D and Jennings, Mariela and O'Donnell, Timothy J}, journal = {Behavior Research Methods}, pages = {1--22}, publisher = {Springer}, title = {A thousand studies for the price of one: Accelerating psychological science with Pushkin}, website = {https://pubmed.ncbi.nlm.nih.gov/30746644/}, year = {2019} }

• Differentiable Antithetic Sampling for Variance Reduction in Stochastic Variational Inference. [PDF] [BibTeX]
  M. Wu, N. Goodman, & S. Ermon. (2019). In AISTATS.

  BibTeX: @inproceedings{wu2018differentiable, author = {Wu, Mike and Goodman, Noah and Ermon, Stefano}, booktitle = {AISTATS}, date-modified = {2019-03-30 17:01:37 +0000}, title = {Differentiable Antithetic Sampling for Variance Reduction in Stochastic Variational Inference}, website = {https://arxiv.org/pdf/1810.02555.pdf}, year = {2019} }

• Definitely, maybe: A new experimental paradigm for investigating the pragmatics of evidential devices across languages. [PDF] [BibTeX]
  J. Degen, A. Trotzke, G. Scontras, E. Wittenberg, & N. D. Goodman. (2019). Journal of Pragmatics.

  BibTeX: @article{Degen2019-Pragmatics, author = {Degen, Judith and Trotzke, Andreas and Scontras, Greg and Wittenberg, Eva and Goodman, Noah D.}, date-modified = {2019-03-30 17:05:03 +0000}, journal = {Journal of Pragmatics}, pages = {33-48}, title = {Definitely, maybe: A new experimental paradigm for investigating the pragmatics of evidential devices across languages}, volume = {140}, website = {https://reader.elsevier.com/reader/sd/pii/S0378216618301383?token=B8F2AA2B18CD2879BC74D0ABDDBF067930A160E7CEB749ACAED7BA4E3789F7DC6D0CD2A13F2547D34F49E3389482AFB5}, year = {2019} }

• On the grammatical source of adjective ordering preferences. [BibTeX]
  G. Scontras, J. Degen, & N. D. Goodman. (2019). Semantics and Pragmatics.

  BibTeX: @article{Scontras2019-SP, author = {Scontras, Greg and Degen, Judith and Goodman, Noah D.}, date-modified = {2019-03-29 16:04:06 +0000}, journal = {Semantics and Pragmatics}, number = {7}, title = {On the grammatical source of adjective ordering preferences}, volume = {12}, year = {2019} }

• The Emergence of Social Norms and Conventions. [PDF] [BibTeX]
  R. X. D. Hawkins, N. D. Goodman, & R. L. Goldstone. (2019). Trends in Cognitive Sciences.

  BibTeX: @article{Hawkins2019-TICS, author = {Hawkins, Robert X. D. and Goodman, Noah D and Goldstone, Robert L}, date-modified = {2019-03-30 17:04:16 +0000}, journal = {Trends in Cognitive Sciences}, number = {2}, pages = {158--169}, title = {The Emergence of Social Norms and Conventions}, volume = {23}, website = {https://rxdhawkins.files.wordpress.com/2018/12/normemergence.pdf}, year = {2019} }

• Pyro: Deep Universal Probabilistic Programming. [PDF] [BibTeX]
  E. Bingham, J. P. Chen, M. Jankowiak, F. Obermeyer, N. Pradhan, T. Karaletsos, R. Singh, P. Szerlip, P. Horsfall, & N. D. Goodman. (2019). Journal of Machine Learning Research.

  BibTeX: @article{pyro_jmlr, author = {Bingham, Eli and Chen, Jonathan P. and Jankowiak, Martin and Obermeyer, Fritz and Pradhan, Neeraj and Karaletsos, Theofanis and Singh, Rohit and Szerlip, Paul and Horsfall, Paul and Goodman, Noah D.}, journal = {Journal of Machine Learning Research}, number = {28}, pages = {1-6}, title = {Pyro: Deep Universal Probabilistic Programming}, volume = {20}, website = {https://jmlr.org/papers/v20/18-403.html}, year = {2019} }

• DisSent: Learning Sentence Representations from Explicit Discourse Relations. [Abstract] [PDF] [BibTeX]
  A. Nie, E. Bennett, & N. Goodman. (2019). In Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics. doi:10.18653/v1/P19-1442

  Abstract: Learning effective representations of sentences is one of the core missions of natural language understanding. Existing models either train on a vast amount of text, or require costly, manually curated sentence relation datasets. We show that with dependency parsing and rule-based rubrics, we can curate a high quality sentence relation task by leveraging explicit discourse relations. We show that our curated dataset provides an excellent signal for learning vector representations of sentence meaning, representing relations that can only be determined when the meanings of two sentences are combined. We demonstrate that the automatically curated corpus allows a bidirectional LSTM sentence encoder to yield high quality sentence embeddings and can serve as a supervised fine-tuning dataset for larger models such as BERT. Our fixed sentence embeddings achieve high performance on a variety of transfer tasks, including SentEval, and we achieve state-of-the-art results on Penn Discourse Treebank’s implicit relation prediction task.

  BibTeX: @inproceedings{nie-etal-2019-dissent, address = {Florence, Italy}, author = {Nie, Allen and Bennett, Erin and Goodman, Noah}, booktitle = {Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics}, doi = {10.18653/v1/P19-1442}, month = jul, pages = {4497--4510}, publisher = {Association for Computational Linguistics}, title = {{D}is{S}ent: Learning Sentence Representations from Explicit Discourse Relations}, website = {https://aclanthology.org/P19-1442/}, year = {2019}, bdsk-url-1 = {https://aclanthology.org/P19-1442/} }

• Learning to Explain: Answering Why-Questions via Rephrasing. [Abstract] [PDF] [BibTeX]
  A. Nie, E. Bennett, & N. Goodman. (2019). In Proceedings of the First Workshop on NLP for Conversational AI. doi:10.18653/v1/W19-4113

  Abstract: Providing plausible responses to why questions is a challenging but critical goal for language based human-machine interaction. Explanations are challenging in that they require many different forms of abstract knowledge and reasoning. Previous work has either relied on human-curated structured knowledge bases or detailed domain representation to generate satisfactory explanations. They are also often limited to ranking pre-existing explanation choices. In our work, we contribute to the under-explored area of generating natural language explanations for general phenomena. We automatically collect large datasets of explanation-phenomenon pairs which allow us to train sequence-to-sequence models to generate natural language explanations. We compare different training strategies and evaluate their performance using both automatic scores and human ratings. We demonstrate that our strategy is sufficient to generate highly plausible explanations for general open-domain phenomena compared to other models trained on different datasets.

  BibTeX: @inproceedings{nie-etal-2019-learning, address = {Florence, Italy}, author = {Nie, Allen and Bennett, Erin and Goodman, Noah}, booktitle = {Proceedings of the First Workshop on NLP for Conversational AI}, doi = {10.18653/v1/W19-4113}, month = aug, pages = {113--120}, publisher = {Association for Computational Linguistics}, title = {Learning to Explain: Answering Why-Questions via Rephrasing}, website = {https://aclanthology.org/W19-4113/}, year = {2019}, bdsk-url-1 = {https://aclanthology.org/W19-4113/} }

• Cake or broccoli? Recency biases children’s verbal responses. [PDF] [BibTeX]
  E. Sumner, E. DeAngelis, M. Hyatt, N. Goodman, & C. Kidd. (2019). PloS One.

  BibTeX: @article{sumner2019cake, author = {Sumner, Emily and DeAngelis, Erika and Hyatt, Mara and Goodman, Noah and Kidd, Celeste}, date-added = {2019-07-11 21:20:43 +0000}, date-modified = {2019-07-11 21:20:43 +0000}, journal = {PloS one}, number = {6}, pages = {e0217207}, publisher = {Public Library of Science}, title = {Cake or broccoli? Recency biases children's verbal responses}, volume = {14}, website = {https://journals.plos.org/plosone/article?id=10.1371/journal.pone.0217207}, year = {2019} }

• Lost in Machine Translation: A Method to Reduce Meaning Loss. [PDF] [BibTeX]
  R. Cohn-Gordon, & N. D. Goodman. (2019). In NAACL-HLT.

  BibTeX: @inproceedings{CohnGordon2019LostIM, author = {Cohn-Gordon, Reuben and Goodman, Noah D.}, booktitle = {NAACL-HLT}, date-added = {2019-09-06 15:40:52 +0000}, date-modified = {2019-09-06 15:40:52 +0000}, title = {Lost in Machine Translation: A Method to Reduce Meaning Loss}, website = {https://www.semanticscholar.org/paper/Lost-in-Machine-Translation%3A-A-Method-to-Reduce-Cohn-Gordon-Goodman/ed7c3eececad3915c865b7b11d88c338b0e0cbe1}, year = {2019} }

• Learning from Omission. [Abstract] [PDF] [BibTeX]
  B. McDowell, & N. Goodman. (2019). In Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics.

  Abstract: Pragmatic reasoning allows humans to go beyond the literal meaning when interpret- ing language in context. Previous work has shown that such reasoning can improve the performance of already-trained language understanding systems. Here, we explore whether pragmatic reasoning during training can improve the quality of learned meanings. Our experiments on reference game data show that end-to-end pragmatic training produces more accurate utterance interpretation models, especially when data is sparse and language is complex.

  BibTeX: @inproceedings{mcdowell-goodman-2019-learning, address = {Florence, Italy}, author = {McDowell, Bill and Goodman, Noah}, booktitle = {Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics}, date-added = {2019-09-06 15:53:47 +0000}, date-modified = {2019-09-06 15:53:59 +0000}, pages = {619--628}, publisher = {Association for Computational Linguistics}, title = {Learning from Omission}, website = {https://aclanthology.org/P19-1059/}, year = {2019} }

• ShapeGlot: Learning Language for Shape Differentiation. [PDF] [BibTeX]
  P. Achlioptas, J. Fan, R. X. D. Hawkins, N. D. Goodman, & L. J. Guibas. (2019). In IEEE International Conference on Computer Vision (ICCV).

  BibTeX: @inproceedings{achlioptas2019shapeglot, author = {Achlioptas, Panos and Fan, Judy and Hawkins, Robert XD and Goodman, Noah D and Guibas, Leonidas J}, booktitle = {IEEE International Conference on Computer Vision (ICCV)}, date-added = {2019-09-06 15:57:22 +0000}, date-modified = {2019-09-06 15:59:45 +0000}, title = {ShapeGlot: Learning Language for Shape Differentiation}, website = {https://arxiv.org/abs/1905.02925}, year = {2019} }

• Disentangling contributions of visual information and interaction history in the formation of graphical conventions. [Abstract] [PDF] [BibTeX]
  R. X. D. Hawkins, M. Sano, N. D. Goodman, & J. E. Fan. (2019). In Proceedings of the 41st Annual Conference of the Cognitive Science Society.

  Abstract: Drawing is a versatile technique for visual communication, ranging from photorealistic renderings to schematic diagrams consisting entirely of symbols. How does a medium spanning such a broad range of appearances reliably convey meaning? A natural possibility is that drawings derive meaning from both their visual properties as well as shared knowledge between people who use them to communicate. Here we evaluate this possibility in a drawing-based reference game in which two participants repeatedly communicated about visual objects. Across a series of controlled experiments, we found that pairs of participants discover increasingly sparse yet effective ways of depicting objects. These gains were specific to those objects that were repeatedly referenced, and went beyond what could be explained by task practice or the visual properties of the drawings alone. We employed modern techniques from computer vision to characterize how the high-level visual features of drawings changed, finding that drawings of the same object became more consistent within a pair of participants and divergent across participants from different pairs. Taken together, these findings suggest that visual communication promotes the emergence of depictions whose meanings are increasingly determined by shared knowledge rather than their visual properties alone.

  BibTeX: @inproceedings{Hawkins2019GraphicalCF, author = {Hawkins, Robert X. D. and Sano, Megumi and Goodman, Noah D. and Fan, Judith E.}, booktitle = {Proceedings of the 41st Annual Conference of the {C}ognitive {S}cience {S}ociety}, date-added = {2019-09-06 16:06:44 +0000}, date-modified = {2019-09-06 16:12:05 +0000}, title = {Disentangling contributions of visual information and interaction history in the formation of graphical conventions}, website = {https://cogtoolslab.github.io/pdf/hawkinssano_cogsci_2019.pdf}, year = {2019} }

• Continual Adaptation for Efficient Machine Communication. [PDF] [BibTeX]
  R. X. D. Hawkins, M. Kwon, D. Sadigh, & N. D. Goodman. (2019). In ICML Workshop on Adaptive & Multitask Learning: Algorithms & Systems. [Best paper award winner.]

  BibTeX: @inproceedings{hawkins2019continual, annote = {[Best paper award winner.]}, author = {Hawkins, Robert X. D. and Kwon, Minae and Sadigh, Dorsa and Goodman, Noah D.}, booktitle = {ICML Workshop on Adaptive \& Multitask Learning: Algorithms \& Systems}, date-added = {2019-09-06 16:24:18 +0000}, date-modified = {2019-09-06 16:28:23 +0000}, title = {Continual Adaptation for Efficient Machine Communication}, website = {https://iliad.stanford.edu/pdfs/publications/hawkins2020continual.pdf}, year = {2019} }

• Pragmatic Inference and Visual Abstraction Enable Contextual Flexibility During Visual Communication. [Abstract] [PDF] [BibTeX]
  J. E. Fan, R. D. Hawkins, M. Wu, & N. D. Goodman. (2019). Computational Brain & Behavior. doi:10.1007/s42113-019-00058-7

  Abstract: Visual modes of communication are ubiquitous in modern life—from maps to data plots to political cartoons. Here, we investigate drawing, the most basic form of visual communication. Participants were paired in an online environment to play a drawing-based reference game. On each trial, both participants were shown the same four objects, but in different locations. The sketcher’s goal was to draw one of these objects so that the viewer could select it from the array. On “close” trials, objects belonged to the same basic-level category, whereas on “far” trials objects belonged to different categories. We found that people exploited shared information to efficiently communicate about the target object: on far trials, sketchers achieved high recognition accuracy while applying fewer strokes, using less ink, and spending less time on their drawings than on close trials. We hypothesized that humans succeed in this task by recruiting two core faculties: visual abstraction, the ability to perceive the correspondence between an object and a drawing of it; and pragmatic inference, the ability to judge what information would help a viewer distinguish the target from distractors. To evaluate this hypothesis, we developed a computational model of the sketcher that embodied both faculties, instantiated as a deep convolutional neural network nested within a probabilistic program. We found that this model fit human data well and outperformed lesioned variants. Together, this work provides the first algorithmically explicit theory of how visual perception and social cognition jointly support contextual flexibility in visual communication.

  BibTeX: @article{Fan2019, author = {Fan, Judith E. and Hawkins, Robert D. and Wu, Mike and Goodman, Noah D.}, date-added = {2019-09-06 16:31:38 +0000}, date-modified = {2019-09-06 16:31:55 +0000}, doi = {10.1007/s42113-019-00058-7}, issn = {2522-087X}, journal = {Computational Brain {\&} Behavior}, title = {Pragmatic Inference and Visual Abstraction Enable Contextual Flexibility During Visual Communication}, website = {https://arxiv.org/pdf/1903.04448.pdf}, year = {2019}, bdsk-url-1 = {https://doi.org/10.1007/s42113-019-00058-7} }

• The first crank of the cultural ratchet: Learning and transmitting concepts through language. [PDF] [BibTeX]
  S. Chopra, M. H. Tessler, & N. D. Goodman. (2019). In Proceedings of the 41st Annual Meeting of the Cognitive Science Society.

  BibTeX: @inproceedings{chopra2019first, author = {Chopra, Sahil and Tessler, Michael Henry and Goodman, Noah D}, booktitle = {Proceedings of the 41st Annual Meeting of the Cognitive Science Society}, date-added = {2020-09-05 00:50:14 +0000}, date-modified = {2020-09-05 01:32:38 +0000}, pages = {226--232}, title = {The first crank of the cultural ratchet: Learning and transmitting concepts through language}, website = {https://cogsci.mindmodeling.org/2019/papers/0060/0060.pdf}, year = {2019} }

• Shaping Visual Representations with Language for Few-shot Classification. [PDF] [BibTeX]
  J. Mu, P. Liang, & N. Goodman. (2019). In NeurIPS Workshop on Visually Grounded Interaction and Language.

  BibTeX: @inproceedings{mu2019shaping, author = {Mu, Jesse and Liang, Percy and Goodman, Noah}, booktitle = {NeurIPS Workshop on Visually Grounded Interaction and Language}, title = {Shaping Visual Representations with Language for Few-shot Classification}, website = {https://arxiv.org/abs/1911.02683}, year = {2019} }

• The interactions of rational, pragmatic agents lead to efficient language structure and use. [PDF] [BibTeX]
  B. N. Peloquin, N. D. Goodman, & M. C. Frank. (2019). In Proceedings of the 41st Annual Conference of the Cognitive Science Society. [Winner of the 2019 Cognitive Science Society computational modeling prize for Language.]

  BibTeX: @inproceedings{peloquin2019interactions, annote = {{[Winner of the 2019 Cognitive Science Society computational modeling prize for Language.]}}, author = {Peloquin, Benjamin N and Goodman, Noah D and Frank, Michael C}, booktitle = {Proceedings of the 41st Annual Conference of the Cognitive Science Society}, pages = {912--917}, title = {The interactions of rational, pragmatic agents lead to efficient language structure and use}, website = {https://cogsci.mindmodeling.org/2019/papers/0171/0171.pdf}, year = {2019} }

• Variational Bayesian optimal experimental design. [PDF] [BibTeX]
  A. Foster, M. Jankowiak, E. Bingham, P. Horsfall, Y. W. Teh, T. Rainforth, & N. Goodman. (2019). In Advances in Neural Information Processing Systems.

  BibTeX: @inproceedings{foster2019variational, author = {Foster, Adam and Jankowiak, Martin and Bingham, Elias and Horsfall, Paul and Teh, Yee Whye and Rainforth, Thomas and Goodman, Noah}, booktitle = {Advances in Neural Information Processing Systems}, date-added = {2020-09-05 00:58:02 +0000}, date-modified = {2020-09-05 00:58:02 +0000}, pages = {14036--14047}, title = {Variational Bayesian optimal experimental design}, website = {https://arxiv.org/abs/1903.05480}, year = {2019} }

• Extracting customer perceptions of product sustainability from online reviews. [PDF] [BibTeX]
  N. El Dehaibi, N. D. Goodman, & E. F. MacDonald. (2019). Journal of Mechanical Design. [Best paper award winner.]

  BibTeX: @article{el2019extracting, annote = {[Best paper award winner.]}, author = {El Dehaibi, Nasreddine and Goodman, Noah D and MacDonald, Erin F}, date-added = {2020-09-05 01:08:08 +0000}, date-modified = {2020-09-05 01:12:00 +0000}, journal = {Journal of Mechanical Design}, number = {12}, publisher = {American Society of Mechanical Engineers Digital Collection}, title = {Extracting customer perceptions of product sustainability from online reviews}, volume = {141}, website = {https://asmedigitalcollection.asme.org/mechanicaldesign/article/141/12/121103/958466/Extracting-Customer-Perceptions-of-Product}, year = {2019} }

Preprints

• Tensor Variable Elimination for Plated Factor Graphs. [PDF] [BibTeX]
  F. Obermeyer, E. Bingham, M. Jankowiak, N. Pradhan, J. Chiu, A. M. Rush, & N. D. Goodman. 2019-Preprint In ICML.

  BibTeX: @inproceedings{obermeyer2019tensor, author = {Obermeyer, Fritz and Bingham, Eli and Jankowiak, Martin and Pradhan, Neeraj and Chiu, Justin and Rush, Alexander M. and Goodman, Noah D.}, booktitle = {ICML}, date-modified = {2019-09-06 15:50:42 +0000}, journal = {arXiv preprint arXiv:1902.03210}, title = {Tensor Variable Elimination for Plated Factor Graphs}, website = {https://arxiv.org/pdf/1902.03210}, year = {2019-Preprint} }

2018

• Learning Physical Parameters from Dynamic Scenes. [Abstract] [PDF] [BibTeX]
  T. Ullman, A. Stuhlmüller, N. Goodman, & J. Tenenbaum. (2018). Cognitive Psychology.

  Abstract: Humans acquire their most basic physical concepts early in development, and continue to enrich and expand their intuitive physics throughout life as they are exposed to more and varied dynamical environments. We introduce a hierarchical Bayesian framework to explain how people can learn physical parameters at multiple levels. In contrast to previous Bayesian models of theory acquisition (Tenenbaum, Kemp, Griffiths, & Goodman, 2011), we work with more expressive probabilistic program representations suitable for learning the forces and properties that govern how objects interact in dynamic scenes unfolding over time. We compare our model to human learners on a challenging task of estimating multiple physical parameters in novel microworlds given short movies. This task requires people to reason simultaneously about multiple interacting physical laws and properties. People are generally able to learn in this setting and are consistent in their judgments. Yet they also make systematic errors indicative of the approximations people might make in solving this computationally demanding problem with limited computational resources. We propose two approximations that complement the top-down Bayesian approach. One approximation model relies on a more bottom-up feature-based inference scheme. The second approximation combines the strengths of the bottom-up and top-down approaches, by taking the feature-based inference as its point of departure for a search in physical-parameter space.

  BibTeX: @article{UllmanEtAl2018, author = {Ullman, Tomer and Stuhlm\"uller, Andreas and Goodman, Noah and Tenenbaum, Joshua}, date-modified = {2018-07-24 21:06:28 +0000}, journal = {Cognitive Psychology}, pages = {57--82}, title = {Learning Physical Parameters from Dynamic Scenes}, volume = {104}, website = {https://stuhlmueller.org/papers/physics-cogpsy2017.pdf}, year = {2018} }

• Extremely costly intensifiers are stronger than quite costly ones. [Abstract] [PDF] [BibTeX]
  E. D. Bennett, & N. D. Goodman. (2018). Cognition.

  Abstract: We show that the wide range in strengths of intensifying degree adverbs (e.g. very and extremely) can be partly explained by pragmatic inference based on differing cost, rather than differing semantics. The pragmatic theory predicts a linear relationship between the meaning of intensifiers and their length and log-frequency. We first test this prediction in three studies, using two different dependent measures, finding that higher utterance cost (i.e. higher word length or surprisal) does predict stronger meanings. In two additional studies we confirm that the relationship between length and meaning is present even for novel words. We discuss the implications for adverbial meaning and the more general question of how extensive non-arbitrary form-meaning association may be in language.

  BibTeX: @article{BennettGoodman2018, author = {Bennett, Erin D and Goodman, Noah D}, journal = {Cognition}, pages = {147-161}, title = {Extremely costly intensifiers are stronger than quite costly ones}, volume = {178}, website = {papers/BennettGoodman2018.pdf}, year = {2018} }

• Emerging Abstractions: Lexical conventions are shaped by communicative context. [Abstract] [PDF] [BibTeX]
  R. X. D. Hawkins, M. Franke, K. Smith, & N. D. Goodman. (2018). In Proceedings of the Fortieth Annual Conference of the Cognitive Science Society.

  Abstract: Words exist for referring at many levels of specificity: from the broadest (thing) to the most specific (Fido). What drives the emergence of these levels of abstraction? Recent computational theories of language evolution suggest that communicative demands of the environment may play a deciding role. We hypothesize that language users are more likely to lexicalize specific names (e.g. Fido) when the context frequently requires making fine distinctions between entities; conversely, they should develop a more compressed system of conventions for abstract categories (e.g. dog) in coarser contexts. We test this hypothesis by manipulating context in a repeated reference game where pairs of participants interactively coordinate on an artificial communication system. We show qualitative differences in the levels of abstraction that emerged in different contexts and introduce a statistical approach to probe the dynamics of emergence. These findings illuminate the local pragmatic learning mechanisms that may drive global language evolution.

  BibTeX: @inproceedings{hawkins2018cogsci, author = {Hawkins, Robert X. D. and Franke, Michael and Smith, Kenny and Goodman, Noah D.}, booktitle = {Proceedings of the Fortieth Annual Conference of the Cognitive Science Society}, date-added = {2017-02-22 13:01:38 +0000}, date-modified = {2017-02-22 13:03:04 +0000}, title = {Emerging Abstractions: Lexical conventions are shaped by communicative context}, website = {papers/HawkinsEtAl2018-Cogsci.pdf}, year = {2018} }

• Happier than thou? A self-enhancement bias in emotion attribution.. [Abstract] [PDF] [BibTeX]
  D. C. Ong, N. D. Goodman, & J. Zaki. (2018). Emotion.

  Abstract: People tend to judge themselves as exhibiting above average levels of desirable traits—including competence, kindness, and life satisfaction—but does this self-enhancement extend to emotional responses? Here, we explore this question by having people attribute emotions to themselves and others following simple gambles. We demonstrate that people display an emotional self-enhancement bias that varies with the context of the emotion-eliciting situation. People judge themselves as experiencing more positive emotional reactions on average, and they also believed that others’ emotions are more sensitive to gamble outcomes, such that people judge others to experience stronger negative affect in response to negative outcomes (Study 1). This self-enhancement bias further tracks social distance, such that people attribute less positive and more negative emotion to more dissimilar, as compared to more similar others (Study 2). People also predict less favorable emotional states for themselves and others experiencing events in the future, as compared to the present (Study 3), suggesting that this attribution bias extends across multiple dimensions of psychological distance. Broadly, these data suggest that people exhibit self-enhancement in emotion attribution, but do so in subtle ways that depend on situational and social factors.

  BibTeX: @article{ong2018happier, author = {Ong, Desmond C and Goodman, Noah D and Zaki, Jamil}, journal = {Emotion}, number = {1}, pages = {116}, publisher = {American Psychological Association}, title = {Happier than thou? A self-enhancement bias in emotion attribution.}, volume = {18}, website = {papers/OngEtAl2017-Emotion.pdf}, year = {2018} }

• An information-theoretic explanation of adjective ordering preferences. [Abstract] [PDF] [BibTeX]
  M. Hahn, J. Degen, N. Goodman, D. Jurafsky, & and Richard Futrell. (2018). In Proceedings of the Fortieth Annual Conference of the Cognitive Science Society.

  Abstract: Across languages, adjectives are subject to ordering restrictions. Recent research shows that these are predicted by adjective subjectivity, but the question remains open why this is the case. We first conduct a corpus study and not only replicate the subjectivity effect, but also find a previously undocumented effect of mutual information between adjectives and nouns. We then describe a rational model of adjective use in which listeners explicitly reason about judgments made by different speakers, formalizing the notion of subjectivity as agreement between speakers. We show that, once incremental processing is combined with memory limitations, our model predicts effects both of subjectivity and mutual information. We confirm the adequacy of our model by evaluating it on corpus data, finding that it correctly predicts ordering in unseen data with an accuracy of 96.2%. This suggests that adjective ordering can be explained by general principles of human communication and language processing.

  BibTeX: @inproceedings{Hahn_information_2018, author = {Hahn, Michael and Degen, Judith and Goodman, Noah and Jurafsky, Dan and and Richard Futrell}, booktitle = {Proceedings of the Fortieth Annual Conference of the Cognitive Science Society}, title = {An information-theoretic explanation of adjective ordering preferences}, website = {papers/hahn-cogsci2018.pdf}, year = {2018} }

• Evaluating Compositionality in Sentence Embeddings. [Abstract] [PDF] [BibTeX]
  I. Dasgupta, D. Guo, A. Stuhlmüller, S. J. Gershman, & N. D. Goodman. (2018). In Proceedings of the Fortieth Annual Conference of the Cognitive Science Society.

  Abstract: An important challenge for human-like AI is compositional semantics. Recent research has attempted to address this by using deep neural networks to learn vector space embeddings of sentences, which then serve as input to other tasks. We present a new dataset for one such task, “natural language inference” (NLI), that cannot be solved using only word-level knowledge and requires some compositionality. We find that the performance of state of the art sentence embeddings (InferSent; Conneau et al., 2017) on our new dataset is poor. We analyze the decision rules learned by InferSent and find that they are consistent with simple heuristics that are ecologically valid in its training dataset. Further, we find that augmenting training with our dataset improves test performance on our dataset without loss of performance on the original training dataset. This highlights the importance of structured datasets in better understanding and improving AI systems.

  BibTeX: @inproceedings{DasguptaEtAl2018-Cogsci, author = {Dasgupta, Ishita and Guo, Demi and Stuhlm{\"u}ller, Andreas and Gershman, Samuel J. and Goodman, Noah D.}, booktitle = {Proceedings of the Fortieth Annual Conference of the Cognitive Science Society}, date-added = {2018-05-21 21:25:06 +0000}, date-modified = {2019-03-29 14:59:54 +0000}, title = {Evaluating Compositionality in Sentence Embeddings}, website = {papers/DasguptaEtAl2018-Cogsci.pdf}, year = {2018} }

• Remembrance of inferences past: Amortization in human hypothesis generation. [Abstract] [PDF] [BibTeX]
  I. Dasgupta, E. Schulz, N. D. Goodman, & S. J. Gershman. (2018). Cognition.

  Abstract: Bayesian models of cognition assume that people compute probability distributions over hypotheses. However, the required computations are frequently intractable or prohibitively expensive. Since people often encounter many closely related distributions, selective reuse of computations (amortized inference) is a computationally efficient use of the brain’s limited resources. We present three experiments that provide evidence for amortization in human probabilistic reasoning. When sequentially answering two related queries about natural scenes, participants’ responses to the second query systematically depend on the structure of the first query. This influence is sensitive to the content of the queries, only appearing when the queries are related. Using a cognitive load manipulation, we find evidence that people amortize summary statistics of previous inferences, rather than storing the entire distribution. These findings support the view that the brain trades off accuracy and computational cost, to make efficient use of its limited cognitive resources to approximate probabilistic inference.

  BibTeX: @article{DasguptaEtAl18_Cognition, author = {Dasgupta, Ishita and Schulz, Eric and Goodman, Noah D. and Gershman, Samuel J.}, date-added = {2018-05-21 21:31:45 +0000}, date-modified = {2018-05-21 21:32:00 +0000}, journal = {Cognition}, pages = {67 - 81}, title = {Remembrance of inferences past: Amortization in human hypothesis generation}, volume = {178}, website = {https://www.biorxiv.org/content/biorxiv/early/2017/12/11/231837.full.pdf}, year = {2018} }

• Planning, Inference and Pragmatics in Sequential Language Games. [Abstract] [PDF] [BibTeX]
  F. Khani, N. D. Goodman, & P. Liang. (2018). Transactions of the Association for Computational Linguistics (TACL).

  Abstract: We study sequential language games in which two players, each with private information, communicate to achieve a common goal. In such games, a successful player must (i) infer the partner’s private information from the partner’s messages, (ii) generate messages that are most likely to help with the goal, and (iii) reason pragmatically about the partner’s strategy. We propose a model that captures all three characteristics and demonstrate their importance in capturing human behavior on a new goal-oriented dataset we collected using crowdsourcing.

  BibTeX: @article{Khani2018, author = {Khani, Fereshte and Goodman, Noah D. and Liang, Percy}, date-added = {2018-07-03 02:36:20 +0000}, date-modified = {2018-07-03 02:39:24 +0000}, journal = {Transactions of the Association for Computational Linguistics (TACL)}, title = {Planning, Inference and Pragmatics in Sequential Language Games}, website = {https://arxiv.org/pdf/1805.11774.pdf}, year = {2018} }

• webppl-oed: A practical optimal experiment design system. [Abstract] [PDF] [BibTeX]
  L. Ouyang, M. H. Tessler, D. Ly, & N. D. Goodman. (2018). In Proceedings of the Fortieth Annual Conference of the Cognitive Science Society.

  Abstract: An essential part of cognitive science is designing experiments that distinguish competing models. This requires patience and ingenuity—there is often a large space of possible experiments one could run but only a small subset that might yield informative results. We need not comb this space by hand: If we use formal models and explicitly declare the space of experiments, we can automate the search for good experiments, looking for those with high expected information gain. Here, we present an automated system for experiment design called webppl-oed. In our system, users simply declare their models and experiment space; in return, they receive a list of experiments ranked by their expected information gain. We demonstrate our system in two case studies, where we use it to design experiments in studies of sequence prediction and categorization. We find strong empirical validation that our automatically designed experiments were indeed optimal.

  BibTeX: @inproceedings{Ouyang2018, author = {Ouyang, Long and Tessler, Michael Henry and Ly, D and Goodman, Noah D}, booktitle = {Proceedings of the Fortieth Annual Conference of the Cognitive Science Society}, date-added = {2018-07-03 02:42:41 +0000}, date-modified = {2018-07-03 02:47:16 +0000}, title = {webppl-oed: A practical optimal experiment design system}, website = {papers/OuyangTessler2018-oed-cogsci-final.pdf}, year = {2018} }

• An incremental iterated response model of pragmatics. [PDF] [BibTeX]
  R. Cohn-Gordon, N. D. Goodman, & and Christopher Potts. (2018). In Proceedings of the Society for Computation in Linguistics (SCiL).

  BibTeX: @inproceedings{Cohn-Gordon2018-SCiL, author = {Cohn-Gordon, Reuben and Goodman, Noah D. and and Christopher Potts}, booktitle = {Proceedings of the Society for Computation in Linguistics (SCiL)}, date-modified = {2019-03-29 15:00:12 +0000}, publisher = {Linguistic Society of America.}, title = {An incremental iterated response model of pragmatics}, website = {https://arxiv.org/pdf/1810.00367.pdf}, year = {2018} }

• Bias and Generalization in Deep Generative Models: An Empirical Study. [PDF] [BibTeX]
  S. Zhao, H. Ren, A. Yuan, J. Song, N. Goodman, & S. Ermon. (2018). In Advances in Neural Information Processing Systems 31 , S. Bengio, H. Wallach, H. Larochelle, K. Grauman, N. Cesa-Bianchi, & R. Garnett (Eds.).

  BibTeX: @inproceedings{NIPS2018_8277, author = {Zhao, Shengjia and Ren, Hongyu and Yuan, Arianna and Song, Jiaming and Goodman, Noah and Ermon, Stefano}, booktitle = {Advances in Neural Information Processing Systems 31}, date-modified = {2019-03-30 17:02:33 +0000}, editor = {Bengio, S. and Wallach, H. and Larochelle, H. and Grauman, K. and Cesa-Bianchi, N. and Garnett, R.}, pages = {10814--10823}, title = {Bias and Generalization in Deep Generative Models: An Empirical Study}, website = {https://papers.nips.cc/paper/8277-bias-and-generalization-in-deep-generative-models-an-empirical-study.pdf}, year = {2018} }

• Multimodal Generative Models for Scalable Weakly-Supervised Learning. [PDF] [BibTeX]
  M. Wu, & N. Goodman. (2018). In Advances in Neural Information Processing Systems 31 , S. Bengio, H. Wallach, H. Larochelle, K. Grauman, N. Cesa-Bianchi, & R. Garnett (Eds.).

  BibTeX: @inproceedings{NIPS2018_7801, author = {Wu, Mike and Goodman, Noah}, booktitle = {Advances in Neural Information Processing Systems 31}, date-modified = {2019-03-30 17:02:10 +0000}, editor = {Bengio, S. and Wallach, H. and Larochelle, H. and Grauman, K. and Cesa-Bianchi, N. and Garnett, R.}, pages = {5576--5586}, title = {Multimodal Generative Models for Scalable Weakly-Supervised Learning}, website = {https://papers.nips.cc/paper/7801-multimodal-generative-models-for-scalable-weakly-supervised-learning.pdf}, year = {2018} }

• Pragmatically Informative Image Captioning with Character-Level Reference. [PDF] [BibTeX]
  R. Cohn-Gordon, N. D. Goodman, & C. Potts. (2018). In NAACL-HLT.

  BibTeX: @inproceedings{CohnGordon2018PragmaticallyII, author = {Cohn-Gordon, Reuben and Goodman, Noah D. and Potts, Christopher}, booktitle = {NAACL-HLT}, date-added = {2019-09-06 15:42:04 +0000}, date-modified = {2019-09-06 15:42:04 +0000}, title = {Pragmatically Informative Image Captioning with Character-Level Reference}, website = {https://arxiv.org/abs/1804.05417}, year = {2018} }

Preprints

• Joint Mapping and Calibration via Differentiable Sensor Fusion. [PDF] [BibTeX]
  J. P. Chen, F. Obermeyer, V. Lyapunov, L. Gueguen, & N. D. Goodman. 2018-Preprint arXiv preprint arXiv:1812.00880.

  BibTeX: @techreport{chen2018joint, author = {Chen, Jonathan P and Obermeyer, Fritz and Lyapunov, Vladimir and Gueguen, Lionel and Goodman, Noah D}, date-modified = {2019-03-29 15:58:22 +0000}, journal = {arXiv preprint arXiv:1812.00880}, title = {Joint Mapping and Calibration via Differentiable Sensor Fusion}, website = {https://arxiv.org/pdf/1812.00880.pdf}, year = {2018-Preprint} }

2017

• Subjectivity predicts adjective ordering preferences. [Abstract] [PDF] [BibTeX]
  G. Scontras, J. Degen, & N. D. Goodman. (2017). Open Mind.

  Abstract: From English to Hungarian to Mokilese, speakers exhibit strong ordering prefer- ences in multi-adjective strings: "the big blue box" sounds far more natural than "the blue big box." We show that an adjective’s distance from the modified noun is pre- dicted not by a rigid syntax, but by the adjective’s meaning: less subjective adjectives occur closer to the nouns they modify. This finding provides an example of a broad linguistic universal—adjective ordering preferences—emerging from general properties of cognition.

  BibTeX: @article{ScontrasEtAl2016-OpenMind, author = {Scontras, Gregory and Degen, Judith and Goodman, Noah D}, date-modified = {2016-09-27 14:43:24 +0000}, journal = {Open Mind}, title = {Subjectivity predicts adjective ordering preferences}, website = {https://alpslab.stanford.edu/papers/2017_ScontrasDegenGoodman.pdf}, year = {2017} }

• Warm (for winter): Comparison class understanding in vague language. [Abstract] [PDF] [BibTeX]
  M. H. Tessler, M. Lopez-Brau, & N. D. Goodman. (2017). In Proceedings of the Thirty-Ninth Annual Conference of the Cognitive Science Society.

  Abstract: Speakers often refer to context only implicitly when using language. The utterance "it’s warm outside" could mean it’s warm relative to other days of the year or just relative to the current season (e.g., it’s warm for winter). Warm vaguely conveys that the temperature is high relative to some contextual comparison class, but little is known about how a listener decides upon such a standard of comparison. Here, we propose that the resolution of a comparison class in context is a pragmatic inference driven by world knowledge and listeners’ internal models of speech production. We introduce a Rational Speech Act model and derive two novel predictions from it, which we validate using a paraphrase experiment to measure listeners’ beliefs about the likely comparison class used by a speaker. Our model makes quantitative predictions given prior world knowledge for the domains in question. We triangulate this knowledge by a second language task in the same domains, using Bayesian data analysis to infer priors from both data sets.

  BibTeX: @inproceedings{tessler2017cogsci, author = {Tessler, Michael Henry and Lopez-Brau, Michael and Goodman, Noah D.}, booktitle = {Proceedings of the Thirty-Ninth Annual Conference of the Cognitive Science Society}, date-added = {2017-02-22 13:01:38 +0000}, date-modified = {2017-02-22 13:03:04 +0000}, title = {Warm (for winter): Comparison class understanding in vague language}, website = {https://iccm-conference.neocities.org/2017/ICCMprogram_files/paper_13.pdf}, year = {2017} }

• "I won’t lie, it wasn’t amazing": Modeling polite indirect speech. [Abstract] [PDF] [BibTeX]
  E. J. Yoon, M. H. Tessler, N. D. Goodman, & M. C. Frank. (2017). In Proceedings of the Thirty-Ninth Annual Conference of the Cognitive Science Society.

  Abstract: Why are we polite when we talk to one another? One hypothesis is that people expect others to choose what to say based on their goals both to transfer information efficiently (an epistemic goal) and to make the listener feel good (a social goal). In our previous work, we found that when these two goals conflict, they sometimes produce white lies. In the current work, we expand on this theory to consider another prominent case of polite speech: indirect remarks using negation (e.g., "It wasn’t amazing"). With minimal extensions from our previous framework, our formal model suggests that a pragmatic speaker will produce more indirect remarks when the speaker wants to be informative and seem considerate at the same time. These predictions were borne out in an experiment on language production. These findings suggest that the conflict between social and epistemic goals can account for a broad range of politeness phenomena.

  BibTeX: @inproceedings{yoon2017cogsci, author = {Yoon, Erica J. and Tessler, Michael Henry and Goodman, Noah D. and Frank, Michael C.}, booktitle = {Proceedings of the Thirty-Ninth Annual Conference of the Cognitive Science Society}, date-added = {2017-02-22 13:01:38 +0000}, date-modified = {2017-02-22 13:03:04 +0000}, title = {"I won't lie, it wasn't amazing": Modeling polite indirect speech}, website = {https://langcog.stanford.edu/papers_new/yoon-2017-cogsci.pdf}, year = {2017} }

• Amortized Hypothesis Generation. [Abstract] [PDF] [BibTeX]
  I. Dasgupta, E. Schulz, N. Goodman, & S. Gershman. (2017). In Proceedings of the Thirty-Ninth Annual Conference of the Cognitive Science Society.

  Abstract: Bayesian models of cognition posit that people compute probability distributions over hypotheses, possibly by constructing a sample-based approximation. Since people encounter many closely related distributions, a computationally efficient strategy is to selectively reuse computations – either the samples themselves or some summary statistic. We refer to these reuse strategies as amortized inference. In two experiments, we present evidence consistent with amortization. When sequentially answering two related queries about natural scenes, we show that answers to the second query vary systematically depending on the structure of the first query. Using a cognitive load manipulation, we find evidence that people cache summary statistics rather than raw sample sets. These results enrich our notions of how the brain approximates probabilistic inference

  BibTeX: @inproceedings{dasgupta2017cogsci, author = {Dasgupta, Ishita and Schulz, Eric and Goodman, Noah and Gershman, Samuel}, booktitle = {Proceedings of the Thirty-Ninth Annual Conference of the Cognitive Science Society}, date-added = {2017-02-22 13:01:38 +0000}, date-modified = {2017-02-22 13:03:04 +0000}, title = {Amortized Hypothesis Generation}, website = {papers/Dasgupta2017Cogsci.pdf}, year = {2017} }

• Convention-formation in iterated reference games. [Abstract] [PDF] [BibTeX]
  R. X. D. Hawkins, M. C. Frank, & N. D. Goodman. (2017). In Proceedings of the Thirty-Ninth Annual Conference of the Cognitive Science Society.

  Abstract: What cognitive mechanisms support the emergence of linguistic conventions from repeated interaction? We present results from a large-scale, multi-player replication of the classic tangrams task which demonstrate three key empirical signatures constraining theories of convention-formation: arbitrariness, stability, and reduction of utterance length over time. These results motivate a theory of convention-formation where agents, though initially uncertain about word meanings in context, assume others are using language with such knowledge. Thus, agents may learn about meanings by reasoning about a knowledgeable, informative partner; if all agents engage in such a process, they successfully coordinate their beliefs, giving rise to a conventional communication system. We formalize this theory in a computational model of language understanding as social inference and demonstrate that it produces all three signatures.

  BibTeX: @inproceedings{hawkins2017cogsci, author = {Hawkins, Robert X. D. and Frank, Michael C. and Goodman, Noah D.}, booktitle = {Proceedings of the Thirty-Ninth Annual Conference of the Cognitive Science Society}, date-added = {2017-02-22 13:01:38 +0000}, date-modified = {2017-02-22 13:03:04 +0000}, title = {Convention-formation in iterated reference games}, website = {papers/HawkinsGoodman2017-Cogsci.pdf}, year = {2017} }

• Avoiding frostbite: It helps to learn from others. Commentary on B. Lake et al., Building machines that learn and think like people.. [PDF] [BibTeX]
  M. H. Tessler, N. D. Goodman, & M. C. Frank. (2017). Behavioral and Brain Sciences. doi:10.1017/S0140525X17000280

  BibTeX: @article{tessler_goodman_frank_2017, author = {Tessler, Michael Henry and Goodman, Noah D. and Frank, Michael C.}, doi = {10.1017/S0140525X17000280}, journal = {Behavioral and Brain Sciences}, pages = {e279}, publisher = {Cambridge University Press}, title = {Avoiding frostbite: It helps to learn from others. Commentary on B. Lake et al., Building machines that learn and think like people.}, volume = {40}, website = {papers/TesslerEtAl2017_commentary.pdf}, year = {2017}, bdsk-url-1 = {https://doi.org/10.1017/S0140525X17000280} }

• Colors in Context: A Pragmatic Neural Model for Grounded Language Understanding. [Abstract] [PDF] [BibTeX]
  W. Monroe, R. X. D. Hawkins, N. D. Goodman, & C. Potts. (2017). Transactions of the Association for Computational Linguistics.

  Abstract: We present a model of pragmatic referring expression interpretation in a grounded communication task (identifying colors from descriptions) that draws upon predictions from two recurrent neural network classifiers, a speaker and a listener, unified by a recursive pragmatic reasoning framework. Experiments show that this combined pragmatic model interprets color descriptions more accurately than the classifiers from which it is built. We observe that pragmatic reasoning helps primarily in the hardest cases: when the model must distinguish very similar colors, or when few utterances adequately express the target color. Our findings make use of a newly-collected corpus of human utterances in color reference games, which exhibit a variety of pragmatic behaviors. We also show that the embedded speaker model reproduces many of these pragmatic behaviors.

  BibTeX: @article{MonroeHawkinsEtAl2017-TACL, author = {Monroe, Will and Hawkins, Robert X. D. and Goodman, N. D. and Potts, Christopher}, date-added = {2017-03-21 16:19:12 +0000}, date-modified = {2017-11-21 17:36:14 +0000}, journal = {Transactions of the Association for Computational Linguistics}, number = {1}, pages = {325--338}, title = {Colors in Context: A Pragmatic Neural Model for Grounded Language Understanding}, volume = {5}, website = {https://aclanthology.org/Q17-1023/}, year = {2017} }

• Empirical evidence for resource-rational anchoring and adjustment. [Abstract] [PDF] [BibTeX]
  F. Lieder, T. L. Griffiths, Q. J. M. Huys, & N. D. Goodman. (2017). Psychonomic Bulletin and Review.

  Abstract: People’s estimates of numerical quantities are systematically biased towards their initial guess. This anchoring bias is usually interpreted as sign of human irrationality, but it has recently been suggested that the anchoring bias instead results from people’s rational use of their finite time and limited cognitive resources. If this were true, then adjustment should decrease with the relative cost of time. To test this hypothesis, we designed a new numerical estimation paradigm that controls people’s knowledge and varies the cost of time and error independently while allowing people to invest as much or as little time and effort into refining their estimate as they wish. Two experiments confirmed the prediction that adjustment decreases with time cost but increases with error cost regardless of whether the anchor was self-generated or provided. These results support the hypothesis that people rationally adapt their number of adjustments to achieve a near-optimal speed-accuracy tradeoff. This suggests that the anchoring bias might be a signature of the rational use of finite time and limited cognitive resources rather than a sign of human irrationality.

  BibTeX: @article{FalkEtAl2017b, author = {Lieder, Falk and Griffiths, Thomas L. and Huys, Quentin J.M. and Goodman, Noah D.}, journal = {Psychonomic Bulletin and Review}, title = {Empirical evidence for resource-rational anchoring and adjustment}, website = {https://www.researchgate.net/publication/315747243_Empirical_evidence_for_resource-rational_anchoring_and_adjustment}, year = {2017} }

• The anchoring bias reflects rational use of cognitive resources. [Abstract] [PDF] [BibTeX]
  F. Lieder, T. L. Griffiths, Q. J. M. Huys, & N. D. Goodman. (2017). Psychonomic Bulletin and Review.

  Abstract: Cognitive biases, such as the anchoring bias, pose a serious challenge to rational accounts of human cognition. We investigate whether rational theories can meet this challenge by taking into account the mind’s bounded cognitive resources. We asked what reasoning under uncertainty would look like if people made rational use of their finite time and limited cognitive resources. To answer this question, we applied a mathematical theory of bounded rationality to the problem of numerical estimation. Our analysis led to a rational process model that can be interpreted in terms of anchoring-and-adjustment. This model provided a unifying explanation for ten anchoring phenomena including the differential effect of accuracy motivation on the bias towards provided versus self-generated anchors. Our results illustrate the potential of resource-rational analysis to provide formal theories that can unify a wide range of empirical results and reconcile the impressive capacities of the human mind with its apparently irrational cognitive biases.

  BibTeX: @article{FalkEtAl2017a, author = {Lieder, Falk and Griffiths, Thomas L. and Huys, Quentin J.M. and Goodman, Noah D.}, journal = {Psychonomic Bulletin and Review}, title = {The anchoring bias reflects rational use of cognitive resources}, website = {https://www.researchgate.net/publication/315710809_The_anchoring_bias_reflects_rational_use_of_cognitive_resources}, year = {2017} }

• Eye-tracking causality. [Abstract] [PDF] [BibTeX]
  T. Gerstenberg, M. F. Peterson, N. D. Goodman, D. A. Lagnado, & J. B. Tenenbaum. (2017). Psychological Science.

  Abstract: How do people make causal judgments? What role, if any, does counterfactual simulation play? Counterfactual theories of causal judgments predict that people compare what actually happened with what would have happened if the candidate cause had been absent. Process theories predict that people focus only on what actually happened, to assess the mechanism linking candidate cause and outcome. We tracked participants’ eye-movements while they judged whether one billiard ball caused another one to go through a gate, or prevented it from going through. Both participants’ looking patterns and judgments demonstrate that counterfactual simulation is critical. Participants simulate where the target ball would have gone if the candidate cause had been removed from the scene. The more certain participants were that the outcome would have been different, the stronger their causal judgments. These results provide the first direct evidence for spontaneous counterfactual simulation in an important domain of high-level cognition.

  BibTeX: @article{Gerstenberg2017, author = {Gerstenberg, Tobias and Peterson, Matthew F. and Goodman, Noah D. and Lagnado, David A. and Tenenbaum, Joshua B.}, date-modified = {2017-11-21 17:34:54 +0000}, journal = {Psychological Science}, number = {12}, pages = {1731--1744}, title = {Eye-tracking causality}, volume = {28}, website = {papers/GerstenbergEtAl17_PsychScience.pdf}, year = {2017} }

• Resolving uncertainty in plural predication. [Abstract] [PDF] [BibTeX]
  G. Scontras, & N. D. Goodman. (2017). Cognition.

  Abstract: Plural predications (e.g., “the boxes are heavy”) are common sources of ambiguity in everyday language, allowing both distributive and collective interpretations (e.g., the boxes each are heavy vs. the boxes together are heavy). This paper investigates the role of context in the disambiguation of plural predication. We address the key phenomenon of “stubborn distributivity,” whereby certain predicates (e.g., big, tall) are claimed to lack collective interpretations altogether. We first validate a new methodology for measuring the interpretation of plural predications. Using this method, we then analyze naturally-occurring plural predications from corpora. We find a role of context, but no evidence of a distinct class of predicates that resists collective interpretations. We further explore the role of context in our final experiments, showing that both the predictability of properties and the knowledgeability of the speaker affect disambiguation. This suggests a pragmatic account of how ambiguous plural predications are interpreted. In particular, stubbornly distributive predicates are so because the collective properties they name are unpredictable, or unstable, in most contexts; this unpredictability results in a noisy collective interpretation, something speakers and listeners recognize as ineffective for communicating efficiently about their world. We formalize the pragmatics of utterance disambiguation within the Bayesian Rational Speech Act framework.

  BibTeX: @article{Scontras2017_Cubert, author = {Scontras, Gregory and Goodman, Noah D.}, date-added = {2017-06-26 17:32:13 +0000}, date-modified = {2017-11-21 17:29:02 +0000}, journal = {Cognition}, pages = {294 - 311}, title = {Resolving uncertainty in plural predication}, volume = {168}, website = {papers/ScontrasGoodman2017_Cognition.pdf}, year = {2017} }

• So Good It Has to Be True: Wishful Thinking in Theory of Mind. [Abstract] [PDF] [BibTeX]
  D. Hawthorne-Madell, & N. D. Goodman. (2017). Open Mind.

  Abstract: In standard decision theory, rational agents are objective, keeping their beliefs independent from their desires. Such agents are the basis for current computational models of Theory of Mind (ToM), but the accuracy of these models are unknown. Do people really think that others do not let their desires color their beliefs? In two experiments we test whether people think that others engage in wishful thinking. We find that participants do think others believe that desirable events are more likely to happen, and that undesirable ones are less likely to happen. However, these beliefs are not well calibrated as people do not let their desires influence their beliefs in the task. Whether accurate or not, thinking that others wishfully think has consequences for reasoning about them. We find one such consequence—people learn more from an informant who thinks an event will happen despite wishing it was otherwise. People’s ToM therefore appears to be more nuanced than the current rational accounts in that it allows other’s desires to directly affect their subjective probability of an event.

  BibTeX: @article{Hawthorne2017_ToM, author = {Hawthorne-Madell, Daniel and Goodman, Noah D.}, journal = {Open Mind}, number = {2}, pages = {101-110}, title = {So Good It Has to Be True: Wishful Thinking in Theory of Mind}, volume = {1}, website = {papers/HawthorneGoodman2017-Wishful.pdf}, year = {2017} }

• Learning Disentangled Representations with Semi-Supervised Deep Generative Models. [Abstract] [PDF] [BibTeX]
  N. Siddharth, B. Paige, V. de Meent, A. Desmaison, F. Wood, N. D. Goodman, P. Kohli, & P. H. S. Torr. (2017). In Advances in Neural Information Processing Systems 30.

  Abstract: Variational autoencoders (VAEs) learn representations of data by jointly training a probabilistic encoder and decoder network. Typically these models encode all features of the data into a single variable. Here we are interested in learning disentangled representations that encode distinct aspects of the data into separate variables. We propose to learn such representations using model architectures that generalize from standard VAEs, employing a general graphical model structure in the encoder and decoder. This allows us to train partially-specified models that make relatively strong assumptions about a subset of interpretable variables and rely on the flexibility of neural networks to learn representations for the remaining variables. We further define a general objective for semi-supervised learning in this model class, which can be approximated using an importance sampling procedure. We evaluate our framework’s ability to learn disentangled representations, both by qualitative exploration of its generative capacity, and quantitative evaluation of its discriminative ability on a variety of models and datasets.

  BibTeX: @inproceedings{SiddharthEtAl2017-NIPS, author = {Siddharth, N and Paige, Brooks and de Meent, Van and Desmaison, Alban and Wood, Frank and Goodman, Noah D and Kohli, Pushmeet and Torr, Philip HS}, booktitle = {Advances in Neural Information Processing Systems 30}, title = {Learning Disentangled Representations with Semi-Supervised Deep Generative Models}, website = {https://arxiv.org/pdf/1706.00400.pdf}, year = {2017} }

• Beyond Reward Prediction Errors: Human Striatum Updates Rule Values During Learning. [PDF] [BibTeX]
  I. Ballard, E. Miller, S. Piantadosi, N. Goodman, & S. McClure. (2017). Cerebral Cortex.

  BibTeX: @article{BallardEtAl-Cortex, author = {Ballard, Ian and Miller, Eric and Piantadosi, Steven and Goodman, Noah and McClure, Samuel}, date-added = {2017-06-27 17:59:10 +0000}, date-modified = {2018-11-16 19:22:26 +0000}, journal = {Cerebral Cortex}, title = {Beyond Reward Prediction Errors: Human Striatum Updates Rule Values During Learning}, website = {https://www.biorxiv.org/content/biorxiv/early/2017/03/09/115253.full.pdf}, year = {2017} }

• Why do you ask? The informational dynamics of questions and answers. [Abstract] [PDF] [BibTeX]
  R. X. D. Hawkins, & N. D. Goodman. (2017). Technical report: PsyArXiv:j2cp6.

  Abstract: Asking questions is one of our most efficient and reliable means of learning about the world. Yet we do not often pose these questions to an impartial oracle; we ask cooperative social partners, in dialogue. In this paper, we aim to reconcile formal models of optimal question asking and answering with classic effects of social context. We begin from the observation that question-answer dialogue is motivated by a two-sided asymmetry in beliefs: questioners have a private goal but lack goal-relevant information about the world, and answerers have private information but lack knowledge about the questioner’s goal. We formalize this problem in a computational framework and derive pragmatic questioner and answerer behavior from recursive social reasoning. Critically, we predict that pragmatic answerers go beyond the literal meaning of the question to be informative with respect to inferred goals, and that pragmatic questioners may therefore select questions to more unambiguously signal their goals. We evaluate our pragmatic models against asocial models in two ways. First, we present computational simulations accounting for three classic answerer effects in psycholinguistics. We then introduce the Hidden Goal paradigm for experimentally eliciting questioner and answerer behavior in scenarios where there is uncertainty about the questioner’s goal. We report data from three experiments in this paradigm and show how our core computational framework can be composed with more sophisticated question semantics, hierarchical goal spaces, and a persistent state over which extended dialogue can unfold. We find that social inference is needed to account for critical aspects of the data.

  BibTeX: @techreport{HawkinsGoodman18_QA, author = {{Hawkins}, R.~X.~D. and {Goodman}, N.~D.}, date-modified = {2019-03-29 15:28:29 +0000}, journal = {Technical report: PsyArXiv:j2cp6}, title = {Why do you ask? The informational dynamics of questions and answers}, website = {https://psyarxiv.com/j2cp6}, year = {2017} }

Preprints

2016

• C3: Lightweight Incrementalized MCMC for Probabilistic Programs using Continuations and Callsite Caching. [Abstract] [PDF] [BibTeX]
  D. Ritchie, A. Stuhlmüller, & N. D. Goodman. (2016). In AISTATS 2016.

  Abstract: Lightweight, source-to-source transformation approaches to implementing MCMC for probabilistic programming languages are popular for their simplicity, support of existing deterministic code, and ability to execute on existing fast runtimes. However, they are also slow, requiring a complete re-execution of the program on every Metropolis Hastings proposal. We present a new extension to the lightweight approach, C3, which enables efficient, incrementalized re-execution of MH proposals. C3 is based on two core ideas: transforming probabilistic programs into continuation passing style (CPS), and caching the results of function calls. We show that on several common models, C3 reduces proposal runtime by 20-100x, in some cases reducing runtime complexity from linear in model size to constant. We also demonstrate nearly an order of magnitude speedup on a complex inverse procedural modeling application.

  BibTeX: @inproceedings{ritchie2015c3, author = {Ritchie, Daniel and Stuhlm\"{u}ller, Andreas and Goodman, Noah D.}, booktitle = {AISTATS 2016}, date-modified = {2015-12-22 19:00:18 +0000}, journal = {Technical report: arXiv:1509.02151}, title = {C3: Lightweight Incrementalized MCMC for Probabilistic Programs using Continuations and Callsite Caching}, website = {https://arxiv.org/abs/1509.02151}, year = {2016} }

• The logical primitives of thought: Empirical foundations for compositional cognitive models. [Abstract] [PDF] [BibTeX]
  S. T. Piantadosi, J. B. Tenenbaum, & N. D. Goodman. (2016). Psychological Review.

  Abstract: The notion of a compositional language of thought (LOT) has been central in computational accounts of cognition from earliest attempts (Boole, 1854; Fodor, 1975) to the present day (Feldman, 2000; Penn, Holyoak, & Povinelli, 2008; Fodor, 2008; Kemp, 2012; Goodman, Tenenbaum, & Gerstenberg, 2015). Recent modeling work shows how statistical inferences over compositionally structured hypothesis spaces might explain learning and development across a variety of domains. However, the primitive components of such representations are typically assumed a priori by modelers and theoreticians rather than determined empirically. We show how different sets of LOT primitives, embedded in a psychologically realistic approximate Bayesian inference framework, systematically predict distinct learning curves in rule-based concept learning experiments. We use this feature of LOT models to design a set of large-scale concept learning experiments that can determine the most likely primitives for psychological concepts involving Boolean connectives and quantification. Subjects’ inferences are most consistent with a rich (nonminimal) set of Boolean operations, including first-order, but not second-order, quantification. Our results more generally show how specific LOT theories can be distinguished empirically.

  BibTeX: @article{Piantadosi2015, author = {Piantadosi, Steven T and Tenenbaum, Joshua B and Goodman, Noah D}, date-added = {2015-10-21 15:05:27 +0000}, date-modified = {2016-8-17 14:55:10 +0000}, journal = {Psychological Review}, number = {4}, pages = {392-424}, title = {The logical primitives of thought: Empirical foundations for compositional cognitive models}, volume = {123}, website = {https://psycnet.apa.org/fulltext/2016-18127-001.pdf}, year = {2016} }

• Pragmatic Reasoning through Semantic Inference. [Abstract] [PDF] [BibTeX]
  L. Bergen, R. Levy, & N. D. Goodman. (2016). Semantics and Pragmatics.

  Abstract: A number of recent proposals have used techniques from game theory and Bayesian cognitive science to formalize Gricean pragmatic reasoning (Frank & Goodman, 2012; Franke, 2009; Goodman & Stuhlmüller, 2013; Jaeger, 2012). We discuss two phenomena which pose a challenge to these accounts of pragmatics: M-implicatures (Horn, 1984) and embedded implicatures which violate Hurford’s constraint (Chierchia, Fox & Spector, 2012; Hurford, 1974). Previous models cannot derive these implicatures, because of basic limitations in their architecture. In order to explain these phenomena, we propose a realignment of the division between semantic content and pragmatic content. Under this proposal, the semantic content of an utterance is not fixed independent of pragmatic inference; rather, pragmatic inference partially determines an utterance’s semantic content. We show how semantic inference can be realized as an extension to the Rational Speech Acts framework (Goodman & Stuhlmüller, 2013). The addition of lexical uncertainty derives both M-implicatures and the relevant embedded implicatures, and preserves the derivations of more standard implicatures.

  BibTeX: @article{BergenLevyGoodman2015, author = {Bergen, Leon and Levy, Roger and Goodman, Noah D}, date-modified = {2017-03-21 15:30:19 +0000}, journal = {Semantics and Pragmatics}, title = {Pragmatic Reasoning through Semantic Inference}, volume = {9}, website = {https://semprag.org/article/view/sp.9.20/pdf}, year = {2016} }

• Pragmatic language interpretation as probabilistic inference. [Abstract] [PDF] [BibTeX]
  N. D. Goodman, & M. C. Frank. (2016). Trends in Cognitive Sciences.

  Abstract: Understanding language is more than the use of fixed conventions and more than decoding combinatorial structure. Instead, comprehenders make exquisitely sensitive inferences about what utterances mean given their knowledge of the speaker, the language, and the context. Building on developments in game theory and probabilistic modeling, we describe the rational speech act (RSA) framework for pragmatic reasoning. RSA models provide a principled way to formalize inferences about meaning in context; they have been used to make successful quantitative predictions about human behavior in a wide variety of different tasks and situations, and they explain why complex phenomena like hyperbole and vagueness occur. More generally, they provide a computational framework for integrating linguistic structure, world knowledge, and context in pragmatic language understanding.

  BibTeX: @article{GoodmanFrank2016-TICS, author = {Goodman, Noah D and Frank, Michael C}, date-added = {2016-8-17 15:05:27 +0000}, date-modified = {2016-8-17 14:55:10 +0000}, journal = {Trends in Cognitive Sciences}, number = {11}, pages = {818-829}, title = {Pragmatic language interpretation as probabilistic inference}, volume = {20}, website = {papers/GoodmanFrank2016-TICS.pdf}, year = {2016} }

• Learning to Generate Compositional Color Descriptions. [Abstract] [PDF] [BibTeX]
  W. Monroe, N. D. Goodman, & C. Potts. (2016). In Proceedings of the 2016 Conference on Empirical Methods on Natural Language Processing (EMNLP 2016).

  Abstract: The production of color language is essential for grounded language generation. Color descriptions have many challenging properties: they can be vague, compositionally complex, and denotationally rich. We present an effective approach to generating color descriptions using recurrent neural networks and a Fourier-transformed color representation. Our model outperforms previous work on a conditional language modeling task over a large corpus of naturalistic color descriptions. In addition, probing the model’s output reveals that it can accurately produce not only basic color terms but also descriptors with non-convex denotations ("greenish"), bare modifiers ("bright", "dull"), and compositional phrases ("faded teal") not seen in training

  BibTeX: @inproceedings{MonroeEtAl2016-EMNLP, author = {Monroe, Will and Goodman, Noah D and Potts, Chris}, booktitle = {Proceedings of the 2016 Conference on Empirical Methods on Natural Language Processing (EMNLP 2016)}, title = {Learning to Generate Compositional Color Descriptions}, website = {https://arxiv.org/abs/1606.03821}, year = {2016} }

• Neurally-Guided Procedural Models: Amortized Inference for Procedural Graphics Programs using Neural Networks. [Abstract] [PDF] [BibTeX]
  D. Ritchie, A. Thomas, P. Hanrahan, & N. D. Goodman. (2016). In Advances in Neural Information Processing Systems (NIPS 2016).

  Abstract: Probabilistic inference algorithms such as Sequential Monte Carlo (SMC) provide powerful tools for constraining procedural models in computer graphics, but they require many samples to produce desirable results. In this paper, we show how to create procedural models which learn how to satisfy constraints. We augment procedural models with neural networks which control how the model makes random choices based on the output it has generated thus far. We call such models neurally-guided procedural models. As a pre-computation, we train these models to maximize the likelihood of example outputs generated via SMC. They are then used as efficient SMC importance samplers, generating high-quality results with very few samples. We evaluate our method on L-system-like models with image-based constraints. Given a desired quality threshold, neurally-guided models can generate satisfactory results up to 10x faster than unguided models.

  BibTeX: @inproceedings{RitchieEtAl2016-NIPS, author = {Ritchie, Daniel and Thomas, Anna and Hanrahan, Pat and Goodman, Noah D.}, booktitle = {Advances in Neural Information Processing Systems (NIPS 2016)}, title = {Neurally-Guided Procedural Models: Amortized Inference for Procedural Graphics Programs using Neural Networks}, website = {https://dritchie.github.io/pdf/ngpm.pdf}, year = {2016} }

• Conversational expectations account for apparent limits on theory of mind use. [Abstract] [PDF] [BibTeX]
  R. X. D. Hawkins, & N. D. Goodman. (2016). In Proceedings of the Thirty-Eighth Annual Conference of the Cognitive Science Society. (2016)

  Abstract: Theory of mind is a powerful cognitive ability: by the age of six, people are capable of accurately reasoning about others’ beliefs and desires. An influential series of language understanding experiments by Keysar and colleagues, however, showed that adults systematically failed to take a speaker’s beliefs into account, revealing limitations on theory of mind. In this paper we argue that these apparent failures are in fact successes. Through a minimal pair of replications comparing scripted vs. unscripted speakers, we show that critical utterances used by Keysar and colleagues are uncooperative: they are less informative than what a speaker would actually produce in that situation. When we allow participants to naturally interact, we find that listener expectations are justified and errors are reduced. This ironically shows that apparent failures of theory of mind are in fact attributable to sophisticated expectations about speaker behavior—that is, to theory of mind.

  BibTeX: @inproceedings{hawkins2016cogsci, annote = {(2016)}, author = {Hawkins, Robert X.D. and Goodman, Noah D.}, booktitle = {Proceedings of the Thirty-Eighth Annual Conference of the Cognitive Science Society}, title = {Conversational expectations account for apparent limits on theory of mind use}, website = {papers/HawkinsGoodman2016-Cogsci.pdf}, year = {2016} }

• Communicating generalizations about events. [Abstract] [PDF] [BibTeX]
  M. H. Tessler, & N. D. Goodman. (2016). In Proceedings of the Thirty-Eighth Annual Conference of the Cognitive Science Society. (2016)

  Abstract: Habitual sentences (e.g. Bill smokes.) generalize an event over time, but how do you know when a habitual sentence is true? We develop a computational model and use this to guide experiments into the truth conditions of habitual language. In Expts. 1 & 2, we measure participants’ prior expectations about the frequency with which an event occurs and validate the predictions of the model for when a habitual sentence is acceptable. In Expt. 3, we show that habituals are sensitive to top-down moderators of expected frequency: It is the expectation of future tendency that matters for habitual language. This work provides the mathematical glue between our intuitive theories’ of others and events and the language we use to talk about them.

  BibTeX: @inproceedings{tessler2016cogsci, annote = {(2016)}, author = {Tessler, Michael Henry and Goodman, Noah D}, booktitle = {Proceedings of the Thirty-Eighth Annual Conference of the Cognitive Science Society}, title = {Communicating generalizations about events}, website = {https://cseweb.ucsd.edu//%7egary/cs200/w17/Tessler2016-cogsci.pdf}, year = {2016} }

• Talking with tact: Polite language as a balance between kindness and informativity. [Abstract] [PDF] [BibTeX]
  E. J. Yoon, M. H. Tessler, N. D. Goodman, & M. C. Frank. (2016). In Proceedings of the Thirty-Eighth Annual Conference of the Cognitive Science Society. (2016)

  Abstract: Conveying information in a false or indirect manner in consideration of listeners’ wants (i.e. being polite) seemingly contradicts an important goal of a cooperative speaker: information transfer. We propose that a cooperative speaker considers both epistemic utility, or utility of providing the listener new and accurate information, and social utility, or utility of maintaining or boosting the listener’s self-image (being polite). We formalize this tradeoff within a probabilistic model of language understanding and test it with empirical data on people’s inferences about the relation between a speaker’s goals, utterances and the true states of the world.

  BibTeX: @inproceedings{yoon2016cogsci, annote = {(2016)}, author = {Yoon, Erica J and Tessler, Michael Henry and Goodman, Noah D and Frank, Michael C}, booktitle = {Proceedings of the Thirty-Eighth Annual Conference of the Cognitive Science Society}, title = {Talking with tact: Polite language as a balance between kindness and informativity}, website = {http://langcog.stanford.edu/papers_new/yoon-2016-cogsci.pdf}, year = {2016} }

• Animal, dog, or dalmatian? Level of abstraction in nominal referring expressions. [Abstract] [PDF] [BibTeX]
  C. Graf, J. Degen, R. X. D. Hawkins, & N. D. Goodman. (2016). In Proceedings of the Thirty-Eighth Annual Conference of the Cognitive Science Society. (2016)

  Abstract: Nominal reference is very flexible—the same object may be called a dalmatian, a dog, or an animal when all are literally true. What accounts for the choices that speakers make in how they refer to objects? The addition of modifiers (e.g. big dog) has been extensively explored in the literature, but fewer studies have explored the choice of noun, including its level of abstraction. We collected freely produced referring expressions in a multi-player reference game experiment, where we manipulated the object’s context. We find that utterance choice is affected by the contextual informativeness of a description, its length and frequency, and the typicality of the object for that description. Finally, we show how these factors naturally enter into a formal model of production within the Rational Speech-Acts framework, and that the resulting model predicts our quantitative production data

  BibTeX: @inproceedings{graf2016cogsci, annote = {(2016)}, author = {Graf, Caroline and Degen, Judith and Hawkins, Robert X.D. and Goodman, Noah D.}, booktitle = {Proceedings of the Thirty-Eighth Annual Conference of the Cognitive Science Society}, title = {Animal, dog, or dalmatian? Level of abstraction in nominal referring expressions}, website = {papers/GrafEtAl2016-Cogsci.pdf}, year = {2016} }

• Emotions in lay explanations of behavior. [Abstract] [PDF] [BibTeX]
  D. C. Ong, J. Zaki, & N. D. Goodman. (2016). In Proceedings of the Thirty-Eighth Annual Conference of the Cognitive Science Society. (2016)

  Abstract: Humans use rich intuitive theories to explain other people’s behavior. Previous work in lay psychology of behavior have tended to treat emotion as causing primarily unintentional behavior (e.g., being sad causes one to cry), neglecting how people incorporate emotions into explanations of rational, intentional actions. Here, we provide preliminary explorations into integrating emotions into a theory of folk psychology. Specifically, we show that in the lay theory, people are willing to endorse emotions as causes of intentional actions. Moreover, people readily attribute beliefs and desires as explanations for emotional expressions. This work provides a first step in elaborating people’s rich understanding of emotions as an important component of intuitive social cognition.

  BibTeX: @inproceedings{ong2016cogsci, annote = {(2016)}, author = {Ong, Desmond C. and Zaki, Jamil and Goodman, Noah D.}, booktitle = {Proceedings of the Thirty-Eighth Annual Conference of the Cognitive Science Society}, title = {Emotions in lay explanations of behavior}, website = {papers/OngZakiGoodman2016-Cogsci.pdf}, year = {2016} }

• What does the crowd believe? A hierarchical approach to estimating subjective beliefs from empirical data. [Abstract] [PDF] [BibTeX]
  M. Franke, F. Dablander, A. Schöller, E. Bennett, J. Degen, M. H. Tessler, J. Kao, & N. D. Goodman. (2016). In Proceedings of the Thirty-Eighth Annual Conference of the Cognitive Science Society. (2016)

  Abstract: People’s beliefs about everyday events are both of theoretical interest in their own right and an important ingredient in model building—especially in Bayesian cognitive models of phenomena such as logical reasoning, future predictions, and language use. Here, we explore several recently used methods for measuring subjective beliefs about unidimensional contiguous properties, such as the likely price of a new watch. We use a hierarchical Bayesian data-analysis model for inferring likely population-level beliefs as the central tendency of participants’ individual-level beliefs. Three different dependent measures are used to infer latent beliefs: (i) slider ratings of (relative) likelihood of intervals of values, (ii) a give-a-number task, and (iii) choice of the more likely of two intervals of values. Our results suggest that using averaged normalized slider ratings for binned quantities is a practical and fairly good approximator of inferred population-level beliefs.

  BibTeX: @inproceedings{franke2016cogsci, annote = {(2016)}, author = {Franke, Michael and Dablander, Fabian and Sch{\"o}ller, Anthea and Bennett, Erin and Degen, Judith and Tessler, Michael Henry and Kao, Justine and Goodman, Noah D.}, booktitle = {Proceedings of the Thirty-Eighth Annual Conference of the Cognitive Science Society}, title = {What does the crowd believe? A hierarchical approach to estimating subjective beliefs from empirical data}, website = {https://www.sfs.uni-tuebingen.de/%7emfranke/Papers/FrankeDablander_2016_What_does_the_crowd_believe.pdf}, year = {2016} }

• The Pragmatics of Spatial Language. [Abstract] [PDF] [BibTeX]
  T. D. Ullman, Y. Xu, & N. D. Goodman. (2016). In Proceedings of the Thirty-Eighth Annual Conference of the Cognitive Science Society. (2016)

  Abstract: How do people understand the pragmatics of spatial language? We propose a rational-speech act model for spatial reasoning, and apply it to the terms ’in’ and ’near’. We examine people’s fine-grain spatial reasoning in this domain by having them locate where an event occurred, given an utterance. Our pragmatic listener model provides a quantitative and qualitative fit to people’s inferences

  BibTeX: @inproceedings{ullman2016cogsci, annote = {(2016)}, author = {Ullman, Tomer D. and Xu, Yang and Goodman, Noah D.}, booktitle = {Proceedings of the Thirty-Eighth Annual Conference of the Cognitive Science Society}, date-added = {2016-04-15 12:58:18 +0000}, date-modified = {2016-04-15 13:02:57 +0000}, title = {The Pragmatics of Spatial Language}, website = {https://www.mit.edu/%7etomeru/papers/spatialImplicature.pdf}, year = {2016} }

• A rational speech-act model of projective content. [Abstract] [PDF] [BibTeX]
  C. Qing, N. D. Goodman, & D. Lassiter. (2016). In Proceedings of the Thirty-Eighth Annual Conference of the Cognitive Science Society. (2016)

  Abstract: Certain content of a linguistic construction can project when the construction is embedded in entailment-canceling environ- ments. For example, the conclusion that John smoked in the past from the utterance John stopped smoking still holds for John didn’t stop smoking, in which the original utterance is embedded under negation. There are two main approaches to account for projection phenomena. The semantic approach adds restrictions of the common ground to the conventional meaning. The pragmatic approach tries to derive projection from general conversational principles. In this paper we build a probabilistic model of language understanding in which the listener jointly infers the world state and what common ground the speaker has assumed. We take change-of-state verbs as an example and model its projective content under negation. Under certain assumptions, the model predicts the projective behavior and its interaction with the question under discussion (QUD), without any special semantic treatment of projective content.

  BibTeX: @inproceedings{qing2016cogsci, annote = {(2016)}, author = {Qing, Ciyang and Goodman, Noah D. and Lassiter, Daniel}, booktitle = {Proceedings of the Thirty-Eighth Annual Conference of the Cognitive Science Society}, date-added = {2016-04-15 13:01:38 +0000}, date-modified = {2016-04-15 13:03:04 +0000}, title = {A rational speech-act model of projective content}, website = {papers/QingGoodmanLassiter2016-Cogsci.pdf}, year = {2016} }

• Learning the Preferences of Ignorant, Inconsistent Agents. [Abstract] [PDF] [BibTeX]
  O. Evans, A. Stuhlmüller, & N. D. Goodman. (2016). In Proceedings of the 30th AAAI Conference on Artificial Intelligence (AAAI-2016).

  Abstract: An important use of machine learning is to learn what people value. What posts or photos should a user be shown? Which jobs or activities would a person find rewarding? In each case, observations of people’s past choices can inform our inferences about their likes and preferences. If we assume that choices are approximately optimal according to some utility function, we can treat preference inference as Bayesian inverse planning. That is, given a prior on utility functions and some observed choices, we invert an optimal decision-making process to infer a posterior distribution on utility functions. However, people often deviate from approximate optimality. They have false beliefs, their planning is sub-optimal, and their choices may be temporally inconsistent due to hyperbolic discounting and other biases. We demonstrate how to incorporate these deviations into algorithms for preference inference by constructing generative models of planning for agents who are subject to false beliefs and time inconsistency. We explore the inferences these models make about preferences, beliefs, and biases. We present a behavioral experiment in which human subjects perform preference inference given the same observations of choices as our model. Results show that human subjects (like our model) explain choices in terms of systematic deviations from optimal behavior and suggest that they take such deviations into account when inferring preferences.

  BibTeX: @inproceedings{evans2016learning, author = {Evans, Owain and Stuhlm\"{u}ller, Andreas and Goodman, Noah D.}, booktitle = {Proceedings of the 30th AAAI Conference on Artificial Intelligence (AAAI-2016)}, date-added = {2015-12-1 12:03:30 +0000}, date-modified = {2015-12-21 15:27:03 +0000}, title = {Learning the Preferences of Ignorant, Inconsistent Agents}, website = {https://stuhlmueller.org/papers/preferences-aaai2016.pdf}, year = {2016} }

• Practical optimal experiment design with probabilistic programs. [Abstract] [PDF] [BibTeX]
  L. Ouyang, M. H. Tessler, D. Ly, & N. D. Goodman. (2016). Technical report: arXiv:1608.05046.

  Abstract: Scientists often run experiments to distinguish competing theories. This requires patience, rigor, and ingenuity - there is often a large space of possible experiments one could run. But we need not comb this space by hand - if we represent our theories as formal models and explicitly declare the space of experiments, we can automate the search for good experiments, looking for those with high expected information gain. Here, we present a general and principled approach to experiment design based on probabilistic programming languages (PPLs). PPLs offer a clean separation between declaring problems and solving them, which means that the scientist can automate experiment design by simply declaring her model and experiment spaces in the PPL without having to worry about the details of calculating information gain. We demonstrate our system in two case studies drawn from cognitive psychology, where we use it to design optimal experiments in the domains of sequence prediction and categorization. We find strong empirical validation that our automatically designed experiments were indeed optimal. We conclude by discussing a number of interesting questions for future research.

  BibTeX: @techreport{OuyangEtAl2016Manuscript, author = {Ouyang, Long and Tessler, Michael Henry and Ly, Daniel and Goodman, Noah D.}, date-modified = {2019-03-29 15:58:39 +0000}, journal = {Technical report: arXiv:1608.05046}, title = {Practical optimal experiment design with probabilistic programs}, website = {https://arxiv.org/abs/1608.05046}, year = {2016} }

• Deep Amortized Inference for Probabilistic Programs. [Abstract] [PDF] [BibTeX]
  D. Ritchie, P. Horsfall, & N. D. Goodman. (2016). Technical report: arXiv:1610.05735.

  Abstract: Probabilistic programming languages (PPLs) are a powerful modeling tool, able to represent any computable probability distribution. Unfortunately, probabilistic program inference is often intractable, and existing PPLs mostly rely on expensive, approximate sampling-based methods. To alleviate this problem, one could try to learn from past inferences, so that future inferences run faster. This strategy is known as amortized inference; it has recently been applied to Bayesian networks and deep generative models. This paper proposes a system for amortized inference in PPLs. In our system, amortization comes in the form of a parameterized guide program. Guide programs have similar structure to the original program, but can have richer data flow, including neural network components. These networks can be optimized so that the guide approximately samples from the posterior distribution defined by the original program. We present a flexible interface for defining guide programs and a stochastic gradient-based scheme for optimizing guide parameters, as well as some preliminary results on automatically deriving guide programs. We explore in detail the common machine learning pattern in which a ‘local’ model is specified by ‘global’ random values and used to generate independent observed data points; this gives rise to amortized local inference supporting global model learning.

  BibTeX: @techreport{ritchie2016DAIPP, author = {Ritchie, Daniel and Horsfall, Paul and Goodman, Noah D.}, journal = {Technical report: arXiv:1610.05735}, title = {{Deep Amortized Inference for Probabilistic Programs}}, website = {https://dritchie.github.io/pdf/daipp.pdf}, year = {2016} }

Preprints

2015

• Coarse-to-Fine Sequential Monte Carlo for Probabilistic Programs. [Abstract] [PDF] [BibTeX]
  A. Stuhlmüller, R. X. D. Hawkins, N. Siddharth, & N. D. Goodman. (2015). Technical report: arXiv:1509.02962.

  Abstract: Many practical techniques for probabilistic inference require a sequence of distributions that interpolate between a tractable distribution and an intractable distribution of interest. Usually, the sequences used are simple, e.g., based on geometric averages between distributions. When models are expressed as probabilistic programs, the models themselves are highly structured objects that can be used to derive annealing sequences that are more sensitive to domain structure. We propose an algorithm for transforming probabilistic programs to coarse-to-fine programs which have the same marginal distribution as the original programs, but generate the data at increasing levels of detail, from coarse to fine. We apply this algorithm to an Ising model, its depth-from-disparity variation, and a factorial hidden Markov model. We show preliminary evidence that the use of coarse-to-fine models can make existing generic inference algorithms more efficient.

  BibTeX: @techreport{stuhlmueller2015coarse, author = {Stuhlm\"{u}ller, Andreas and Hawkins, Robert X. D. and Siddharth, N. and Goodman, Noah D.}, date-modified = {2015-10-21 15:03:41 +0000}, journal = {Technical report: arXiv:1509.02962}, title = {Coarse-to-Fine Sequential Monte Carlo for Probabilistic Programs}, website = {https://arxiv.org/abs/1509.02962}, year = {2015} }

• The Design and Implementation of Probabilistic Programming Languages. [PDF] [BibTeX]
  N. D. Goodman, & A. Stuhlmüller. (2015). (http://dippl.org)

  BibTeX: @book{dippl, annote = {(http://dippl.org)}, author = {Goodman, N. D. and Stuhlm\"uller, A.}, date-modified = {2015-12-21 15:28:06 +0000}, title = {The Design and Implementation of Probabilistic Programming Languages}, website = {http://dippl.org}, year = {2015} }

• Ad-hoc scalar implicature in preschool children. [Abstract] [PDF] [BibTeX]
  A. J. Stiller, N. D. Goodman, & M. C. Frank. (2015). Language Learning and Development.

  Abstract: If a speaker tells us that "some guests were late to the party," we typically infer that not all were. Implicatures, in which an ambiguous statement ("some and possibly all") is strengthened pragmatically (to "some and not all"), are a paradigm case of pragmatic reasoning. Inferences of this sort are difficult for young children, but recent work suggests that this mismatch may stem from issues in understanding the relationship between lexical items like "some" and "all," rather than broader pragmatic deficits. We tested children’s ability to make non-quantificational pragmatic inferences by constructing contextually-derived "ad-hoc" implicatures, using sets of pictures with contrasting features. We found that four-year-olds and some three-year-olds were able to make implicatures successfully using these displays. Hence, apparent failures in scalar implicature are likely due to difficulties specific to the constructions and tasks used in previous work; these difficulties may have masked aspects of children’s underlying pragmatic competence.

  BibTeX: @article{StillerEtAl2014, author = {Stiller, A. J. and Goodman, N. D. and Frank, M. C.}, date-modified = {2015-12-21 20:28:04 +0000}, journal = {Language Learning and Development}, number = {2}, pages = {176--190}, title = {Ad-hoc scalar implicature in preschool children}, volume = {11}, website = {https://langcog.stanford.edu/papers/SGF-lldinpress.pdf}, year = {2015} }

• Relevant and robust. A response to Marcus and Davis. [Abstract] [PDF] [BibTeX]
  N. D. Goodman, M. C. Frank, T. L. Griffiths, J. B. Tenenbaum, P. Battaglia, & J. Hamrick. (2015). Psychological Science.

  Abstract: Computational models in psychology are precise, fully explicit scientific hypotheses. Over the past 15 years, probabilistic modeling of human cognition has yielded quantitative theories of a wide variety of reasoning and learning phenomena. Recently, Marcus and Davis (2013) critique several examples of this work, using these critiques to question the basic validity of the probabilistic approach. Contra the broad rhetoric of their article, the points made by Marcus and Davis – while useful to consider – do not indicate systematic problems with the probabilistic modeling enterprise.

  BibTeX: @article{GoodmanEtAl2015-PsychSci, author = {Goodman, N. D. and Frank, M. C. and Griffiths, T. L. and Tenenbaum, J. B. and Battaglia, P. and Hamrick, J.}, date-modified = {2015-12-21 20:28:40 +0000}, journal = {Psychological Science}, number = {4}, pages = {539--541}, title = {Relevant and robust. A response to Marcus and Davis}, volume = {26}, website = {papers/GoodmanEtAl2015-PsychSci.pdf}, year = {2015} }

• Probabilistic Semantics and Pragmatics: Uncertainty in Language and Thought. [Abstract] [PDF] [BibTeX]
  N. D. Goodman, & D. Lassiter. (2015). In The Handbook of Contemporary Semantic Theory, 2nd Edition , S. Lappin & C. Fox (Eds.).

  Abstract: Language is used to communicate ideas. Ideas are mental tools for coping with a complex and uncertain world. Thus human conceptual structures should be key to language meaning, and probability – the mathematics of uncertainty – should be indispensable for describing both language and thought. Indeed, probabilistic models are enormously useful in modeling human cognition (Tenenbaum et al., 2011) and aspects of natural language (Bod et al., 2003; Chater et al., 2006). With a few early exceptions (e.g. Adams, 1975; Cohen, 1999b), probabilistic tools have only recently been used in natural language semantics and pragmatics. In this chapter we synthesize several of these modeling advances, exploring a formal model of interpretation grounded, via lexical semantics and pragmatic inference, in conceptual structure.

  BibTeX: @inbook{GoodmanLassiter2015-Chapter, author = {Goodman, Noah D. and Lassiter, Daniel}, booktitle = {The Handbook of Contemporary Semantic Theory, 2nd Edition}, editor = {Lappin, Shalom and Fox, Chris}, publisher = {Wiley-Blackwell}, title = {Probabilistic Semantics and Pragmatics: Uncertainty in Language and Thought}, website = {papers/GoodmanLassiter2015-Chapter.pdf}, year = {2015} }

• Concepts in a probabilistic language of thought. [Abstract] [PDF] [BibTeX]
  N. D. Goodman, J. B. Tenenbaum, & T. Gerstenberg. (2015). In The Conceptual Mind: New Directions in the Study of Concepts , Morgolis & Lawrence (Eds.).

  Abstract: Knowledge organizes our understanding of the world, determining what we expect given what we have already seen. Our predictive representations have two key properties: they are productive, and they are graded. Productive generalization is possible because our knowledge decomposes into concepts – elements of knowledge that are combined and recombined to describe particular situations. Gradedness is the observable effect of accounting for uncertainty – our knowledge encodes degrees of belief that lead to graded probabilistic predictions. To put this a different way, concepts form a combinatorial system that enables description of many different situations; each such situation specifies a distribution over what we expect to see in the world, given what we have seen. We may think of this system as a probabilistic language of thought (PLoT) in which representations are built from language-like composition of concepts and the content of those representations is a probability distribution on world states. The purpose of this chapter is to formalize these ideas in computational terms, to illustrate key properties of the PLoT approach with a concrete example, and to draw connections with other views of conceptual structure.

  BibTeX: @inbook{GoodmanEtAl2015-Chapter, author = {Goodman, Noah D and Tenenbaum, Joshua B and Gerstenberg, T}, booktitle = {The Conceptual Mind: New Directions in the Study of Concepts}, editor = {Morgolis and Lawrence}, publisher = {MIT Press}, title = {Concepts in a probabilistic language of thought}, website = {papers/GoodmanEtAl2015-Chapter.pdf}, year = {2015} }

• How many kinds of reasoning? Inference, probability, and natural language semantics. [Abstract] [PDF] [BibTeX]
  D. Lassiter, & N. D. Goodman. (2015). Cognition.

  Abstract: The "new paradigm" unifying deductive and inductive reasoning in a Bayesian framework (Oaksford & Chater, 2007; Over, 2009) has been claimed to be falsified by results which show sharp differences between reasoning about necessity vs. plausibility (Heit & Rotello, 2010; Rips, 2001; Rotello & Heit, 2009). We provide a probabilistic model of reasoning with modal expressions such as "necessary" and "plausible" informed by recent work in formal semantics of natural language, and show that it predicts the possibility of non-linear response patterns which have been claimed to be problematic. Our model also makes a strong monotonicity prediction, while two-dimensional theories predict the possibility of reversals in argument strength depending on the modal word chosen. Predictions were tested using a novel experimental paradigm that replicates the previously-reported response patterns with a minimal manipulation, changing only one word of the stimulus between conditions. We found a spectrum of reasoning "modes" corresponding to different modal words, and strong support for our model’s monotonicity prediction. This indicates that probabilistic approaches to reasoning can account in a clear and parsimonious way for data previously argued to falsify them, as well as new, more fine-grained, data. It also illustrates the importance of careful attention to the semantics of language employed in reasoning experiments.

  BibTeX: @article{LassiterGoodman2015-Cognition, author = {Lassiter, D. and Goodman, N. D.}, date-modified = {2015-12-21 20:30:32 +0000}, journal = {Cognition}, pages = {123--134}, title = {How many kinds of reasoning? Inference, probability, and natural language semantics}, volume = {136}, website = {papers/LassiterGoodman2015-Cognition.pdf}, year = {2015} }

• Generating Design Suggestions under Tight Constraints with Gradient-based Probabilistic Programming. [Abstract] [PDF] [BibTeX]
  D. Ritchie, S. Lin, N. D. Goodman, & P. Hanrahan. (2015). In Proceedings of Eurographics 2015. [Best paper award honorable mention.]

  Abstract: We present a system for generating suggestions from highly-constrained, continuous design spaces. We formulate suggestion as sampling from a probability distribution; constraints are represented as factors that concentrate probability mass around sub-manifolds of the design space. These sampling problems are intractable using typical random walk MCMC techniques, so we adopt Hamiltonian Monte Carlo (HMC), a gradient-based MCMC method. We implement HMC in a high-performance probabilistic programming language, and we evaluate its ability to efficiently generate suggestions for two different, highly-constrained example applications: vector art coloring and designing stable stacking structures.

  BibTeX: @inproceedings{RitchieEtAl2015-Eurographics, annote = {[Best paper award honorable mention.]}, author = {Ritchie, Daniel and Lin, Sharon and Goodman, Noah D. and Hanrahan, Pat}, booktitle = {Proceedings of Eurographics 2015}, title = {{Generating Design Suggestions under Tight Constraints with Gradient-based Probabilistic Programming}}, website = {papers/RitchieEtAl2015-Eurographics.pdf}, year = {2015} }

• The strategic use of noise in pragmatic reasoning. [Abstract] [PDF] [BibTeX]
  L. Bergen, & N. D. Goodman. (2015). Topics in Cognitive Science.

  Abstract: We combine two recent probabilistic approaches to natural language understanding, exploring the formal pragmatics of communication on a noisy channel. We first extend a model of rational communication between a speaker and listener, to allow for the possibility that messages are corrupted by noise. In this model, common knowledge of a noisy channel leads to the use and correct understanding of sentence fragments. A further extension of the model, which allows the speaker to intentionally reduce the noise rate on a word, is used to model prosodic emphasis. We show that the model derives several well-known changes in meaning associated with prosodic emphasis. Our results show that nominal amounts of actual noise can be leveraged for communicative purposes.

  BibTeX: @article{BergenGoodman2015-TiCS, author = {Bergen, L. and Goodman, N. D.}, date-modified = {2015-12-21 20:30:03 +0000}, journal = {Topics in Cognitive Science}, number = {2}, pages = {336--350}, title = {The strategic use of noise in pragmatic reasoning}, volume = {7}, website = {papers/BergenGoodman2015-TiCS.pdf}, year = {2015} }

• Rational use of cognitive resources: Levels of analysis between the computational and the algorithmic. [Abstract] [PDF] [BibTeX]
  T. L. Griffiths, F. Lieder, & N. D. Goodman. (2015). Topics in Cognitive Science.

  Abstract: Marr’s levels of analysis – computational, algorithmic, and implementation – have served cognitive science well over the last 30 years. But the recent increase in the popularity of the computational level raises a new challenge: How do we begin to relate models at different levels of analysis? We propose that it is possible to define levels of analysis that lie between the computational and the algorithmic, providing a way to build a bridge between computational- and algorithmic-level models. The key idea is to push the notion of rationality, often used in defining computational-level models, deeper toward the algorithmic level. We offer a simple recipe for reverse-engineering the mind’s cognitive strategies by deriving optimal algorithms for a series of increasingly more realistic abstract computational architectures, which we call "resource-rational analysis."

  BibTeX: @article{GriffithsEtAl2015-TiCS, author = {Griffiths, T. L. and Lieder, F. and Goodman, N. D.}, date-modified = {2015-12-21 20:29:34 +0000}, journal = {Topics in Cognitive Science}, number = {2}, pages = {217--229}, title = {Rational use of cognitive resources: Levels of analysis between the computational and the algorithmic}, volume = {7}, website = {papers/GriffithsEtAl2015-TiCS.pdf}, year = {2015} }

• A computational model of linguistic humor in puns. [Abstract] [PDF] [BibTeX]
  J. T. Kao, R. Levy, & N. D. Goodman. (2015). Cognitive Science. (Code at https://github.com/amoudgl/pun-model)

  Abstract: Humor plays an essential role in human interactions. Precisely what makes something funny, however, remains elusive. While research on natural language understanding has made significant advancements in recent years, there has been little direct integration of humor research with computational models of language understanding. In this paper, we propose two information-theoretic measures—ambiguity and distinctiveness—derived from a simple model of sentence processing. We test these measures on a set of puns and regular sentences and show that they correlate significantly with human judgments of funniness. Moreover, within a set of puns, the distinctiveness measure distinguishes exceptionally funny puns from mediocre ones. Our work is the first, to our knowledge, to integrate a model of general language understanding and humor theory to quantitatively predict humor at a fine-grained level. We present it as an example of a framework for applying models of language processing to understand higher-level linguistic and cognitive phenomena.

  BibTeX: @article{KaoEtAl2015-CognitiveScience, annote = {(Code at https://github.com/amoudgl/pun-model)}, author = {Kao, Justine T and Levy, Roger and Goodman, Noah D}, date-modified = {2019-02-10 11:04:49 +0000}, journal = {Cognitive Science}, title = {A computational model of linguistic humor in puns}, website = {papers/KaoEtAl2015-CognitiveScience.pdf}, year = {2015} }

• Adjectival vagueness in a Bayesian model of interpretation. [Abstract] [PDF] [BibTeX]
  D. Lassiter, & N. D. Goodman. (2015). Synthese.

  Abstract: We derive a probabilistic account of the vagueness and context-sensitivity of scalar adjectives from a Bayesian approach to communication and interpretation. We describe an iterated reasoning architecture for pragmatic interpretation and illustrate it with a simple scalar implicature example. We then show how to enrich the apparatus to handle pragmatic reasoning about the values of free variables, explore its predictions about the interpretation of scalar adjectives, and show how this model implements Edgington’s (1992; 1997) account of the sorites paradox, with variations. The Bayesian approach has a number of explanatory virtues: in particular, it does not require any specialpurpose machinery for handling vagueness, and it is integrated with a promising new approach to pragmatics and other areas of cognitive science.

  BibTeX: @article{LassiterGoodman2015-Synthese, author = {Lassiter, D. and Goodman, N. D.}, date-modified = {2015-06-28 14:22:52 +0000}, journal = {Synthese}, title = {Adjectival vagueness in a Bayesian model of interpretation}, website = {papers/LassiterGoodman2015-Synthese.pdf}, year = {2015} }

• Not by number alone: The effect of teacher’s knowledge and its value in evaluating "sins of omission". [Abstract] [PDF] [BibTeX]
  I. Bass, D. Hawthorne, N. D. Goodman, & H. Gweon. (2015). In Proceedings of the Thirty-Seventh Annual Conference of the Cognitive Science Society.

  Abstract: What constitutes good teaching, and what factors do learners consider when evaluating teachers? Prior developmental work suggests that even young children accurately recognize and evaluate under-informativeness. Building on prior work, we propose a Bayesian model of teacher evaluation that infers the teacher’s quality from how carefully he selected demonstrations given what he knew. We test the predictions of our model across 15 conditions in which participants saw a teacher who demonstrated all or a subset of functions of a novel device and rated his helpfulness. Our results suggest that human adults seamlessly integrate information about the number of functions taught, their values, as well as what the teacher knew, to make nuanced judgments about the quality of teaching; the quantitative pattern is well predicted by our model.

  BibTeX: @inproceedings{BassEtAl2015-Cogsci, author = {Bass, Ilona and Hawthorne, Daniel and Goodman, Noah D. and Gweon, Hyowon}, booktitle = {Proceedings of the Thirty-Seventh Annual Conference of the {C}ognitive {S}cience {S}ociety}, date-modified = {2015-07-30 15:56:39 +0000}, title = {Not by number alone: The effect of teacher's knowledge and its value in evaluating "sins of omission"}, website = {papers/BassEtAl2015-Cogsci.pdf}, year = {2015} }

• Extremely costly intensifiers are stronger than quite costly ones. [Abstract] [PDF] [BibTeX]
  E. Bennett, & N. D. Goodman. (2015). In Proceedings of the Thirty-Seventh Annual Conference of the Cognitive Science Society.

  Abstract: We show how the wide range in strengths of intensifying degree adverbs (e.g. very and extremely) could be explained by pragmatic inference based on differing cost, rather than differing semantics. This predicts a linear relationship between the meaning of intensifiers and their length and log-frequency. We test this prediction in two studies, using two different dependent measures, finding that higher cost does predict stronger meanings. We discuss the implications for adverbial meaning and the more general question of how extensive non-arbitrary form-meaning association may be in language.

  BibTeX: @inproceedings{BennettGoodman2015-Cogsci, author = {Bennett, Erin and Goodman, Noah D.}, booktitle = {Proceedings of the Thirty-Seventh Annual Conference of the {C}ognitive {S}cience {S}ociety}, date-modified = {2015-07-30 15:55:54 +0000}, title = {Extremely costly intensifiers are stronger than quite costly ones}, website = {papers/BennettGoodman2015-Cogsci.pdf}, year = {2015} }

• Wonky worlds: Listeners revise world knowledge when utterances are odd. [Abstract] [PDF] [BibTeX]
  J. Degen, M. H. Tessler, & N. D. Goodman. (2015). In Proceedings of the Thirty-Seventh Annual Conference of the Cognitive Science Society.

  Abstract: World knowledge enters into pragmatic utterance interpretation in complex ways, and may be defeasible in light of speakers’ utterances. Yet there is to date a surprising lack of systematic investigation into the role of world knowledge in pragmatic inference. In this paper, we show that a state-of-the-art model of pragmatic interpretation greatly overestimates the influence of world knowledge on the interpretation of utterances like Some of the marbles sank. We extend the model to capture the idea that the listener is uncertain about the background knowledge the speaker is bringing to the conversation. This extension greatly improves model predictions of listeners’ interpretation and also makes good qualitative predictions about listeners’ judgments of how ’normal’ the world is in light of a speaker’s statement. Theoretical and methodological implications are discussed.

  BibTeX: @inproceedings{DegenEtAl2015-Cogsci, author = {Degen, Judith and Tessler, Michael Henry and Goodman, Noah D.}, booktitle = {Proceedings of the Thirty-Seventh Annual Conference of the {C}ognitive {S}cience {S}ociety}, date-modified = {2015-07-30 15:58:02 +0000}, title = {Wonky worlds: Listeners revise world knowledge when utterances are odd}, website = {papers/DegenEtAl2015-Cogsci.pdf}, year = {2015} }

• How, whether, why: Causal judgments as counterfactual contrasts. [Abstract] [PDF] [BibTeX]
  T. Gerstenberg, N. D. Goodman, D. A. Lagnado, & J. B. Tenenbaum. (2015). In Proceedings of the Thirty-Seventh Annual Conference of the Cognitive Science Society.

  Abstract: How do people make causal judgments? Here, we propose a counterfactual simulation model (CSM) of causal judgment that unifies different views on causation. The CSM predicts that people’s causal judgments are influenced by whether a candidate cause made a difference to whether the outcome occurred as well as to how it occurred. We show how whethercausation and how-causation can be implemented in terms of different counterfactual contrasts defined over the same intuitive generative model of the domain. We test the model in an intuitive physics domain where people make judgments about colliding billiard balls. Experiment 1 shows that participants’ counterfactual judgments about what would have happened if one of the balls had been removed, are well-explained by an approximately Newtonian model of physics. In Experiment 2, participants judged to what extent two balls were causally responsible for a third ball going through a gate or missing the gate. As predicted by the CSM, participants’ judgments increased with their belief that a ball was a whether-cause, a how-cause, as well as sufficient for bringing about the outcome.

  BibTeX: @inproceedings{GerstenbergEtAl2015-Cogsci, author = {Gerstenberg, Tobias and Goodman, Noah D. and Lagnado, David A and Tenenbaum, Joshua B.}, booktitle = {Proceedings of the Thirty-Seventh Annual Conference of the {C}ognitive {S}cience {S}ociety}, date-modified = {2015-07-30 15:56:09 +0000}, title = {How, whether, why: Causal judgments as counterfactual contrasts}, website = {papers/GerstenbergEtAl2015-Cogsci.pdf}, year = {2015} }

• Why do you ask? Good questions provoke informative answers. [Abstract] [PDF] [BibTeX]
  R. X. D. Hawkins, A. Stuhlmüller, J. Degen, & N. D. Goodman. (2015). In Proceedings of the Thirty-Seventh Annual Conference of the Cognitive Science Society.

  Abstract: What makes a question useful? What makes an answer appropriate? In this paper, we formulate a family of increasingly sophisticated models of question-answer behavior within the Rational Speech Act framework. We compare these models based on three different pieces of evidence: first, we demonstrate how our answerer models capture a classic effect in psycholinguistics showing that an answerer’s level of informativeness varies with the inferred questioner goal, while keeping the question constant. Second, we jointly test the questioner and answerer components of our model based on empirical evidence from a question-answer reasoning game. Third, we examine a special case of this game to further distinguish among the questioner models. We find that sophisticated pragmatic reasoning is needed to account for some of the data. People can use questions to provide cues to the answerer about their interest, and can select answers that are informative about inferred interests.

  BibTeX: @inproceedings{HawkinsEtAl2015-Cogsci, author = {Hawkins, Robert X. D. and Stuhlm\"uller, Andreas and Degen, Judith and Goodman, Noah D.}, booktitle = {Proceedings of the Thirty-Seventh Annual Conference of the {C}ognitive {S}cience {S}ociety}, date-modified = {2015-07-30 15:57:43 +0000}, title = {Why do you ask? Good questions provoke informative answers}, website = {papers/HawkinsEtAl2015-Cogsci.pdf}, year = {2015} }

• So good it has to be true: Wishful thinking in theory of mind. [Abstract] [PDF] [BibTeX]
  D. Hawthorne, & N. D. Goodman. (2015). In Proceedings of the Thirty-Seventh Annual Conference of the Cognitive Science Society.

  Abstract: In standard decision theory, rational agents are objective, keeping their beliefs independent from their desires (Berger, 1985). Such agents are the basis for current computational models of Theory of Mind (ToM), but this fundamental assumption of the theory remains untested. Do people think that others’ beliefs are objective, or do they think that others’ desires color their beliefs? We describe a Bayesian framework for exploring this relationship and its implications. Motivated by this analysis, we conducted two experiments testing the a priori independence of beliefs and desires in people’s ToM and find that, contrary to fully-normative accounts, people think that others engage in wishful thinking. In the first experiment, we found that people think others believe both that desirable events are more likely to happen, and that undesirable ones are less likely to happen. In the second experiment, we found that social learning leverages this intuitive understanding of wishful thinking: participants learned more from the beliefs of an informant whose desires were contrary to his beliefs. People’s ToM therefore appears to be more nuanced than the current rational accounts, but consistent with a model in which desire directly affects the subjective probability of an event.

  BibTeX: @inproceedings{HawthorneGoodman2015-Cogsci, author = {Hawthorne, Daniel and Goodman, Noah D.}, booktitle = {Proceedings of the Thirty-Seventh Annual Conference of the {C}ognitive {S}cience {S}ociety}, date-modified = {2015-07-30 15:57:11 +0000}, title = {So good it has to be true: Wishful thinking in theory of mind}, website = {papers/HawthorneGoodman2015-Cogsci.pdf}, year = {2015} }

• A Resource-Rational Approach to the Causal Frame Problem. [Abstract] [PDF] [BibTeX]
  T. F. Icard III, & N. D. Goodman. (2015). In Proceedings of the Thirty-Seventh Annual Conference of the Cognitive Science Society.

  Abstract: The causal frame problem is an epistemological puzzle about how the mind is able to disregard seemingly irrelevant causal knowledge, and focus on those factors that promise to be useful in making an inference or coming to a decision. Taking a subject’s causal knowledge to be (implicitly) represented in terms of directed graphical models, the causal frame problem can be construed as the question of how to determine a reasonable "submodel" of one’s "full model" of the world, so as to optimize the balance between accuracy in prediction on the one hand, and computational costs on the other. We propose a framework for addressing this problem, and provide several illustrative examples based on HMMs and Bayes nets. We also show that our framework can account for some of the recent empirical phenomena associated with alternative neglect.

  BibTeX: @inproceedings{IcardGoodman2015-Cogsci, author = {Icard III, Thomas Frederick and Goodman, Noah D.}, booktitle = {Proceedings of the Thirty-Seventh Annual Conference of the {C}ognitive {S}cience {S}ociety}, date-modified = {2015-07-30 15:54:55 +0000}, title = {A Resource-Rational Approach to the Causal Frame Problem}, website = {papers/IcardGoodman2015-Cogsci.pdf}, year = {2015} }

• Let’s talk (ironically) about the weather: Modeling verbal irony. [Abstract] [PDF] [BibTeX]
  J. T. Kao, & N. D. Goodman. (2015). In Proceedings of the Thirty-Seventh Annual Conference of the Cognitive Science Society.

  Abstract: Verbal irony plays an important role in how we communicate and express opinions about the world. While there exist many theories and empirical findings about how people use and understand verbal irony, there is to our knowledge no formal model of how people incorporate shared background knowledge and linguistic information to communicate ironically. Here we argue that a computational approach previously shown to model hyperbole (Kao, Wu, Bergen, & Goodman, 2014) can also explain irony once we extend it to a richer space of affective subtext. We then describe two behavioral experiments that examine peoples interpretations of utterances in contexts that afford irony. We show that by minimally extending the hyperbole model to account for two dimensions of affect – valence and arousal – our model produces interpretations that closely match humans’. We discuss the implications of our model on informal theories of irony and its relationship to other types of nonliteral language understanding.

  BibTeX: @inproceedings{KaoEtAl2015-Cogsci, author = {Kao, Justine T and Goodman, Noah D.}, booktitle = {Proceedings of the Thirty-Seventh Annual Conference of the {C}ognitive {S}cience {S}ociety}, date-modified = {2015-07-30 15:56:19 +0000}, title = {Let's talk (ironically) about the weather: Modeling verbal irony}, website = {papers/KaoEtAl2015-Cogsci.pdf}, year = {2015} }

• Emergent Collective Sensing in Human Groups. [Abstract] [PDF] [BibTeX]
  P. M. Krafft, R. X. D. Hawkins, A. Pentland, N. D. Goodman, & J. B. Tenenbaum. (2015). In Proceedings of the Thirty-Seventh Annual Conference of the Cognitive Science Society. [Winner of the 2015 Cognitive Science Society computational modeling prize for Applied Cognition.]

  Abstract: Despite its importance, human collective intelligence remains enigmatic. We know what features are predictive of collective intelligence in human groups, but we do not understand the specific mechanisms that lead to the emergence of this distributed information processing ability. In contrast, there is a well-developed literature of experiments that have exposed the mechanisms of collective intelligence in nonhuman animal species. We adapt a recent experiment designed to study collective sensing in groups of fish in order to better understand the mechanisms that may underly the emergence of collective intelligence in human groups. We find that humans in our experiments act at a high level like fish but with two additional behaviors: independent exploration and targeted copying. These distinctively human activities may partially explain the emergence of collective sensing in our task environment at group sizes and on times scales orders of magnitudes smaller than were observed in fish.

  BibTeX: @inproceedings{KrafftEtAl2015-Cogsci, annote = {[Winner of the 2015 Cognitive Science Society computational modeling prize for Applied Cognition.]}, author = {Krafft, Peter M. and Hawkins, Robert X. D. and Pentland, Alex and Goodman, Noah D. and Tenenbaum, Joshua B.}, booktitle = {Proceedings of the Thirty-Seventh Annual Conference of the {C}ognitive {S}cience {S}ociety}, date-modified = {2015-07-30 15:55:33 +0000}, title = {Emergent Collective Sensing in Human Groups}, website = {papers/KrafftEtAl2015-Cogsci.pdf}, year = {2015} }

• Near-misses sting even when they are uncontrollable. [Abstract] [PDF] [BibTeX]
  D. C. Ong, N. D. Goodman, & J. Zaki. (2015). In Proceedings of the Thirty-Seventh Annual Conference of the Cognitive Science Society.

  Abstract: Observers often judge agents who miss a desired outcome by a small, compared to a large, margin to be less happy. This near-miss effect has typically been examined in situations where the agents have control over outcomes (e.g., missing a flight). Here, we extend this work in three ways. First, we show that near-miss effects play into observers’ intuitive theories of emotion even for randomly-determined outcomes over which agents demonstrably have no control. Second, we find data consistent with a hypothesis in which – even in randomly determined cases – near-miss effects reflect an illusion of control over those events. Finally, we integrate near-miss effects into a broader model of affective cognition, and quantify the psychological cost of a missing a desired outcome by relatively little distance, relative to winning or losing that outcome.

  BibTeX: @inproceedings{OngEtAl2015-Cogsci, author = {Ong, Desmond C. and Goodman, Noah D. and Zaki, Jamil}, booktitle = {Proceedings of the Thirty-Seventh Annual Conference of the {C}ognitive {S}cience {S}ociety}, date-modified = {2015-07-30 15:56:29 +0000}, title = {Near-misses sting even when they are uncontrollable}, website = {papers/OngEtAl2015-Cogsci.pdf}, year = {2015} }

• Controlling Procedural Modeling Programs with Stochastically-Ordered Sequential Monte Carlo. [Abstract] [PDF] [BibTeX]
  D. Ritchie, B. Mildenhall, N. D. Goodman, & P. Hanrahan. (2015). In SIGGRAPH 2015.

  Abstract: We present a method for controlling the output of procedural modeling programs using Sequential Monte Carlo (SMC). Previous probabilistic methods for controlling procedural models use Markov Chain Monte Carlo (MCMC), which receives control feedback only for completely-generated models. In contrast, SMC receives feedback incrementally on incomplete models, allowing it to reallocate computational resources and converge quickly. To handle the many possible sequentializations of a structured, recursive procedural modeling program, we develop and prove the correctness of a new SMC variant, Stochastically-Ordered Sequential Monte Carlo (SOSMC). We implement SOSMC for general-purpose programs using a new programming primitive: the stochastic future. Finally, we show that SOSMC reliably generates high-quality outputs for a variety of programs and control scoring functions. For small computational budgets, SOSMC’s outputs often score nearly twice as high as those of MCMC or normal SMC.

  BibTeX: @inproceedings{RitchieEtAl2015-SIGGRAPH, author = {Ritchie, Daniel and Mildenhall, B. and Goodman, N. D. and Hanrahan, P.}, booktitle = {SIGGRAPH 2015}, date-modified = {2015-07-30 15:55:23 +0000}, title = {Controlling Procedural Modeling Programs with Stochastically-Ordered Sequential Monte Carlo}, website = {papers/RitchieEtAl2015-SIGGRAPH.pdf}, year = {2015} }

• Toddlers Always Get the Last Word: Recency biases in early verbal behavior. [Abstract] [PDF] [BibTeX]
  E. Sumner, E. DeAngelis, M. Hyatt, N. D. Goodman, & C. Kidd. (2015). In Proceedings of the Thirty-Seventh Annual Conference of the Cognitive Science Society.

  Abstract: A popular conception about language development is that comprehension precedes production. Although this is certainly true during the earliest stages of phonological development, once a child possesses the basic articulatory skills required for imitation, it need not necessarily be the case. A child could produce a word without possessing the fully formed lexical representation through imitation. In some cases, such as in response to questions containing fixed choices, this behavior could be mistaken for a deeper understanding of the words’ semantic content. In this paper, we present evidence that 2to 3-year-old children exhibit a robust recency bias when verbally responding to two-alternative choice questions (i.e., they select the second, most recently mentioned option), possibly due to the availability of the second word in phonological memory. We find further evidence of this effect outside of a laboratory setting in naturalistic conversational contexts in CHILDES (MacWhinney, 2000), a large corpus of transcribed child-adult interactions.

  BibTeX: @inproceedings{SumnerEtAl2015-Cogsci, author = {Sumner, Emily and DeAngelis, Erika and Hyatt, Mara and Goodman, Noah D. and Kidd, Celeste}, booktitle = {Proceedings of the Thirty-Seventh Annual Conference of the {C}ognitive {S}cience {S}ociety}, date-modified = {2015-07-30 15:57:29 +0000}, title = {Toddlers Always Get the Last Word: Recency biases in early verbal behavior}, website = {papers/SumnerEtAl2015-Cogsci.pdf}, year = {2015} }

• Affective Cognition: Exploring lay theories of emotion. [Abstract] [PDF] [BibTeX]
  D. C. Ong, J. Zaki, & N. D. Goodman. (2015). Cognition.

  Abstract: Humans skillfully reason about others’ emotions, a phenomenon we term affective cognition. Despite its importance, few formal, quantitative theories have described the mechanisms supporting this phenomenon. We propose that affective cognition involves applying domain-general reasoning processes to domain-specific content knowledge. Observers’ knowledge about emotions is represented in rich and coherent lay theories, which comprise consistent relationships between situations, emotions, and behaviors. Observers utilize this knowledge in deciphering social agents’ behavior and signals (e.g., facial expressions), in a manner similar to rational inference in other domains. We construct a computational model of a lay theory of emotion, drawing on tools from Bayesian statistics, and test this model across four experiments in which observers drew inferences about others’ emotions in a simple gambling paradigm. This work makes two main contributions. First, the model accurately captures observers’ flexible but consistent reasoning about the ways that events and others’ emotional responses to those events relate to each other. Second, our work models the problem of emotional cue integration—reasoning about others’ emotion from multiple emotional cues—as rational inference via Bayes’ rule, and we show that this model tightly tracks human observers’ empirical judgments. Our results reveal a deep structural relationship between affective cognition and other forms of inference, and suggest wide-ranging applications to basic psychological theory and psychiatry.

  BibTeX: @article{OngEtAl2015-Cognition, author = {Ong, Desmond C. and Zaki, Jamil and Goodman, Noah D.}, date-modified = {2016-01-01 14:37:13 +0000}, journal = {Cognition}, pages = {141--162}, title = {Affective Cognition: Exploring lay theories of emotion}, volume = {143}, website = {papers/OngZakiGoodman2015.pdf}, year = {2015} }

• Evaluating Models of Computation and Storage in Human Sentence Processing. [Abstract] [PDF] [BibTeX]
  T. Luong, T. O’Donnell, & N. D. Goodman. (2015). In CogACLL 2015.

  Abstract: We examine the ability of several models of computation and storage to explain reading time data. Specifically, we demonstrate on both the Dundee and the MIT reading time corpora, that fragment grammars, a model that optimizes the tradeoff between computation and storage, is able to better explain people’s reaction times than two baseline models which exclusively favor either storage or computation. Additionally, we make a contribution by extending an existing incremental parser to handle more general grammars and scale well to larger rule and data sets

  BibTeX: @inproceedings{Luong2015, author = {Luong, Thang and O'Donnell, Timothy and Goodman, Noah D.}, booktitle = {CogACLL 2015}, date-added = {2015-07-30 15:58:46 +0000}, date-modified = {2015-07-30 16:00:44 +0000}, title = {Evaluating Models of Computation and Storage in Human Sentence Processing}, website = {http://people.linguistics.mcgill.ca/%7etimothy.odonnell/papers/reading-time.pdf}, year = {2015} }

Preprints

2014

• Reasoning about Reasoning by Nested Conditioning: Modeling Theory of Mind with Probabilistic Programs. [Abstract] [PDF] [BibTeX]
  A. Stuhlmüller, & N. D. Goodman. (2014). J. Cognitive Systems Research.

  Abstract: A wide range of human reasoning patterns can be explained as conditioning in probabilistic models; however, conditioning has traditionally been viewed as an operation applied to such models, not represented in such models. We describe how probabilistic programs can explicitly represent conditioning as part of a model. This enables us to describe reasoning about others’ reasoning using nested conditioning. Much of human reasoning is about the beliefs, desires, and intentions of other people; we use probabilistic programs to formalize these inferences in a way that captures the flexibility and inherent uncertainty of reasoning about other agents. We express examples from game theory, artificial intelligence, and linguistics as recursive probabilistic programs and illustrate how this representation language makes it easy to explore new directions in each of these fields. We discuss the algorithmic challenges posed by these kinds of models and describe how Dynamic Programming techniques can help address these challenges.

  BibTeX: @article{Stuhlmueller2013, author = {Stuhlm{\"u}ller, A. and Goodman, N. D.}, date-added = {2013-08-17 17:09:29 +0000}, date-modified = {2015-12-21 20:35:50 +0000}, journal = {J. Cognitive Systems Research}, pages = {80--99}, title = {Reasoning about Reasoning by Nested Conditioning: Modeling Theory of Mind with Probabilistic Programs}, volume = {28}, website = {https://web.stanford.edu/%7engoodman/papers/StuhlmuellerGoodman-CogSys-2013.pdf}, year = {2014} }

• One and Done? Optimal Decisions From Very Few Samples. [Abstract] [PDF] [BibTeX]
  E. Vul, N. D. Goodman, T. L. Griffiths, & J. B. Tenenbaum. (2014). Cognitive Science.

  Abstract: In many learning or inference tasks human behavior approximates that of a Bayesian ideal observer, suggesting that, at some level, cognition can be described as Bayesian inference. However, a number of findings have highlighted an intriguing mismatch between human behavior and standard assumptions about optimality: People often appear to make decisions based on just one or a few samples from the appropriate posterior probability distribution, rather than using the full distribution. Although sampling-based approximations are a common way to implement Bayesian inference, the very limited numbers of samples often used by humans seem insufficient to approximate the required probability distributions very accurately. Here, we consider this discrepancy in the broader framework of statistical decision theory, and ask: If people are making decisions based on samples – but as samples are costly – how many samples should people use to optimize their total expected or worst-case reward over a large number of decisions? We find that under reasonable assumptions about the time costs of sampling, making many quick but locally suboptimal decisions based on very few samples may be the globally optimal strategy over long periods. These results help to reconcile a large body of work showing sampling-based or probability matching behavior with the hypothesis that human cognition can be understood in Bayesian terms, and they suggest promising future directions for studies of resource-constrained cognition.

  BibTeX: @article{Vul2014, author = {Vul, E. and Goodman, N. D. and Griffiths, T. L. and Tenenbaum, J. B.}, date-modified = {2015-12-21 20:36:53 +0000}, journal = {Cognitive Science}, number = {4}, pages = {599--637}, title = {One and Done? Optimal Decisions From Very Few Samples}, volume = {38}, website = {https://web.stanford.edu/%7engoodman/papers/VulGoodmanGriffithsTenenbaum-COGS-2014.pdf}, year = {2014} }

• Inferring word meanings by assuming that speakers are informative. [Abstract] [PDF] [BibTeX]
  M. C. Frank, & N. D. Goodman. (2014). Cognitive Psychology.

  Abstract: Language comprehension is more than a process of decoding the literal meaning of a speaker’s utterance. Instead, by making the assumption that speakers choose their words to be informative in context, listeners routinely make pragmatic inferences that go beyond the linguistic data. If language learners make these same assumptions, they should be able to infer word meanings in otherwise ambiguous situations. We use probabilistic tools to formalize these kinds of informativeness inferences – extending a model of pragmatic language comprehension to the acquisition setting – and present four experiments whose data suggest that preschool children can use informativeness to infer word meanings and that adult judgments track quantitatively with informativeness.

  BibTeX: @article{FrankGoodmanICOM, author = {Frank, M. C. and Goodman, N. D.}, date-added = {2012-01-29 18:09:51 -0800}, date-modified = {2015-12-21 20:31:35 +0000}, journal = {Cognitive Psychology}, pages = {80--96}, title = {Inferring word meanings by assuming that speakers are informative}, volume = {75}, website = {https://langcog.stanford.edu/papers/FG-cogpsych2014.pdf}, year = {2014} }

• Probabilistic Models of Cognition. [Abstract] [PDF] [BibTeX]
  N. D. Goodman, & J. B. Tenenbaum. (2014). (https://probmods.org)

  Abstract: In this book, we explore the probabilistic approach to cognitive science, which models learning and reasoning as inference in complex probabilistic models. In particular, we examine how a broad range of empirical phenomena in cognitive science (including intuitive physics, concept learning, causal reasoning, social cognition, and language understanding) can be modeled using a functional probabilistic programming language called Church.

  BibTeX: @book{probmods, annote = {(https://probmods.org)}, author = {Goodman, N. D. and Tenenbaum, J. B.}, date-added = {2014-03-26 15:51:38 +0000}, date-modified = {2014-03-26 15:53:06 +0000}, title = {Probabilistic Models of Cognition}, website = {https://probmods.org}, year = {2014} }

• Formalizing the pragmatics of metaphor understanding. [Abstract] [PDF] [BibTeX]
  J. T. Kao, L. Bergen, & N. D. Goodman. (2014). In Proceedings of the Thirty-Sixth Annual Conference of the Cognitive Science Society.

  Abstract: While the ubiquity and importance of nonliteral language are clear, people’s ability to use and understand it remains a mystery. Metaphor in particular has been studied extensively across many disciplines in cognitive science. One approach focuses on the pragmatic principles that listeners utilize to infer meaning from metaphorical utterances. While this approach has generated a number of insights about how people understand metaphor, to our knowledge there is no formal model showing that effects in metaphor understanding can arise from basic principles of communication. Building upon recent advances in formal models of pragmatics, we describe a computational model that uses pragmatic reasoning to interpret metaphorical utterances. We conduct behavioral experiments to evaluate the model’s performance and show that our model produces metaphorical interpretations that closely fit behavioral data. We discuss implications of the model for metaphor understanding, principles of communication, and formal models of language understanding

  BibTeX: @inproceedings{KaoEtAl2014-Cogsci, author = {Kao, Justine T and Bergen, Leon and Goodman, Noah D.}, booktitle = {Proceedings of the Thirty-Sixth Annual Conference of the {C}ognitive {S}cience {S}ociety}, date-added = {2014-04-04 15:57:18 +0000}, date-modified = {2014-04-07 15:34:44 +0000}, title = {Formalizing the pragmatics of metaphor understanding}, website = {papers/KaoEtAl2014-Cogsci.pdf}, year = {2014} }

• The strategic use of noise in pragmatic reasoning. [Abstract] [PDF] [BibTeX]
  L. Bergen, & N. D. Goodman. (2014). In Proceedings of the Thirty-Sixth Annual Conference of the Cognitive Science Society. [Winner of the 2014 Cognitive Science Society computational modeling prize for Language.]

  Abstract: We combine two recent probabilistic approaches to natural language understanding, exploring the formal pragmatics of communication on a noisy channel. We show that nominal amounts of actual noise can be leveraged for communicative purposes. Common knowledge of a noisy channel leads to the use and correct understanding of sentence fragments. Prosodic emphasis, interpreted as an intentional action to reduce noise on a word, results in strengthened meanings.

  BibTeX: @inproceedings{Bergen2014, annote = {[Winner of the 2014 Cognitive Science Society computational modeling prize for Language.]}, author = {Bergen, Leon and Goodman, Noah D.}, booktitle = {Proceedings of the Thirty-Sixth Annual Conference of the {C}ognitive {S}cience {S}ociety}, date-added = {2014-04-04 15:59:55 +0000}, date-modified = {2014-04-07 15:34:52 +0000}, title = {The strategic use of noise in pragmatic reasoning}, website = {https://onlinelibrary.wiley.com/doi/pdf/10.1111/tops.12144}, year = {2014} }

• Lost your marbles? The puzzle of dependent measures in experimental pragmatics. [Abstract] [PDF] [BibTeX]
  J. Degen, & N. D. Goodman. (2014). In Proceedings of the Thirty-Sixth Annual Conference of the Cognitive Science Society.

  Abstract: A rarely discussed but important issue in research on pragmatic inference is the choice of dependent measure for estimating the robustness of pragmatic inferences and their sensitivity to contextual manipulations. Here we present the results from three studies exploring the effect of contextual manipulations on scalar implicature. In all three studies we manipulate the salient question under discussion and the perceptual availability of relevant set sizes. The studies differ only in the dependent measure used: Exp. 1 uses truth judgements, Exp. 2 uses word probability ratings, and Exp. 3 uses a direct measure of sentence interpretation. We argue that the first two are effectively measures of production, and find they are sensitive to our contextual manipulations. In contrast the interpretation measure shows no effect of context. We argue that this methodologically troubling finding can be understood and predicted by using the framework of probabilistic pragmatics.

  BibTeX: @inproceedings{DegenGoodman2014-Cogsci, author = {Degen, Judith and Goodman, Noah D.}, booktitle = {Proceedings of the Thirty-Sixth Annual Conference of the {C}ognitive {S}cience {S}ociety}, title = {Lost your marbles? The puzzle of dependent measures in experimental pragmatics}, website = {papers/DegenGoodman2014-Cogsci.pdf}, year = {2014} }

• Amortized inference in probabilistic reasoning. [Abstract] [PDF] [BibTeX]
  S. Gershman, & N. D. Goodman. (2014). In Proceedings of the Thirty-Sixth Annual Conference of the Cognitive Science Society.

  Abstract: Recent studies of probabilistic reasoning have postulated general-purpose inference algorithms that can be used to answer arbitrary queries. These algorithms are memoryless, in the sense that each query is processed independently, without reuse of earlier computation. We argue that the brain operates in the setting of amortized inference, where numerous related queries must be answered (e.g., recognizing a scene from multiple viewpoints); in this setting, memoryless algorithms can be computationally wasteful. We propose a simple form of flexible reuse, according to which shared inferences are cached and composed together to answer new queries. We present experimental evidence that humans exploit this form of reuse: the answer to a complex query can be systematically predicted from a person’s response to a simpler query if the simpler query was presented first and entails a sub-inference (i.e., a sub-component of the more complex query). People are also faster at answering a complex query when it is preceded by a sub-inference. Our results suggest that the astonishing efficiency of human probabilistic reasoning may be supported by interactions between inference and memory.

  BibTeX: @inproceedings{Gershman:2014wt, author = {Gershman, Sam and Goodman, Noah D.}, booktitle = {Proceedings of the Thirty-Sixth Annual Conference of the {C}ognitive {S}cience {S}ociety}, date-added = {2014-04-07 15:38:04 +0000}, date-modified = {2014-04-07 15:38:45 +0000}, title = {Amortized inference in probabilistic reasoning}, website = {https://web.stanford.edu/%7engoodman/papers/amortized_inference.pdf}, year = {2014} }

• From counterfactual simulation to causal judgment. [Abstract] [PDF] [BibTeX]
  T. Gerstenberg, N. D. Goodman, D. A. Lagnado, & J. B. Tenenbaum. (2014). In Proceedings of the Thirty-Sixth Annual Conference of the Cognitive Science Society.

  Abstract: In this paper, we demonstrate that people’s causal judgments are inextricably linked to counterfactuals. In our experiments, participants judge whether one billiard ball A caused another ball B to go through a gate. Our counterfactual simulation model predicts that people arrive at their causal judgments by comparing what actually happened with the result of mentally simulating what would have happened in the relevant counterfactual world. We test our model against actualist theories of causation which aim to explain causation just in terms of what actually happened. Our experimental stimuli contrast cases in which we hold constant what actually happened but vary the counterfactual outcome. In support of our model, we find that participants’ causal judgments differ drastically between such cases. People’s cause and prevention judgments increase with their subjective degree of belief that the counterfactual outcome would have been different from what actually happened.

  BibTeX: @inproceedings{GerstenbergEtAl2014-Cogsci, author = {Gerstenberg, Tobias and Goodman, Noah D. and Lagnado, David A and Tenenbaum, Joshua B}, booktitle = {Proceedings of the Thirty-Sixth Annual Conference of the {C}ognitive {S}cience {S}ociety}, title = {From counterfactual simulation to causal judgment}, website = {papers/GerstenbergEtAl2014-Cogsci.pdf}, year = {2014} }

• Some arguments are probably valid: Syllogistic reasoning as communication. [Abstract] [PDF] [BibTeX]
  M. H. Tessler, & N. D. Goodman. (2014). In Proceedings of the Thirty-Sixth Annual Conference of the Cognitive Science Society.

  Abstract: Syllogistic reasoning lies at the intriguing intersection of natural and formal reasoning, of language and logic. Syllogisms comprise a formal system of reasoning yet use natural language quantifiers, and invite natural language conclusions. How can we make sense of the interplay between logic and language? We develop a computational-level theory that considers reasoning over concrete situations, constructed probabilistically by sampling. The base model can be enriched to consider the pragmatics of natural language arguments. The model predictions are compared with behavioral data from a recent meta-analysis. The flexibility of the model is then explored in a data set of syllogisms using the generalized quantifiers most and few. We conclude by relating our model to two extant theories of syllogistic reasoning – Mental Models and Probability Heuristics.

  BibTeX: @inproceedings{Tessler:2014wu, author = {Tessler, Michael Henry and Goodman, Noah D.}, booktitle = {Proceedings of the Thirty-Sixth Annual Conference of the {C}ognitive {S}cience {S}ociety}, title = {Some arguments are probably valid: Syllogistic reasoning as communication}, website = {https://web.stanford.edu/%7engoodman/papers/cogsci14-syllogisms_tessler.pdf}, year = {2014} }

• A rational account of pedagogical reasoning: Teaching by, and learning from, examples. [Abstract] [PDF] [BibTeX]
  P. Shafto, N. D. Goodman, & T. L. Griffiths. (2014). Cognitive Psychology.

  Abstract: Much of learning and reasoning occurs in pedagogical situations – situations in which a person who knows a concept chooses examples for the purpose of helping a learner acquire the concept. We introduce a model of teaching and learning in pedagogical settings that predicts which examples teachers should choose and what learners should infer given a teacher’s examples. We present three experiments testing the model predictions for rule-based, prototype, and causally structured concepts. The model shows good quantitative and qualitative fits to the data across all three experiments, predicting novel qualitative phenomena in each case. We conclude by discussing implications for understanding concept learning and implications for theoretical claims about the role of pedagogy in human learning.

  BibTeX: @article{ShaftoEtAl2014, author = {Shafto, Patrick and Goodman, Noah D and Griffiths, Thomas L.}, date-modified = {2015-12-21 20:35:12 +0000}, journal = {Cognitive Psychology}, pages = {55--89}, title = {A rational account of pedagogical reasoning: Teaching by, and learning from, examples}, volume = {71}, website = {https://web.stanford.edu/%7engoodman/papers/shaftogg14.pdf}, year = {2014} }

• Generating Efficient MCMC Kernels from Probabilistic Programs. [Abstract] [PDF] [BibTeX]
  L. Yang, P. Hanrahan, & N. D. Goodman. (2014). In AISTATS 2014.

  Abstract: Universal probabilistic programming languages (such as Church) trade performance for abstraction: any model can be represented compactly as an arbitrary stochastic computation, but costly online analyses are required for inference. We present a technique that recovers hand-coded levels of performance from a universal probabilistic language, for the Metropolis-Hastings (MH) MCMC inference algorithm. It takes a Church program as input and traces its execution to remove computation overhead. It then analyzes the trace for each proposal, using slicing, to identify the minimal computation needed to evaluate the MH acceptance probability. Generated incremental code is much faster than a baseline implementation (up to 600x) and usually as fast as handcoded MH kernels.

  BibTeX: @inproceedings{YangEtAl2014, author = {Yang, L. and Hanrahan, P. and Goodman, N. D.}, booktitle = {AISTATS 2014}, date-modified = {2015-12-22 19:02:19 +0000}, title = {Generating Efficient MCMC Kernels from Probabilistic Programs}, website = {https://web.stanford.edu/%7engoodman/papers/aistats2014-shred.pdf}, year = {2014} }

• Nonliteral understanding of number words. [Abstract] [PDF] [BibTeX]
  J. T. Kao, J. Wu, L. Bergen, & N. D. Goodman. (2014). Proceedings of the National Academy of Sciences.

  Abstract: One of the most puzzling and important facts about communication is that people do not always mean what they say; speakers often use imprecise, exaggerated, or otherwise literally false descriptions to communicate experiences and attitudes. Here, we focus on the nonliteral interpretation of number words, in particular hyperbole (interpreting unlikely numbers as exaggerated and conveying affect) and pragmatic halo (interpreting round numbers imprecisely). We provide a computational model of number interpretation as social inference regarding the communicative goal, meaning, and affective subtext of an utterance. We show that our model predicts humans’ interpretation of number words with high accuracy. Our model is the first to our knowledge to incorporate principles of communication and empirically measured background knowledge to quantitatively predict hyperbolic and pragmatic halo effects in number interpretation. This modeling framework provides a unified approach to nonliteral language understanding more generally.

  BibTeX: @article{KaoEtAl2014-PNAS, author = {Kao, Justine T and Wu, Jean and Bergen, Leon and Goodman, Noah D}, date-added = {2014-07-02 14:29:32 +0000}, date-modified = {2015-12-21 20:34:42 +0000}, journal = {Proceedings of the {N}ational {A}cademy of {S}ciences}, number = {33}, pages = {12002--12007}, title = {Nonliteral understanding of number words}, volume = {111}, website = {papers/KaoEtAl2014-PNAS.pdf}, year = {2014} }

• Uncertainty and denial: a resource-rational model of the value of information. [Abstract] [PDF] [BibTeX]
  E. Pierson, & N. D. Goodman. (2014). PLoS ONE.

  Abstract: Classical decision theory predicts that people should be indifferent to information that is not useful for making decisions, but this model often fails to describe human behavior. Here we investigate one such scenario, where people desire information about whether an event (the gain/loss of money) will occur even though there is no obvious decision to be made on the basis of this information. We find a curious dual trend: if information is costless, as the probability of the event increases people want the information more; if information is not costless, people’s desire for the information peaks at an intermediate probability. People also want information more as the importance of the event increases, and less as the cost of the information increases. We propose a model that explains these results, based on the assumption that people have limited cognitive resources and obtain information about which events will occur so they can determine whether to expend effort planning for them.

  BibTeX: @article{PiersonGoodman2014-PLoSONE, author = {Pierson, E. and Goodman, N. D.}, date-modified = {2015-12-21 20:32:45 +0000}, journal = {PLoS ONE}, number = {11}, pages = {e113342}, title = {Uncertainty and denial: a resource-rational model of the value of information}, volume = {9}, website = {papers/PiersonGoodman2014-PLoSONE.pdf}, year = {2014} }

• Learning physics from dynamical scenes. [Abstract] [PDF] [BibTeX]
  T. D. Ullman, A. Stuhlmüller, N. D. Goodman, & J. B. Tenenbaum. (2014). In Proceedings of the Thirty-Sixth Annual Conference of the Cognitive Science Society.

  Abstract: Humans acquire their most basic physical concepts early in development, but continue to enrich and expand their intuitive physics throughout life as they are exposed to more and varied dynamical environments. We introduce a hierarchical Bayesian framework to explain how people can learn physical theories across multiple timescales and levels of abstraction. In contrast to previous Bayesian models of theory acquisition (Tenenbaum, Kemp, Griffiths, & Goodman, 2011), we work with more expressive probabilistic program representations suitable for learning the forces and properties that govern how objects interact in dynamic scenes unfolding over time. We compare our model and human learners on a challenging task of inferring novel physical laws in microworlds given short movies. People are generally able to perform this task and behave in line with model predictions. Yet they also make systematic errors suggestive of how a top-down Bayesian approach to learning might be complemented by a more bottomup feature-based approximate inference scheme, to best explain theory learning at an algorithmic level.

  BibTeX: @inproceedings{ullman:cogsci2014, author = {Ullman, Tomer D. and Stuhlm\"{u}ller, Andreas and Goodman, Noah D. and Tenenbaum, Joshua B.}, booktitle = {Proceedings of the Thirty-Sixth Annual Conference of the {C}ognitive {S}cience {S}ociety}, title = {Learning physics from dynamical scenes}, website = {https://stuhlmueller.org/papers/physics-cogsci2014.pdf}, year = {2014} }

Preprints

2013

• Did she jump because she was the big sister or because the trampoline was safe? Causal inference and the development of social attribution. [Abstract] [PDF] [BibTeX]
  E. Seiver, A. Gopnik, & N. D. Goodman. (2013). Child Development.

  Abstract: Children rely on both evidence and prior knowledge to make physical causal inferences; this study explores whether they make attributions about others’ behavior in the same manner. A total of one hundred and fifty-nine 4- and 6-year-olds saw 2 dolls interacting with 2 activities, and explained the dolls’ actions. In the person condition, each doll acted consistently across activities, but differently from each other. In the situation condition, the two dolls acted differently for each activity, but both performed the same actions. Both age groups provided more "person" explanations (citing features of the doll) in the person condition than in the situation condition. In addition, 6-year-olds showed an overall bias toward "person" explanations. As in physical causal inference, social causal inference combines covariational evidence and prior knowledge.

  BibTeX: @article{seiver2013did, author = {Seiver, Elizabeth and Gopnik, Alison and Goodman, Noah D}, journal = {Child {D}evelopment}, number = {2}, pages = {443--454}, title = {Did she jump because she was the big sister or because the trampoline was safe? Causal inference and the development of social attribution}, volume = {84}, website = {https://onlinelibrary.wiley.com/action/cookieAbsent}, year = {2013} }

• The mentalistic basis of core social cognition: experiments in preverbal infants and a computational model. [Abstract] [PDF] [BibTeX]
  K. J. Hamlin, T. Ullman, J. B. Tenenbaum, N. D. Goodman, & C. Baker. (2013). Developmental Science.

  Abstract: Evaluating individuals based on their proand anti-social behaviors is fundamental to successful human interaction. Recent research suggests that even preverbal infants engage in social evaluation; however, it remains an open question whether infants’ judgments are driven uniquely by an analysis of the mental states that motivate others’ helpful and unhelpful actions, or whether non-mentalistic inferences are at play. Here we present evidence from 10-month-olds, motivated and supported by a Bayesian computational model, for mentalistic social evaluation in the first year of life. A video abstract of this article can be viewed at //youtu.be/rD_Ry5oqCYE

  BibTeX: @article{Hamlin2013, author = {Hamlin, Kiley J and Ullman, Tomer and Tenenbaum, Josh B. and Goodman, Noah D. and Baker, Chris}, date-added = {2013-08-17 20:15:14 +0000}, date-modified = {2013-08-17 20:15:48 +0000}, journal = {{D}evelopmental {S}cience}, number = {2}, pages = {209--226}, publisher = {Wiley Online Library}, title = {The mentalistic basis of core social cognition: experiments in preverbal infants and a computational model}, volume = {16}, website = {https://web.stanford.edu/%7engoodman/papers/HamlinUllmanetalDevSci2013.pdf}, year = {2013} }

• Context, scale structure, and statistics in the interpretation of positive-form adjectives. [Abstract] [PDF] [BibTeX]
  D. Lassiter, & N. D. Goodman. (2013). In Semantics and Linguistic Theory (SALT) 23.

  Abstract: Relative adjectives in the positive form exhibit vagueness and context sensitivity. We suggest that these phenomena can be explained by the interaction of a free threshold variable in the meaning of the positive form with a probabilistic model of pragmatic inference. We describe a formal model of utterance interpretation as coordination, which jointly infers the value of the threshold variable and the intended meaning of the sentence. We report simulations exploring the effect of background statistical knowledge on adjective interpretation in this model. Motivated by these simulation results, we suggest that this approach can account for the correlation between scale structure and the relative/absolute distinction while also allowing for exceptions noted in previous work. Finally, we argue for a probabilistic explanation of why the sorites paradox is compelling with relative adjectives even though the second premise is false on a universal interpretation, and show that this account predicts Kennedy’s (2007) observation that the sorites paradox is more compelling with relative than with absolute adjectives.

  BibTeX: @inproceedings{Lassiter2013, author = {Lassiter, D. and Goodman, N. D.}, booktitle = {Semantics and {L}inguistic {T}heory {(SALT)} 23}, date-added = {2013-08-17 20:32:09 +0000}, date-modified = {2015-12-21 20:38:44 +0000}, pages = {587--610}, title = {Context, scale structure, and statistics in the interpretation of positive-form adjectives}, website = {https://journals.linguisticsociety.org/proceedings/index.php/SALT/article/view/2658/2404}, year = {2013} }

• Knowledge and implicature: Modeling language understanding as social cognition. [Abstract] [PDF] [BibTeX]
  N. D. Goodman, & A. Stuhlmüller. (2013). Topics in Cognitive Science.

  Abstract: Is language understanding a special case of social cognition? To help evaluate this view, we can formalize it as the rational speech-act theory: Listeners assume that speakers choose their utterances approximately optimally, and listeners interpret an utterance by using Bayesian inference to "invert" this model of the speaker. We apply this framework to model scalar implicature ("some" implies "not all," and "N" implies "not more than N"). This model predicts an interaction between the speaker’s knowledge state and the listener’s interpretation. We test these predictions in two experiments and find good fit between model predictions and human judgments.

  BibTeX: @article{Goodman2013, author = {Goodman, N.D. and Stuhlm{\"u}ller, A.}, journal = {Topics in Cognitive Science}, page = {173--184}, title = {Knowledge and implicature: Modeling language understanding as social cognition}, volume = {5}, website = {https://onlinelibrary.wiley.com/doi/pdf/10.1111/tops.12007}, year = {2013} }

• The principles and practice of probabilistic programming. [Abstract] [PDF] [BibTeX]
  N. D. Goodman. (2013). POPL 2013. (Extended abstract of keynote talk.)

  Abstract: Probabilities describe degrees of belief, and probabilistic inference describes rational reasoning under uncertainty. It is no wonder, then, that probabilistic models have exploded onto the scene of modern artificial intelligence, cognitive science, and applied statistics: these are all sciences of inference under uncertainty. But as probabilistic models have become more sophisticated, the tools to formally describe them and to perform probabilistic inference have wrestled with new complexity. Just as programming beyond the simplest algorithms requires tools for abstraction and composition, complex probabilistic modeling requires new progress in model representation – probabilistic programming languages. These languages provide compositional means for describing complex probability distributions; implementations of these languages provide generic inference engines: tools for performing efficient probabilistic inference over an arbitrary program.

  BibTeX: @misc{goodman2013principles, annote = {(Extended abstract of keynote talk.)}, author = {Goodman, Noah D}, booktitle = {POPL 2013}, title = {The principles and practice of probabilistic programming}, website = {https://web.stanford.edu/%7engoodman/papers/POPL2013-abstract.pdf}, year = {2013} }

• Learned helplessness and generalization. [Abstract] [PDF] [BibTeX]
  F. Lieder, N. D. Goodman, & Q. J. M. Huys. (2013). In Proceedings of the Thirty-Fifth Annual Conference of the Cognitive Science Society.

  Abstract: In learned helplessness experiments, subjects first experience a lack of control in one situation, and then show learning deficits when performing or learning another task in another situation. Generalization, thus, is at the core of the learned helplessness phenomenon. Substantial experimental and theoretical effort has been invested into establishing that a state- and task-independent belief about controllability is necessary. However, to what extent generalization is also sufficient to explain the transfer has not been examined. Here, we show qualitatively and quantitatively that Bayesian learning of action-outcome contingencies at three levels of abstraction is sufficient to account for the key features of learned helplessness, including escape deficits and impairment of appetitive learning after inescapable shocks.

  BibTeX: @inproceedings{lieder2013learned, author = {Lieder, Falk and Goodman, Noah D and Huys, Quentin JM}, booktitle = {Proceedings of the Thirty-Fifth Annual Conference of the {C}ognitive {S}cience {S}ociety}, title = {Learned helplessness and generalization}, website = {https://web.stanford.edu/%7engoodman/papers/LiederGoodmanHuys2013.pdf}, year = {2013} }

• The Funny Thing About Incongruity: A Computational Model of Humor in Puns. [Abstract] [PDF] [BibTeX]
  J. T. Kao, R. Levy, & N. D. Goodman. (2013). In Proceedings of the Thirty-Fifth Annual Conference of the Cognitive Science Society.

  Abstract: What makes something funny? Humor theorists posit that incongruity – perceiving a situation from different viewpoints and finding the resulting interpretations to be incompatible – contributes to sensations of mirth. In this paper, we use a computational model of sentence comprehension to formalize incongruity and test its relationship to humor in puns. By combining a noisy channel model of language comprehension and standard information theoretic measures, we derive two dimensions of incongruity – ambiguity of meaning and distinctiveness of viewpoints – and use them to predict humans’ judgments of funniness. Results showed that both ambiguity and distinctiveness are significant predictors of humor. Additionally, our model automatically identifies specific features of a pun that make it amusing. We thus show how a probabilistic model of sentence comprehension can help explain essential features of the complex phenomenon of linguistic humor.

  BibTeX: @inproceedings{kaofunny, author = {Kao, Justine T and Levy, Roger and Goodman, Noah D}, booktitle = {Proceedings of the Thirty-Fifth Annual Conference of the {C}ognitive {S}cience {S}ociety}, title = {The Funny Thing About Incongruity: A Computational Model of Humor in Puns}, website = {https://web.stanford.edu/%7engoodman/papers/KaoLevyGoodman.pdf}, year = {2013} }

• Learning and using language via recursive pragmatic reasoning about other agents. [Abstract] [PDF] [BibTeX]
  N. J. Smith, N. Goodman, & M. Frank. (2013). In Advances in Neural Information Processing Systems (NIPS 2013) , C. J. C. Burges, L. Bottou, M. Welling, Z. Ghahramani, & K. Q. Weinberger (Eds.).

  Abstract: Language users are remarkably good at making inferences about speakers’ intentions in context, and children learning their native language also display substantial skill in acquiring the meanings of unknown words. These two cases are deeply related: Language users invent new terms in conversation, and language learners learn the literal meanings of words based on their pragmatic inferences about how those words are used. While pragmatic inference and word learning have both been independently characterized in probabilistic terms, no current work unifies these two. We describe a model in which language learners assume that they jointly approximate a shared, external lexicon and reason recursively about the goals of others in using this lexicon. This model captures phenomena in word learning and pragmatic inference; it additionally leads to insights about the emergence of communicative systems in conversation and the mechanisms by which pragmatic inferences become incorporated into word meanings.

  BibTeX: @inproceedings{smith2013learning, author = {Smith, Nathaniel J and Goodman, Noah and Frank, Michael}, booktitle = {Advances in Neural Information Processing Systems (NIPS 2013)}, editor = {Burges, C.J.C. and Bottou, L. and Welling, M. and Ghahramani, Z. and Weinberger, K.Q.}, pages = {3039--3047}, publisher = {Curran Associates, Inc.}, title = {Learning and using language via recursive pragmatic reasoning about other agents}, website = {https://papers.nips.cc/paper/4929-learning-and-using-language-via-recursive-pragmatic-reasoning-about-other-agents.pdf}, year = {2013} }

• Learning Stochastic Inverses. [Abstract] [PDF] [BibTeX]
  A. Stuhlmüller, J. Taylor, & N. Goodman. (2013). In Advances in Neural Information Processing Systems (NIPS 2013).

  Abstract: We describe a class of algorithms for amortized inference in Bayesian networks. In this setting, we invest computation upfront to support rapid online inference for a wide range of queries. Our approach is based on learning an inverse factorization of a model’s joint distribution: a factorization that turns observations into root nodes. Our algorithms accumulate information to estimate the local conditional distributions that constitute such a factorization. These stochastic inverses can be used to invert each of the computation steps leading to an observation, sampling backwards in order to quickly find a likely explanation. We show that estimated inverses converge asymptotically in number of (prior or posterior) training samples. To make use of inverses before convergence, we describe the Inverse MCMC algorithm, which uses stochastic inverses to make block proposals for a Metropolis-Hastings sampler. We explore the efficiency of this sampler for a variety of parameter regimes and Bayes nets.

  BibTeX: @inproceedings{stuhlmuller2013learning, author = {Stuhlm{\"u}ller, Andreas and Taylor, Jacob and Goodman, Noah}, booktitle = {Advances in Neural Information Processing Systems (NIPS 2013)}, title = {Learning Stochastic Inverses}, website = {https://web.stanford.edu/%7engoodman/papers/inverses-nips-2013.pdf}, year = {2013} }

• Grounding Lexical Meaning in Core Cognition. [Abstract] [PDF] [BibTeX]
  N. D. Goodman. (2013). (Unpublished manuscript.)

  Abstract: Words are potentially one of the clearest windows on human knowledge and conceptual structure. But what do words mean? In this project we aim to construct and explore a formal model of lexical semantics grounded, via pragmatic inference, in core conceptual structures . Flexible human cognition is derived in large part from our ability to imagine possible worlds. A rich set of concepts, intuitive theories, and other mental representations support imagining and reasoning about possible worlds – together we call these core cognition. Here we posit that the collection of core concepts also forms the set of primitive elements available for lexical semantics: word meanings are built from pieces of core cognition. We propose to study lexical semantics in the setting of an architecture for language understanding that integrates literal meaning with pragmatic inference. This architecture supports underspecified and uncertain lexical meaning, leading to subtle interactions between meaning, conceptual structure, and context. We will explore several cases of lexical semantics where these interactions are particularly important: indexicals, scalar adjectives, generics, and modals. We formalize both core cognition and the natural language architecture using the Church probabilistic programming language. In this project we aim to contribute to our understanding of the connection between words and mental representations; from this we expect to gain critical insights into many aspects of psychology, to construct vastly more useful thinking machines, and to interface natural and artificial intelligences more efficiently.

  BibTeX: @unpublished{GoodmanGrant2013, annote = {(Unpublished manuscript.)}, author = {Goodman, N. D.}, title = {Grounding Lexical Meaning in Core Cognition}, website = {https://web.stanford.edu/%7engoodman/papers/LexSemSquibb.pdf}, year = {2013} }

Preprints

2012

• Comparing pluralities. [Abstract] [PDF] [BibTeX]
  G. Scontras, P. Graff, & N. D. Goodman. (2012). Cognition.

  Abstract: What does it mean to compare sets of objects along a scale, for example by saying "the men are taller than the women"? We explore comparison of pluralities in two experiments, eliciting comparison judgments while varying the properties of the members of each set. We find that a plurality is judged as "bigger" when the mean size of its members is larger than the mean size of the competing plurality. These results are incompatible with previous accounts, in which plural comparison is inferred from many instances of singular comparison between the members of the sets (Matushansky and Ruys, 2006). Our results suggest the need for a type of predication that ascribes properties to plural entities, not just individuals, based on aggregate statistics of their members. More generally, these results support the idea that sets and their properties are actively represented as single units.

  BibTeX: @article{scontras2012comparing, author = {Scontras, Gregory and Graff, Peter and Goodman, Noah D}, journal = {Cognition}, number = {1}, pages = {190--197}, title = {Comparing pluralities}, volume = {123}, website = {https://web.stanford.edu/%7engoodman/papers/ScontrasGraffGoodman-ComparingPluralities.pdf}, year = {2012} }

• Bootstrapping in a language of thought: A formal model of numerical concept learning. [Abstract] [PDF] [BibTeX]
  S. T. Piantadosi, J. B. Tenenbaum, & N. D. Goodman. (2012). Cognition.

  Abstract: In acquiring number words, children exhibit a qualitative leap in which they transition from understanding a few number words, to possessing a rich system of interrelated numerical concepts. We present a computational framework for understanding this inductive leap as the consequence of statistical inference over a sufficiently powerful representational system. We provide an implemented model that is powerful enough to learn number word meanings and other related conceptual systems from naturalistic data. The model shows that bootstrapping can be made computationally and philosophically wellfounded as a theory of number learning. Our approach demonstrates how learners may combine core cognitive operations to build sophisticated representations during the course of development, and how this process explains observed developmental patterns in number word learning.

  BibTeX: @article{piantadosi2012bootstrapping, author = {Piantadosi, Steven T and Tenenbaum, Joshua B and Goodman, Noah D}, journal = {Cognition}, number = {2}, pages = {199--217}, title = {Bootstrapping in a language of thought: A formal model of numerical concept learning}, volume = {123}, website = {https://web.stanford.edu/%7engoodman/papers/COGNIT2374.pdf}, year = {2012} }

• How many kinds of reasoning? Inference, probability, and natural language semantics. [Abstract] [PDF] [BibTeX]
  D. Lassiter, & N. D. Goodman. (2012). In 34th Annual Conference of the Cognitive Science Society.

  Abstract: Previous research (Heit & Rotello, 2010; Rips, 2001; Rotello & Heit, 2009) has suggested that differences between inductive and deductive reasoning cannot be explained by probabilistic theories, and instead support two-process accounts of reasoning. We provide a probabilistic model that predicts the observed non-linearities and makes quantitative predictions about responses as a function of argument strength. Predictions were tested using a novel experimental paradigm that elicits the previously-reported response patterns with a minimal manipulation, changing only one word between conditions. We also found a good fit with quantitative model predictions, indicating that a probabilistic theory of reasoning can account in a clear and parsimonious way for qualitative and quantitative data previously argued to falsify them. We also relate our model to recent work in linguistics, arguing that careful attention to the semantics of language used to pose reasoning problems will sharpen the questions asked in the psychology of reasoning.

  BibTeX: @inproceedings{lassiter2012many, author = {Lassiter, Daniel and Goodman, Noah D}, journal = {34th Annual Conference of the {C}ognitive {S}cience {S}ociety}, title = {How many kinds of reasoning? Inference, probability, and natural language semantics}, website = {https://web.stanford.edu/%7engoodman/papers/LassiterGoodman12.pdf}, year = {2012} }

• Noisy Newtons: Unifying process and dependency accounts of causal attribution. [Abstract] [PDF] [BibTeX]
  T. Gerstenberg, N. Goodman, D. A. Lagnado, & J. B. Tenenbaum. (2012). In Proceedings of the Thirty-Fourth Annual Conference of the Cognitive Science Society.

  Abstract: There is a long tradition in both philosophy and psychology to separate process accounts from dependency accounts of causation. In this paper, we motivate a unifying account that explains people’s causal attributions in terms of counterfactuals defined over probabilistic generative models. In our experiments, participants see two billiard balls colliding and indicate to what extent ball A caused/prevented ball B to go through a gate. Our model predicts that people arrive at their causal judgments by comparing what actually happened with what they think would have happened, had the collision between A and B not taken place. Participants’ judgments about what would have happened are highly correlated with a noisy model of Newtonian physics. Using those counterfactual judgments, we can predict participants’ cause and prevention judgments very accurately (r = .99). Our framework also allows us to capture intrinsically counterfactual judgments such as almost caused/prevented.

  BibTeX: @inproceedings{gerstenberg2012noisy, author = {Gerstenberg, Tobias and Goodman, Noah and Lagnado, David A and Tenenbaum, Joshua B}, booktitle = {Proceedings of the Thirty-Fourth Annual Conference of the {C}ognitive {S}cience {S}ociety}, title = {Noisy Newtons: Unifying process and dependency accounts of causal attribution}, website = {https://web.stanford.edu/%7engoodman/papers/physics_cogsci2nd.pdf}, year = {2012} }

• Ping pong in Church: Productive use of concepts in human probabilistic inference. [Abstract] [PDF] [BibTeX]
  T. Gerstenberg, & N. D. Goodman. (2012). In Proceedings of the 34th annual conference of the Cognitive Science Society.

  Abstract: How do people make inferences from complex patterns of evidence across diverse situations? What does a computational model need in order to capture the abstract knowledge people use for everyday reasoning? In this paper, we explore a novel modeling framework based on the probabilistic language of thought (PLoT) hypothesis, which conceptualizes thinking in terms of probabilistic inference over compositionally structured representations. The core assumptions of the PLoT hypothesis are realized in the probabilistic programming language Church (Goodman, Mansinghka, Roy, Bonawitz, & Tenenbaum, 2008). Using "ping pong tournaments" as a case study, we show how a single Church program concisely represents the concepts required to specify inferences from diverse patterns of evidence. In two experiments, we demonstrate a very close fit between our model’s predictions and participants’ judgments. Our model accurately predicts how people reason with confounded and indirect evidence and how different sources of information are integrated.

  BibTeX: @inproceedings{gerstenberg2012ping, author = {Gerstenberg, Tobias and Goodman, Noah D}, journal = {Proceedings of the 34th annual conference of the {C}ognitive {S}cience {S}ociety}, title = {Ping pong in Church: Productive use of concepts in human probabilistic inference}, website = {https://web.stanford.edu/%7engoodman/papers/GerstenbergGoodman2012.pdf}, year = {2012} }

• That’s what she (could have) said: How alternative utterances affect language use. [Abstract] [PDF] [BibTeX]
  L. Bergen, N. D. Goodman, & R. Levy. (2012). In Proceedings of the thirty-fourth annual conference of the Cognitive Science Society.

  Abstract: We investigate the effects of alternative utterances on pragmatic interpretation of language. We focus on two specific cases: specificity implicatures (less specific utterances imply the negation of more specific utterances) and Horn implicatures (more complex utterances are assigned to less likely meanings). We present models of these phenomena in terms of recursive social reasoning. Our most sophisticated model is not only able to handle specificity implicature but is also the first formal account of Horn implicatures that correctly predicts human behavior in signaling games with no prior conventions, without appeal to specialized equilibrium selection criteria. Two experiments provide evidence that these implicatures are generated in the absence of prior linguistic conventions or language evolution. Taken together, our modeling and experimental results suggest that the pragmatic effects of alternative utterances can be driven by cooperative social reasoning.

  BibTeX: @inproceedings{bergen2012s, author = {Bergen, Leon and Goodman, Noah D and Levy, Roger}, journal = {Proceedings of the thirty-fourth annual conference of the {C}ognitive {S}cience {S}ociety}, title = {That's what she (could have) said: How alternative utterances affect language use}, website = {https://web.stanford.edu/%7engoodman/papers/BergenEtAl2012.pdf}, year = {2012} }

• Knowledge and implicature: Modeling language understanding as social cognition. [Abstract] [PDF] [BibTeX]
  N. D. Goodman, & A. Stuhlmüller. (2012). In Proceedings of the Thirty-Fourth Annual Conference of the Cognitive Science Society. [Winner of the 2012 Cognitive Science Society computational modeling prize for Language.]

  Abstract: Is language understanding a special case of social cognition? To help evaluate this view, we can formalize it as the rational speech-act theory: Listeners assume that speakers choose their utterances approximately optimally, and listeners interpret an utterance by using Bayesian inference to "invert" this model of the speaker. We apply this framework to model scalar implicature ("some" implies "not all," and "N" implies "not more than N"). This model predicts an interaction between the speaker’s knowledge state and the listener’s interpretation. We test these predictions in two experiments and find good fit between model predictions and human judgments.

  BibTeX: @inproceedings{Goodman2012knowledge, annote = {[Winner of the 2012 Cognitive Science Society computational modeling prize for Language.]}, author = {Goodman, N.D. and Stuhlm{\"u}ller, A.}, date-added = {2013-07-22 14:48:45 +0000}, date-modified = {2013-07-22 14:48:56 +0000}, journal = {Proceedings of the Thirty-Fourth Annual Conference of the {C}ognitive {S}cience {S}ociety.}, title = {Knowledge and implicature: Modeling language understanding as social cognition}, website = {https://web.stanford.edu/%7engoodman/papers/KnowledgeImplicature-v2.pdf}, year = {2012} }

• Learning from others: The consequences of psychological reasoning for human learning. [Abstract] [PDF] [BibTeX]
  P. Shafto, N. D. Goodman, & M. C. Frank. (2012). Perspectives on Psychological Science.

  Abstract: From early childhood, human beings learn not only from collections of facts about the world, but also in social contexts: from observation of other people, from communication, and from explicit teaching. In these contexts, the data are the result of human actions – actions that come about because of people’s goals and intentions. To interpret the implications of others’ actions correctly, learners must understand the people generating the data. Most models of learning, however, assume that data are randomly collected facts about the world, and cannot explain how social contexts influence learning. We provide a Bayesian analysis of learning from knowledgeable others, which formalizes how a learner may reason from a person’s actions and goals to infer the actor’s knowledge about the world. We illustrate this framework using two examples from causal learning and conclude by discussing the implications for cognition, social reasoning, and cognitive development.

  BibTeX: @article{shafto2012learning, author = {Shafto, Patrick and Goodman, Noah D and Frank, Michael C}, journal = {Perspectives on Psychological Science}, number = {4}, pages = {341--351}, publisher = {Sage Publications}, title = {Learning from others: The consequences of psychological reasoning for human learning}, volume = {7}, website = {https://web.stanford.edu/%7engoodman/papers/shaftoperspectives.pdf}, year = {2012} }

• Synthesizing open worlds with constraints using locally annealed reversible jump MCMC. [Abstract] [PDF] [BibTeX]
  Y.-T. Yeh, L. Yang, M. Watson, N. D. Goodman, & P. Hanrahan. (2012). In SIGGRAPH 2012.

  Abstract: We present a novel Markov chain Monte Carlo (MCMC) algorithm that generates samples from transdimensional distributions encoding complex constraints. We use factor graphs, a type of graphical model, to encode constraints as factors. Our proposed MCMC method, called locally annealed reversible jump MCMC, exploits knowledge of how dimension changes affect the structure of the factor graph. We employ a sequence of annealed distributions during the sampling process, allowing us to explore the state space across different dimensionalities more freely. This approach is motivated by the application of layout synthesis where relationships between objects are characterized as constraints. In particular, our method addresses the challenge of synthesizing open world layouts where the number of objects are not fixed and optimal configurations for different numbers of objects may be drastically different. We demonstrate the applicability of our approach on two open world layout synthesis problems: coffee shops and golf courses.

  BibTeX: @inproceedings{yeh2012synthesizing, author = {Yeh, Yi-Ting and Yang, Lingfeng and Watson, Matthew and Goodman, Noah D and Hanrahan, Pat}, journal = {SIGGRAPH 2012}, number = {4}, pages = {56}, title = {Synthesizing open worlds with constraints using locally annealed reversible jump MCMC}, volume = {31}, website = {https://web.stanford.edu/%7engoodman/papers/owl.pdf}, year = {2012} }

• Predicting pragmatic reasoning in language games. [Abstract] [PDF] [BibTeX]
  M. C. Frank, & N. D. Goodman. (2012). Science.

  Abstract: One of the most astonishing features of human language is its capacity to convey information efficiently in context. Many theories provide informal accounts of communicative inference, yet there have been few successes in making precise, quantitative predictions about pragmatic reasoning. We examine judgments about simple referential communication games, modeling behavior in these games by assuming that speakers attempt to be informative, and that listeners use Bayesian inference to recover speakers’ intended referents. Our model provides a close, parameter-free fit to human judgments, suggesting that using information-theoretic tools to predict pragmatic reasoning may lead to more effective formal models of communication.

  BibTeX: @article{frank2012predicting, author = {Frank, Michael C and Goodman, Noah D}, journal = {Science}, number = {6084}, pages = {998--998}, publisher = {American Association for the Advancement of Science}, title = {Predicting pragmatic reasoning in language games}, volume = {336}, website = {https://web.stanford.edu/%7engoodman/papers/FrankGoodman-Science2012.pdf}, year = {2012} }

• Learning design patterns with bayesian grammar induction. [Abstract] [PDF] [BibTeX]
  J. Talton, L. Yang, R. Kumar, M. Lim, N. D. Goodman, & R. Mech. (2012). In Proceedings of the 25th annual ACM symposium on User interface software and technology. [Nominated for Best Paper Award.]

  Abstract: Design patterns have proven useful in many creative fields, providing content creators with archetypal, reusable guidelines to leverage in projects. Creating such patterns, however, is a time-consuming, manual process, typically relegated to a few experts in any given domain. In this paper, we describe an algorithmic method for learning design patterns directly from data using techniques from natural language processing and structured concept learning. Given a set of labeled, hierarchical designs as input, we induce a probabilistic formal grammar over these exemplars. Once learned, this grammar encodes a set of generative rules for the class of designs, which can be sampled to synthesize novel artifacts. We demonstrate the method on geometric models and Web pages, and discuss how the learned patterns can drive new interaction mechanisms for content creators.

  BibTeX: @inproceedings{talton2012learning, annote = {[Nominated for Best Paper Award.]}, author = {Talton, Jerry and Yang, Lingfeng and Kumar, Ranjitha and Lim, Maxine and Goodman, Noah D and Mech, R}, booktitle = {Proceedings of the 25th annual ACM symposium on User interface software and technology}, pages = {63--74}, title = {Learning design patterns with bayesian grammar induction}, website = {https://web.stanford.edu/%7engoodman/papers/Talton12LDPBG.pdf}, year = {2012} }

• A dynamic programming algorithm for inference in recursive probabilistic programs. [Abstract] [PDF] [BibTeX]
  A. Stuhlmüller, & N. D. Goodman. (2012). In Second Statistical Relational AI workshop at UAI 2012 (StaRAI-12).

  Abstract: We describe a dynamic programming algorithm for computing the marginal distribution of discrete probabilistic programs. This algorithm takes a functional interpreter for an arbitrary probabilistic programming language and turns it into an efficient marginalizer. Because direct caching of sub-distributions is impossible in the presence of recursion, we build a graph of dependencies between sub-distributions. This factored sum-product network makes (potentially cyclic) dependencies between subproblems explicit, and corresponds to a system of equations for the marginal distribution. We solve these equations by fixed-point iteration in topological order. We illustrate this algorithm on examples used in teaching probabilistic models, computational cognitive science research, and game theory.

  BibTeX: @inproceedings{stuhlmuller2012dynamic, author = {Stuhlm{\"u}ller, Andreas and Goodman, Noah D}, journal = {Second Statistical Relational AI workshop at UAI 2012 (StaRAI-12)}, title = {A dynamic programming algorithm for inference in recursive probabilistic programs}, website = {https://arxiv.org/abs/1206.3555}, year = {2012} }

• Theory learning as stochastic search in the language of thought. [Abstract] [PDF] [BibTeX]
  T. Ullman, N. D. Goodman, & J. B. Tenenbaum. (2012). Cognitive Development.

  Abstract: We present an algorithmic model for the development of children’s intuitive theories within a hierarchical Bayesian framework, where theories are described as sets of logical laws generated by a probabilistic context-free grammar. We contrast our approach with connectionist and other emergentist approaches to modeling cognitive development: while their subsymbolic representations provide a smooth error surface that supports efficient gradient-based learning, our symbolic representations are better suited to capturing children’s intuitive theories but give rise to a harder learning problem, which can only be solved by exploratory search. Our algorithm attempts to discover the theory that best explains a set of observed data by performing stochastic search at two levels of abstraction: an outer loop in the space of theories, and an inner loop in the space of explanations or models generated by each theory given a particular dataset. We show that this stochastic search is capable of learning appropriate theories in several everyday domains, and discuss its dynamics in the context of empirical studies of children’s learning.

  BibTeX: @article{Ullman2012, author = {Ullman, T. and Goodman, N. D. and Tenenbaum, J. B.}, date-modified = {2015-12-21 20:43:28 +0000}, journal = {Cognitive {D}evelopment}, number = {4}, pages = {455--480}, title = {Theory learning as stochastic search in the language of thought}, volume = {27}, website = {https://www.mit.edu/%7etomeru/papers/tlss-final.pdf}, year = {2012} }

• Burn-in, bias, and the rationality of anchoring. [Abstract] [PDF] [BibTeX]
  F. Lieder, T. L. Griffiths, & N. D. Goodman. (2012). In Advances in Neural Information Processing Systems.

  Abstract: Bayesian inference provides a unifying framework for learning, reasoning, and decision making. Unfortunately, exact Bayesian inference is intractable in all but the simplest models. Therefore minds and machines have to approximate Bayesian inference. Approximate inference algorithms can achieve a wide range of time-accuracy tradeoffs, but what is the optimal tradeoff? We investigate timeaccuracy tradeoffs using the Metropolis-Hastings algorithm as a metaphor for the mind’s inference algorithm(s). We characterize the optimal time-accuracy tradeoff mathematically in terms of the number of iterations and the resulting bias as functions of time cost, error cost, and the difficulty of the inference problem. We find that reasonably accurate decisions are possible long before the Markov chain has converged to the posterior distribution, i.e. during the period known as "burn-in". Therefore the strategy that is optimal subject to the mind’s bounded processing speed and opportunity costs may perform so few iterations that the resulting samples are biased towards the initial value. The resulting cognitive process model provides a rational basis for the anchoring-and-adjustment heuristic. The model’s quantitative predictions match published data on anchoring in numerical estimation tasks. In conclusion, resource-rationality – the optimal use of finite computational resources – naturally leads to a biased mind.

  BibTeX: @inproceedings{lieder2012burn, author = {Lieder, Falk and Griffiths, Thomas L and Goodman, Noah D}, booktitle = {Advances in Neural Information Processing Systems}, pages = {2699--2707}, title = {Burn-in, bias, and the rationality of anchoring}, website = {https://web.stanford.edu/%7engoodman/papers/LiederGriffithsGoodman2013NIPS.pdf}, year = {2012} }

Preprints

2011

• Learning a theory of causality.. [Abstract] [PDF] [BibTeX]
  N. D. Goodman, T. D. Ullman, & J. B. Tenenbaum. (2011). Psychological Review.

  Abstract: The very early appearance of abstract knowledge is often taken as evidence for innateness. We explore the relative learning speeds of abstract and specific knowledge within a Bayesian framework, and the role for innate structure. We focus on knowledge about causality, seen as a domain-general intuitive theory, and ask whether this knowledge can be learned from cooccurrence of events. We begin by phrasing the causal Bayes nets theory of causality, and a range of alternatives, in a logical language for relational theories. This allows us to explore simultaneous inductive learning of an abstract theory of causality and a causal model for each of several causal systems. We find that the correct theory of causality can be learned relatively quickly, often becoming available before specific causal theories have been learned – an effect we term the blessing of abstraction. We then explore the effect of providing a variety of auxiliary evidence, and find that a collection of simple "perceptual input analyzers" can help to bootstrap abstract knowledge. Together these results suggest that the most efficient route to causal knowledge may be to build in not an abstract notion of causality, but a powerful inductive learning mechanism and a variety of perceptual supports. While these results are purely computational, they have implications for cognitive development, which we explore in the conclusion.

  BibTeX: @article{goodman2011learning, author = {Goodman, Noah D and Ullman, Tomer D and Tenenbaum, Joshua B}, journal = {Psychological {R}eview}, number = {1}, pages = {110}, publisher = {American Psychological Association}, title = {Learning a theory of causality.}, volume = {118}, website = {https://web.stanford.edu/%7engoodman/papers/LTBC_psychreview_final.pdf}, year = {2011} }

• Lightweight implementations of probabilistic programming languages via transformational compilation. [Abstract] [PDF] [BibTeX]
  D. Wingate, A. Stuhlmüller, & N. D. Goodman. (2011). In Proceedings of the 14th international conference on Artificial Intelligence and Statistics.

  Abstract: We describe a general method of transforming arbitrary programming languages into probabilistic programming languages with straightforward MCMC inference engines. Random choices in the program are "named" with information about their position in an execution trace; these names are used in conjunction with a database holding values of random variables to implement MCMC inference in the space of execution traces. We encode naming information using lightweight source-to-source compilers. Our method enables us to reuse existing infrastructure (compilers, profilers, etc.) with minimal additional code, implying fast models with low development overhead. We illustrate the technique on two languages, one functional and one imperative: Bher, a compiled version of the Church language which eliminates interpretive overhead of the original MIT-Church implementation, and Stochastic Matlab, a new open-source language.

  BibTeX: @inproceedings{Wingate2011a, author = {Wingate, D. and Stuhlm{\"u}ller, A. and Goodman, N.D.}, booktitle = {Proceedings of the 14th international conference on Artificial Intelligence and Statistics}, pages = {770--778}, title = {Lightweight implementations of probabilistic programming languages via transformational compilation}, website = {http://proceedings.mlr.press/v15/wingate11a/wingate11a.pdf}, year = {2011} }

• Productivity and reuse in language. [Abstract] [PDF] [BibTeX]
  T. J. O’donnell, J. Snedeker, J. B. Tenenbaum, & N. D. Goodman. (2011). In Proceedings of the Thirty-Third Annual Conference of the Cognitive Science Society. [Winner of the 2011 Cognitive Science Society computational modeling prize for Language.]

  Abstract: We present a Bayesian model of the mirror image problems of linguistic productivity and reuse. The model, known as Fragment Grammar, is evaluated against several morphological datasets; its performance is compared to competing theoretical accounts including full-parsing, full-listing, and exemplar-based models. The model is able to learn the correct patterns of productivity and reuse for two very different systems: the English past tense which is characterized by a sharp dichotomy in productivity between regular and irregular forms and English derivational morphology which is characterized by a graded cline from very productive (-ness) to very unproductive (-th).

  BibTeX: @inproceedings{o2011productivity, annote = {[Winner of the 2011 Cognitive Science Society computational modeling prize for Language.]}, author = {O'donnell, Timothy J and Snedeker, Jesse and Tenenbaum, Joshua B and Goodman, Noah D}, journal = {Proceedings of the Thirty-Third Annual Conference of the {C}ognitive {S}cience {S}ociety}, title = {Productivity and reuse in language}, website = {https://web.stanford.edu/%7engoodman/papers/odonnell-cogsci11.pdf}, year = {2011} }

• Ad-hoc scalar implicature in adults and children. [Abstract] [PDF] [BibTeX]
  A. Stiller, N. D. Goodman, & M. C. Frank. (2011). In Proceedings of the 33rd Annual Meeting of the Cognitive Science Society.

  Abstract: Linguistic communication relies on pragmatic implicatures such as the inference that if "some students passed the test," not all did. Yet young children perform poorly on tests of implicature, especially scalar implicatures using "some" and "all," until quite late in development. We investigate the origins of scalar implicature using tasks in which the scale arises from real-world context rather than conventional contrasts between lexical items. Experiment 1 shows that these ad-hoc implicatures are easy for preschool children, suggesting that children have an early competence at pragmatic inference, and that failures in standard scalar implicature tasks are due instead to problems contrasting lexical items. Experiments 2 and 3 compare a Gricean, counterfactual account of implicature with a linguistic alternatives account and find that neither predicts effects of contextual informativeness. We conclude that an account of pragmatic implicature must integrate world knowledge, linguistic structure, and social reasoning.

  BibTeX: @inproceedings{stiller2011ad, author = {Stiller, Alex and Goodman, Noah D and Frank, Michael C}, journal = {Proceedings of the 33rd Annual Meeting of the {C}ognitive {S}cience {S}ociety}, title = {Ad-hoc scalar implicature in adults and children}, website = {https://web.stanford.edu/%7engoodman/papers/SGF-cogsci2011.pdf}, year = {2011} }

• Nonstandard Interpretations of Probabilistic Programs for Efficient Inference. [Abstract] [PDF] [BibTeX]
  D. Wingate, N. D. Goodman, A. Stuhlmueller, & J. M. Siskind. (2011). In Advances in Neural Information Processing Systems (NIPS 2011).

  Abstract: Probabilistic programming languages allow modelers to specify a stochastic process using syntax that resembles modern programming languages. Because the program is in machine-readable format, a variety of techniques from compiler design and program analysis can be used to examine the structure of the distribution represented by the probabilistic program. We show how nonstandard interpretations of probabilistic programs can be used to craft efficient inference algorithms: information about the structure of a distribution (such as gradients or dependencies) is generated as a monad-like side computation while executing the program. These interpretations can be easily coded using special-purpose objects and operator overloading. We implement two examples of nonstandard interpretations in two different languages, and use them as building blocks to construct inference algorithms: automatic differentiation, which enables gradient based methods, and provenance tracking, which enables efficient construction of global proposals.

  BibTeX: @inproceedings{wingate2011nonstandard, author = {Wingate, David and Goodman, Noah D and Stuhlmueller, Andreas and Siskind, Jeffrey Mark}, booktitle = {Advances in Neural Information Processing Systems (NIPS 2011)}, pages = {1152--1160}, title = {Nonstandard Interpretations of Probabilistic Programs for Efficient Inference}, website = {https://web.stanford.edu/%7engoodman/papers/WGSS-NIPS11.pdf}, year = {2011} }

• How to grow a mind: Statistics, structure, and abstraction. [Abstract] [PDF] [BibTeX]
  J. B. Tenenbaum, C. Kemp, T. L. Griffiths, & N. D. Goodman. (2011). Science.

  Abstract: In coming to understand the world – in learning concepts, acquiring language, and grasping causal relations – our minds make inferences that appear to go far beyond the data available. How do we do it? This review describes recent approaches to reverse-engineering human learning and cognitive development and, in parallel, engineering more humanlike machine learning systems. Computational models that perform probabilistic inference over hierarchies of flexibly structured representations can address some of the deepest questions about the nature and origins of human thought: How does abstract knowledge guide learning and reasoning from sparse data? What forms does our knowledge take, across different domains and tasks? And how is that abstract knowledge itself acquired?

  BibTeX: @article{tenenbaum2011grow, author = {Tenenbaum, Joshua B and Kemp, Charles and Griffiths, Thomas L and Goodman, Noah D}, journal = {Science}, number = {6022}, pages = {1279--1285}, title = {How to grow a mind: Statistics, structure, and abstraction}, volume = {331}, website = {https://web.stanford.edu/%7engoodman/papers/tkgg-science11-reprint.pdf}, year = {2011} }

• Bayesian policy search with policy priors. [Abstract] [PDF] [BibTeX]
  D. Wingate, N. D. Goodman, D. M. Roy, L. P. Kaelbling, & J. B. Tenenbaum. (2011). In Proceedings of the Twenty-Second international joint conference on Artificial Intelligence (IJCAI 11). [Winner of the Best Poster prize]

  Abstract: We consider the problem of learning to act in partially observable, continuous-state-and-action worlds where we have abstract prior knowledge about the structure of the optimal policy in the form of a distribution over policies. Using ideas from planning-as-inference reductions and Bayesian unsupervised learning, we cast Markov Chain Monte Carlo as a stochastic, hill-climbing policy search algorithm. Importantly, this algorithm’s search bias is directly tied to the prior and its MCMC proposal kernels, which means we can draw on the full Bayesian toolbox to express the search bias, including nonparametric priors and structured, recursive processes like grammars over action sequences. Furthermore, we can reason about uncertainty in the search bias itself by constructing a hierarchical prior and reasoning about latent variables that determine the abstract structure of the policy. This yields an adaptive search algorithm – our algorithm learns to learn a structured policy efficiently. We show how inference over the latent variables in these policy priors enables intra- and intertask transfer of abstract knowledge. We demonstrate the flexibility of this approach by learning meta search biases, by constructing a nonparametric finite state controller to model memory, by discovering motor primitives using a simple grammar over primitive actions, and by combining all three.

  BibTeX: @inproceedings{wingate2011bayesian, annote = {[Winner of the Best Poster prize]}, author = {Wingate, David and Goodman, Noah D and Roy, Daniel M and Kaelbling, Leslie P and Tenenbaum, Joshua B}, booktitle = {Proceedings of the Twenty-Second international joint conference on Artificial Intelligence (IJCAI 11)}, title = {Bayesian policy search with policy priors}, website = {https://web.stanford.edu/%7engoodman/papers/WingateEtAl-PolicyPrios.pdf}, year = {2011} }

• The imaginary fundamentalists: The unshocking truth about Bayesian cognitive science. [Abstract] [PDF] [BibTeX]
  N. Chater, N. Goodman, T. L. Griffiths, C. Kemp, M. Oaksford, & J. B. Tenenbaum. (2011). Behavioral and Brain Sciences. (Commentary on Jones and Love.)

  Abstract: The prominence of Bayesian modeling of cognition has increased recently largely because of mathematical advances in specifying and deriving predictions from complex probabilistic models. Much of this research aims to demonstrate that cognitive behavior can be explained from rational principles alone, without recourse to psychological or neurological processes and representations. We note commonalities between this rational approach and other movements in psychology – namely, Behaviorism and evolutionary psychology – that set aside mechanistic explanations or make use of optimality assumptions. Through these comparisons, we identify a number of challenges that limit the rational program’s potential contribution to psychological theory. Specifically, rational Bayesian models are significantly unconstrained, both because they are uninformed by a wide range of process-level data and because their assumptions about the environment are generally not grounded in empirical measurement. The psychological implications of most Bayesian models are also unclear. Bayesian inference itself is conceptually trivial, but strong assumptions are often embedded in the hypothesis sets and the approximation algorithms used to derive model predictions, without a clear delineation between psychological commitments and implementational details. Comparing multiple Bayesian models of the same task is rare, as is the realization that many Bayesian models recapitulate existing (mechanistic level) theories. Despite the expressive power of current Bayesian models, we argue they must be developed in conjunction with mechanistic considerations to offer substantive explanations of cognition. We lay out several means for such an integration, which take into account the representations on which Bayesian inference operates, as well as the algorithms and heuristics that carry it out. We argue this unification will better facilitate lasting contributions to psychological theory, avoiding the pitfalls that have plagued previous theoretical movements.

  BibTeX: @article{chater2011imaginary, annote = {(Commentary on Jones and Love.)}, author = {Chater, Nick and Goodman, Noah and Griffiths, Thomas L and Kemp, Charles and Oaksford, Mike and Tenenbaum, Joshua B}, journal = {Behavioral and Brain Sciences}, number = {04}, pages = {194--196}, publisher = {Cambridge University Press}, title = {The imaginary fundamentalists: The unshocking truth about Bayesian cognitive science}, volume = {34}, website = {https://journals.cambridge.org/download.php?file=%2FBBS%2FBBS34_04%2FS0140525X11000239a.pdf&code=1a1eedeede20981ce332fec41a70bb64#page=-142}, year = {2011} }

• The double-edged sword of pedagogy: Instruction limits spontaneous exploration and discovery. [Abstract] [PDF] [BibTeX]
  E. Bonawitz, P. Shafto, H. Gweon, N. D. Goodman, E. Spelke, & L. Schulz. (2011). Cognition.

  Abstract: Motivated by computational analyses, we look at how teaching affects exploration and discovery. In Experiment 1, we investigated children’s exploratory play after an adult pedagogically demonstrated a function of a toy, after an interrupted pedagogical demonstration, after a naı̈ve adult demonstrated the function, and at baseline. Preschoolers in the pedagogical condition focused almost exclusively on the target function; by contrast, children in the other conditions explored broadly. In Experiment 2, we show that children restrict their exploration both after direct instruction to themselves and after overhearing direct instruction given to another child; they do not show this constraint after observing direct instruction given to an adult or after observing a non-pedagogical intentional action. We discuss these findings as the result of rational inductive biases. In pedagogical contexts, a teacher’s failure to provide evidence for additional functions provides evidence for their absence; such contexts generalize from child to child (because children are likely to have comparable states of knowledge) but not from adult to child. Thus, pedagogy promotes efficient learning but at a cost: children are less likely to perform potentially irrelevant actions but also less likely to discover novel information.

  BibTeX: @article{BonawitzEtAl2011-Cognition, author = {Bonawitz, Elizabeth and Shafto, Patrick and Gweon, Hyowon and Goodman, Noah D and Spelke, Elizabeth and Schulz, Laura}, journal = {Cognition}, number = {3}, pages = {322--330}, title = {The double-edged sword of pedagogy: Instruction limits spontaneous exploration and discovery}, volume = {120}, website = {papers/BonawitzEtAl2011-Cognition.pdf}, year = {2011} }

• Where science starts: Spontaneous experiments in preschoolers’ exploratory play. [Abstract] [PDF] [BibTeX]
  C. Cook, N. D. Goodman, & L. E. Schulz. (2011). Cognition.

  Abstract: Probabilistic models of expected information gain require integrating prior knowledge about causal hypotheses with knowledge about possible actions that might generate data relevant to those hypotheses. Here we looked at whether preschoolers (mean: 54 months) recognize "action possibilities" (affordances) in the environment that allow them to isolate variables when there is information to be gained. By manipulating the physical properties of the stimuli, we were able to affect the degree to which candidate variables could be isolated; by manipulating the base rate of candidate causes, we were able to affect the potential for information gain. Children’s exploratory play was sensitive to both manipulations: given unambiguous evidence children played indiscriminately and rarely tried to isolate candidate causes; given ambiguous evidence, children both selected (Experiment 1) and designed (Experiment 2) informative interventions.

  BibTeX: @article{cook2011science, author = {Cook, Claire and Goodman, Noah D and Schulz, Laura E}, journal = {Cognition}, number = {3}, pages = {341--349}, title = {Where science starts: Spontaneous experiments in preschoolers' exploratory play}, volume = {120}, website = {https://web.stanford.edu/%7engoodman/papers/CookGoodmanSchulz2011.pdf}, year = {2011} }

• Inducing probabilistic programs by Bayesian program merging. [Abstract] [PDF] [BibTeX]
  I. Hwang, A. Stuhlmüller, & N. D. Goodman. (2011). Technical report: arXiv:1110.5667.

  Abstract: This report outlines an approach to learning generative models from data. We express models as probabilistic programs, which allows us to capture abstract patterns within the examples. By choosing our language for programs to be an extension of the algebraic data type of the examples, we can begin with a program that generates all and only the examples. We then introduce greater abstraction, and hence generalization, incrementally to the extent that it improves the posterior probability of the examples given the program. Motivated by previous approaches to model merging and program induction, we search for such explanatory abstractions using program transformations. We consider two types of transformation: Abstraction merges common subexpressions within a program into new functions (a form of anti-unification). Deargumentation simplifies functions by reducing the number of arguments. We demonstrate that this approach finds key patterns in the domain of nested lists, including parameterized sub-functions and stochastic recursion.

  BibTeX: @techreport{hwang2011inducing, author = {Hwang, Irvin and Stuhlm{\"u}ller, Andreas and Goodman, Noah D}, journal = {Technical report: arXiv:1110.5667}, title = {Inducing probabilistic programs by Bayesian program merging}, website = {https://arxiv.org/abs/1110.5667}, year = {2011} }

Preprints

2010

• Help or hinder: Bayesian models of social goal inference.. [Abstract] [PDF] [BibTeX]
  T. Ullman, C. L. Baker, O. Macindoe, O. Evans, N. D. Goodman, & J. B. Tenenbaum. (2010). In Advances in Neural Information Processing Systems (NIPS 2010).

  Abstract: Everyday social interactions are heavily influenced by our snap judgments about others’ goals. Even young infants can infer the goals of intentional agents from observing how they interact with objects and other agents in their environment: e.g., that one agent is ‘helping’ or ‘hindering’ another’s attempt to get up a hill or open a box. We propose a model for how people can infer these social goals from actions, based on inverse planning in multiagent Markov decision problems (MDPs). The model infers the goal most likely to be driving an agent’s behavior by assuming the agent acts approximately rationally given environmental constraints and its model of other agents present. We also present behavioral evidence in support of this model over a simpler, perceptual cue-based alternative.

  BibTeX: @inproceedings{Ullman2010, author = {Ullman, T. and Baker, C. L. and Macindoe, O. and Evans, O. and Goodman, N. D. and Tenenbaum, J. B.}, booktitle = {Advances in Neural Information Processing Systems (NIPS 2010)}, title = {Help or hinder: Bayesian models of social goal inference.}, website = {https://web.mit.edu/tomeru/www/papers/nips2010.pdf}, year = {2010} }

• Theory Acquisition as Stochastic Search. [Abstract] [PDF] [BibTeX]
  T. D. Ullman, N. D. Goodman, & J. B. Tenenbaum. (2010). In Proceedings of Thirty Second Annual Meeting of the Cognitive Science Society.

  Abstract: We present an algorithmic model for the development of children’s intuitive theories within a hierarchical Bayesian framework, where theories are described as sets of logical laws generated by a probabilistic context-free grammar. Our algorithm performs stochastic search at two levels of abstraction – an outer loop in the space of theories, and an inner loop in the space of explanations or models generated by each theory given a particular dataset – in order to discover the theory that best explains the observed data. We show that this model is capable of learning correct theories in several everyday domains, and discuss the dynamics of learning in the context of children’s cognitive development.

  BibTeX: @inproceedings{UllmanEtAl2010-Cogsci, author = {Ullman, T.D. and Goodman, N.D. and Tenenbaum, J.B.}, booktitle = {Proceedings of Thirty Second Annual Meeting of the {C}ognitive {S}cience {S}ociety}, title = {Theory Acquisition as Stochastic Search}, website = {papers/UllmanEtAl2010-Cogsci.pdf}, year = {2010} }

• Beyond Boolean logic: exploring representation languages for learning complex concepts. [Abstract] [PDF] [BibTeX]
  S. T. Piantadosi, J. B. Tenenbaum, & N. D. Goodman. (2010). In Proceedings of the 32nd Annual Conference of the Cognitive Science Society.

  Abstract: We study concept learning for semantically-motivated, settheoretic concepts. We first present an experiment in which we show that subjects learn concepts which cannot be represented by a simple Boolean logic. We then present a computational model which is similarly capable of learning these concepts, and show that it provides a good fit to human learning curves. Additionally, we compare the performance of several potential representation languages which are richer than Boolean logic in predicting human response distributions.

  BibTeX: @inproceedings{PiantadosiEtAl2010-Cogsci, author = {Piantadosi, Steven T and Tenenbaum, Joshua B and Goodman, Noah D}, journal = {Proceedings of the 32nd Annual Conference of the {C}ognitive {S}cience {S}ociety}, pages = {859--864}, title = {Beyond Boolean logic: exploring representation languages for learning complex concepts}, website = {papers/PiantadosiEtAl2010-Cogsci.pdf}, year = {2010} }

• Learning structured generative concepts. [Abstract] [PDF] [BibTeX]
  A. Stuhlmüller, J. B. Tenenbaum, & N. D. Goodman. (2010). In Proceedings of the Thirty-Second Annual Conference of the Cognitive Science Society.

  Abstract: Many real world concepts, such as "car", "house", and "tree", are more than simply a collection of features. These objects are richly structured, defined in terms of systems of relations, subparts, and recursive embeddings. We describe an approach to concept representation and learning that attempts to capture such structured objects. This approach builds on recent probabilistic approaches, viewing concepts as generative processes, and on recent rule-based approaches, constructing concepts inductively from a language of thought. Concepts are modeled as probabilistic programs that describe generative processes; these programs are described in a compositional language. In an exploratory concept learning experiment, we investigate human learning from sets of tree-like objects generated by processes that vary in their abstract structure, from simple prototypes to complex recursions. We compare human categorization judgements to predictions of the true generative process as well as a variety of exemplar-based heuristics.

  BibTeX: @inproceedings{StuhlmullerEtAl2010-Cogsci, author = {Stuhlm{\"u}ller, Andreas and Tenenbaum, Joshua B and Goodman, Noah D}, journal = {Proceedings of the Thirty-Second Annual Conference of the {C}ognitive {S}cience {S}ociety}, title = {Learning structured generative concepts}, website = {papers/StuhlmullerEtAl2010-Cogsci.pdf}, year = {2010} }

• Prior expectations in pedagogical situations. [Abstract] [PDF] [BibTeX]
  P. Shafto, N. D. Goodman, B. Gerstle, & F. Ladusaw. (2010). In Proceedings of the Thirty-Second Annual Conference of the Cognitive Science Society.

  Abstract: Much of human learning occurs in social situations, and among these, pedagogical situations may afford the most powerful learning. In pedagogical situations, a teacher chooses the concept that they are going to teach and the examples that they use to teach the concept. If learners know that a teacher is helpful and understands the implications, this could support strong inferences. In previous work, Shafto and Goodman (2008) proposed and tested a model of pedagogical data selection. We integrate special-purpose pedagogical expectations in this framework, and derive a task that allows independent assessment of pedagogical expectations. Two experiments contrast people’s expectations about pedagogical and communicative situations. The results show that people’s expectations differ in these situations, and that in pedagogical situations people expect teachers to present generalizable and semantically coherent knowledge. We discuss the implications for modeling learning in pedagogical settings, as well as for understanding human learning more broadly.

  BibTeX: @inproceedings{ShaftoEtAl2010-Cogsci, author = {Shafto, Patrick and Goodman, Noah D and Gerstle, Ben and Ladusaw, Francy}, journal = {Proceedings of the Thirty-Second Annual Conference of the {C}ognitive {S}cience {S}ociety}, title = {Prior expectations in pedagogical situations}, website = {papers/ShaftoEtAl2010-Cogsci.pdf}, year = {2010} }

• The Structure and Dynamics of Scientific Theories: A Hierarchical Bayesian Perspective. [Abstract] [PDF] [BibTeX]
  L. Henderson, N. D. Goodman, J. B. Tenenbaum, & J. F. Woodward. (2010). Philosophy of Science.

  Abstract: Hierarchical Bayesian models (HBMs) provide an account of Bayesian inference in a hierarchically structured hypothesis space. Scientific theories are plausibly regarded as organized into hierarchies in many cases, with higher levels sometimes called ’paradigms’ and lower levels encoding more specific or concrete hypotheses. Therefore, HBMs provide a useful model for scientific theory change, showing how higher-level theory change may be driven by the impact of evidence on lower levels. HBMs capture features described in the Kuhnian tradition, particularly the idea that higher-level theories guide learning at lower levels. In addition, they help resolve certain issues for Bayesians, such as scientific preference for simplicity and the problem of new theories.

  BibTeX: @article{HendersonEtAl2010-PhilosSci, author = {Henderson, Leah and Goodman, Noah D and Tenenbaum, Joshua B and Woodward, James F}, journal = {Philosophy of Science}, number = {2}, pages = {172--200}, title = {The Structure and Dynamics of Scientific Theories: A Hierarchical Bayesian Perspective}, volume = {77}, website = {papers/HendersonEtAl2010-PhilosSci.pdf}, year = {2010} }

• Predicting object and scene descriptions with an information-theoretic model of pragmatics. [Abstract] [PDF] [BibTeX]
  M. Frank, A. Kenney, N. Goodman, J. Tenenbaum, A. Torralba, & A. Oliva. (2010). Journal of Vision.

  Abstract: A picture may be worth a thousand words, but its description will likely use far fewer. How do speakers choose which aspects of a complex image to describe? Grice’s pragmatic maxims (e.g., “be relevant”, “be informative”) have served as an informal guide for understanding how speakers select which pieces of information to include in descriptions. We present a formalization of Grice’s maxim of informativeness (“choose descriptions proportional to the number of bits they convey about the referent with respect to context”) and test its ability to capture human performance. Experiment 1: Participants saw sets of four simple objects that varied on two dimensions (e.g., texture and shape) and were asked to provide the relative probabilities of using two different adjectives (e.g., polka-dot vs. square) to describe a target object relative to the distractor objects. Participants’ mean probabilities were highly correlated with the information theoretic model’s predictions for the relative informativeness of the two adjectives (r=.92,p<.0001). Experiment 2: Participants described street scenes from a database of hand-segmented and labeled images (LabelMe). In the context condition, participants were presented with a set of six scenes, one target and five distractors, and were asked to name five objects in the target scene so that another observer could pick that scene out of the set. In the no-context condition, another group of participants performed the same task without seeing the distractors. The information-theoretic model was strongly correlated with differences in object labeling between context and no-context conditions (r=.67,p<.0001), suggesting that the model captures the effect of context on descriptor choice. Our results suggest that speakers’ image descriptions conform to optimal pragmatic norms and that information theory can define norms for the linguistic compression of visual information. This constitutes a first step towards understanding how the visual world is captured and communicated by language users.

  BibTeX: @article{frank2010predicting, author = {Frank, Michael and Kenney, Avril and Goodman, Noah and Tenenbaum, Joshua and Torralba, Antonio and Oliva, Aude}, journal = {Journal of Vision}, number = {7}, pages = {1241--1241}, publisher = {Association for Research in Vision and Ophthalmology}, title = {Predicting object and scene descriptions with an information-theoretic model of pragmatics}, volume = {10}, website = {https://jov.arvojournals.org/article.aspx?articleid=2138025}, year = {2010} }

• Learning to learn causal models. [Abstract] [PDF] [BibTeX]
  C. Kemp, N. D. Goodman, & J. B. Tenenbaum. (2010). Cognitive Science.

  Abstract: Learning to understand a single causal system can be an achievement, but humans must learn about multiple causal systems over the course of a lifetime. We present a hierarchical Bayesian framework that helps to explain how learning about several causal systems can accelerate learning about systems that are subsequently encountered. Given experience with a set of objects, our framework learns a causal model for each object and a causal schema that captures commonalities among these causal models. The schema organizes the objects into categories and specifies the causal powers and characteristic features of these categories and the characteristic causal interactions between categories. A schema of this kind allows causal models for subsequent objects to be rapidly learned, and we explore this accelerated learning in four experiments. Our results confirm that humans learn rapidly about the causal powers of novel objects, and we show that our framework accounts better for our data than alternative models of causal learning.

  BibTeX: @article{KempEtAl2010-CognitiveScience, author = {Kemp, Charles and Goodman, Noah D and Tenenbaum, Joshua B}, journal = {Cognitive Science}, number = {7}, pages = {1185--1243}, publisher = {Blackwell Publishing Ltd}, title = {Learning to learn causal models}, volume = {34}, website = {papers/KempEtAl2010-CognitiveScience.pdf}, year = {2010} }

• Optimal habits can develop spontaneously through sensitivity to local cost. [Abstract] [PDF] [BibTeX]
  T. M. Desrochers, D. Z. Jin, N. D. Goodman, & A. M. Graybiel. (2010). Proceedings of the National Academy of Sciences.

  Abstract: Habits and rituals are expressed universally across animal species. These behaviors are advantageous in allowing sequential behaviors to be performed without cognitive overload, and appear to rely on neural circuits that are relatively benign but vulnerable to takeover by extreme contexts, neuropsychiatric sequelae, and processes leading to addiction. Reinforcement learning (RL) is thought to underlie the formation of optimal habits. However, this theoretic formulation has principally been tested experimentally in simple stimulus-response tasks with relatively few available responses. We asked whether RL could also account for the emergence of habitual action sequences in realistically complex situations in which no repetitive stimulus-response links were present and in which many response options were present. We exposed naive macaque monkeys to such experimental conditions by introducing a unique free saccade scan task. Despite the highly uncertain conditions and no instruction, the monkeys developed a succession of stereotypical, self-chosen saccade sequence patterns. Remarkably, these continued to morph for months, long after session-averaged reward and cost (eye movement distance) reached asymptote. Prima facie, these continued behavioral changes appeared to challenge RL. However, trial-by-trial analysis showed that pattern changes on adjacent trials were predicted by lowered cost, and RL simulations that reduced the cost reproduced the monkeys’ behavior. Ultimately, the patterns settled into stereotypical saccade sequences that minimized the cost of obtaining the reward on average. These findings suggest that brain mechanisms underlying the emergence of habits, and perhaps unwanted repetitive behaviors in clinical disorders, could follow RL algorithms capturing extremely local explore/exploit tradeoffs.

  BibTeX: @article{DesrochersEtAl2010-PNAS, author = {Desrochers, Theresa M and Jin, Dezhe Z and Goodman, Noah D and Graybiel, Ann M}, journal = {Proceedings of the National Academy of Sciences}, number = {47}, pages = {20512--20517}, publisher = {National Acad Sciences}, title = {Optimal habits can develop spontaneously through sensitivity to local cost}, volume = {107}, website = {papers/DesrochersEtAl2010-PNAS.pdf}, year = {2010} }

Preprints

2009

• Informative communication in word production and word learning. [Abstract] [PDF] [BibTeX]
  M. C. Frank, N. D. Goodman, P. Lai, & J. B. Tenenbaum. (2009). In Proceedings of the 31st Annual Conference of the Cognitive Science Society.

  Abstract: Language does not directly code facts about the world. Instead, speakers and listeners rely on shared assumptions to allow them to communicate more efficiently. Writers like Grice and Sperber & Wilson have proposed that communication is assumed to be "informative" or "relevant," but the predictions of these accounts are often informal or post-hoc. Here we propose a formal analogue to these accounts: that communicators choose what they want to say by how informative it would be about their intended meaning. We derive quantitative predictions about how this assumption would be used in language production and learning and test these predictions via two experiments. This work takes a first step towards formalizing the pragmatic assumptions necessary for effective communication in under-constrained, real-world situations.

  BibTeX: @inproceedings{FrankEtAl2009-Cogsci, author = {Frank, M. C. and Goodman, N. D. and Lai, P. and Tenenbaum, J. B.}, booktitle = {Proceedings of the 31st Annual Conference of the {C}ognitive {S}cience {S}ociety}, title = {Informative communication in word production and word learning}, website = {papers/FrankEtAl2009-Cogsci.pdf}, year = {2009} }

• Cause and intent: Social reasoning in causal learning. [Abstract] [PDF] [BibTeX]
  N. D. Goodman, C. L. Baker, & J. B. Tenenbaum. (2009). In Proceedings of the 31st Annual Conference of the Cognitive Science Society.

  Abstract: The acquisition of causal knowledge is a primary goal of childhood; yet most of this knowledge is known already to adults. We argue that causal learning which leverages social reasoning is a rapid and important route to knowledge. We present a computational model integrating knowledge about causality with knowledge about intentional agency, but using a domaingeneral mechanism for reasoning. Inference in this model predicts qualitatively different learning than an equivalent model based on causality alone or a hybrid causal-encoding model. We test these predictions experimentally with adult participants, and discuss the relation of these results to the developmental phenomenon of over-imitation.

  BibTeX: @inproceedings{GoodmanEtAl2009-Cogsci, author = {Goodman, N. D. and Baker, C. L. and Tenenbaum, J. B.}, booktitle = {Proceedings of the 31st Annual Conference of the {C}ognitive {S}cience {S}ociety}, title = {Cause and intent: Social reasoning in causal learning}, website = {papers/GoodmanEtAl2009-Cogsci.pdf}, year = {2009} }

• Continuity of discourse provides information for word learning. [Abstract] [PDF] [BibTeX]
  M. C. Frank, N. D. Goodman, J. B. Tenenbaum, & A. Fernald. (2009). In Proceedings of the 31st Annual Cognitive Science Society.

  Abstract: Utterances that are close in time are more likely to share the same referent. A word learner who is using information about the speaker’s intended referents should be able to take advantage of this continuity and learn words more efficiently by aggregating information across multiple utterances. In the current study we use corpus data to explore the continuity of reference in caregivers’ speech to infants. We measure the degree of referential continuity in two corpora and then use regression modeling to test whether reference continuity is informative about speakers’ referential intentions. We conclude by developing a simple discourse-continuity prior within a Bayesian model of word learning. Our results suggest that discourse continuity may be a valuable information source in early word learning.

  BibTeX: @inproceedings{FrankGoodmanTenenbaum2009-Cogsci, author = {Frank, Michael C and Goodman, Noah D and Tenenbaum, Joshua B and Fernald, Anne}, journal = {Proceedings of the 31st Annual {C}ognitive {S}cience {S}ociety}, title = {Continuity of discourse provides information for word learning}, website = {papers/FrankGoodmanTenenbaum2009-Cogsci.pdf}, year = {2009} }

• How tall is Tall? compositionality, statistics, and gradable adjectives. [Abstract] [PDF] [BibTeX]
  L. A. Schmidt, N. D. Goodman, D. Barner, & J. B. Tenenbaum. (2009). In Proceedings of the 31st annual conference of the Cognitive Science Society.

  Abstract: What is tall? Like many words, tall is fundamentally compositional in meaning – whether an item is tall depends on the statistical properties of the set of items it is being compared to. Despite preliminary evidence to this effect, no mathematical models of how tall operates in a given context have ever been empirically evaluated. We compare a number of statistical models to adults and children in judging which items are tall in various contexts, including both threshold-based and categorization-based models. We find that non-parametric statistical models of gradable adjectives best describes the judgments of people across a wide variety of contexts

  BibTeX: @inproceedings{SchmidtEtAl2009-Cogsci, author = {Schmidt, Lauren A and Goodman, Noah D and Barner, David and Tenenbaum, Joshua B}, journal = {Proceedings of the 31st annual conference of the {C}ognitive {S}cience {S}ociety}, pages = {2759--2764}, title = {How tall is Tall? compositionality, statistics, and gradable adjectives}, website = {papers/SchmidtEtAl2009-Cogsci.pdf}, year = {2009} }

• Fragment grammars: Exploring computation and reuse in language. [Abstract] [PDF] [BibTeX]
  T. J. O’Donnell, J. B. Tenenbaum, & N. D. Goodman. (2009). Technical Report MIT-CSAIL-TR-2009-013.

  Abstract: Language relies on a division of labor between stored units and structure building operations which combine the stored units into larger structures. This division of labor leads to a tradeoff: more structure-building means less need to store while more storage means less need to compute structure. We develop a hierarchical Bayesian model called fragment grammar to explore the optimum balance between structure-building and reuse. The model is developed in the context of stochastic functional programming (SFP), and in particular, using a probabilistic variant of Lisp known as the Church programming language [17]. We show how to formalize several probabilistic models of language structure using Church, and how fragment grammar generalizes one of them— adaptor grammars [21]. We conclude with experimental data with adults and preliminary evaluations of the model on natural language corpus data.

  BibTeX: @techreport{ODonnellEtAl2009-CSAIL, author = {O'Donnell, Timothy J and Tenenbaum, Joshua B and Goodman, Noah D}, journal = {Technical Report MIT-CSAIL-TR-2009-013}, publisher = {Massachusetts Institute of Technology}, title = {Fragment grammars: Exploring computation and reuse in language}, website = {papers/ODonnellEtAl2009-CSAIL.pdf}, year = {2009} }

• Using speakers’ referential intentions to model early cross-situational word learning. [Abstract] [PDF] [BibTeX]
  M. C. Frank, N. D. Goodman, & J. B. Tenenbaum. (2009). Psychological Science.

  Abstract: Word learning is a "chicken and egg" problem. If a child could understand speakers’ utterances, it would be easy to learn the meanings of individual words, and once a child knows what many words mean, it is easy to infer speakers’ intended meanings. To the beginning learner, however, both individual word meanings and speakers’ intentions are unknown. We describe a computational model of word learning that solves these two inference problems in parallel, rather than relying exclusively on either the inferred meanings of utterances or cross-situational word-meaning associations. We tested our model using annotated corpus data and found that it inferred pairings between words and object concepts with higher precision than comparison models. Moreover, as the result of making probabilistic inferences about speakers’ intentions, our model explains a variety of behavioral phenomena described in the word-learning literature. These phenomena include mutual exclusivity, one-trial learning, cross-situational learning, the role of words in object individuation, and the use of inferred intentions to disambiguate reference.

  BibTeX: @article{Frank2009, author = {Frank, M. C. and Goodman, N. D. and Tenenbaum, J. B.}, date-modified = {2015-12-21 20:44:47 +0000}, journal = {Psychological Science}, number = {5}, pages = {578--585}, title = {Using speakers' referential intentions to model early cross-situational word learning}, volume = {20}, website = {https://langcog.stanford.edu/papers/FGT-psychscience2009.pdf}, year = {2009} }

• One and done: Globally optimal behavior from locally suboptimal decisions.. [Abstract] [PDF] [BibTeX]
  E. Vul, N. D. Goodman, T. L. Griffiths, & J. B. Tenenbaum. (2009). In Proceedings of the 31st Annual Conference of the Cognitive Science Society.

  Abstract: In many learning or inference tasks human behavior approximates that of a Bayesian ideal observer, suggesting that, at some level, cognition can be described as Bayesian inference. However, a number of findings have highlighted an intriguing mismatch between human behavior and standard assumptions about optimality: People often appear to make decisions based on just one or a few samples from the appropriate posterior probability distribution, rather than using the full distribution. Although sampling-based approximations are a common way to implement Bayesian inference, the very limited numbers of samples often used by humans seem insufficient to approximate the required probability distributions very accurately. Here, we consider this discrepancy in the broader framework of statistical decision theory, and ask: If people are making decisions based on samples – but as samples are costly – how many samples should people use to optimize their total expected or worst-case reward over a large number of decisions? We find that under reasonable assumptions about the time costs of sampling, making many quick but locally suboptimal decisions based on very few samples may be the globally optimal strategy over long periods. These results help to reconcile a large body of work showing sampling-based or probability matching behavior with the hypothesis that human cognition can be understood in Bayesian terms, and they suggest promising future directions for studies of resource-constrained cognition.

  BibTeX: @inproceedings{Vul2009, author = {Vul, E. and Goodman, N. D. and Griffiths, T. L. and Tenenbaum, J. B.}, booktitle = {Proceedings of the 31st Annual Conference of the {C}ognitive {S}cience {S}ociety}, date-added = {2009-11-02 16:37:59 -0500}, date-modified = {2009-11-02 16:38:55 -0500}, title = {One and done: Globally optimal behavior from locally suboptimal decisions.}, website = {https://stanford.edu/%7engoodman/papers/VulEtAl2009.pdf}, year = {2009} }

• Learning a theory of causality. [Abstract] [PDF] [BibTeX]
  N. D. Goodman, T. D. Ullman, & J. B. Tenenbaum. (2009). In Proceedings of the 31st Annual Conference of the Cognitive Science Society.

  Abstract: We consider causality as a domain-general intuitive theory and ask whether this intuitive theory can be learned from cooccurrence of events. We begin by phrasing the causal Bayes nets theory of causality, and a range of alternatives, in a logical language for relational theories. This allows us to explore simultaneous inductive learning of an abstract theory of causality and a causal model for each of several causal systems. We find that the correct theory of causality can be learned relatively quickly, often becoming available before specific causal theories have been learned—an effect we term the "blessing of abstraction". We then explore the effect of providing a variety of auxiliary evidence, and find that a collection of simple "input analyzers" can help to bootstrap abstract knowledge. Together these results suggest that the most efficient route to causal knowledge may be to build in not an abstract notion of causality, but a powerful inductive learning mechanism and a variety of perceptual supports. While these results are purely computational, they have implications for cognitive development, which we explore in the conclusion.

  BibTeX: @inproceedings{GoodmanUllmanTenenbaum2009-Cogsci, author = {Goodman, N. D. and Ullman, T. D. and Tenenbaum, J. B.}, booktitle = {Proceedings of the 31st Annual Conference of the {C}ognitive {S}cience {S}ociety}, title = {Learning a theory of causality}, website = {papers/GoodmanUllmanTenenbaum2009-Cogsci.pdf}, year = {2009} }

• The infinite latent events model. [Abstract] [PDF] [BibTeX]
  D. Wingate, N. D. Goodman, D. M. Roy, & J. B. Tenenbaum. (2009). In Proceedings of the Twenty-Fifth Conference on Uncertainty in Artificial Intelligence.

  Abstract: We present the Infinite Latent Events Model, a nonparametric hierarchical Bayesian distribution over infinite dimensional Dynamic Bayesian Networks with binary state representations and noisy-OR-like transitions. The distribution can be used to learn structure in discrete timeseries data by simultaneously inferring a set of latent events, which events fired at each timestep, and how those events are causally linked. We illustrate the model on a sound factorization task, a network topology identification task, and a video game task.

  BibTeX: @inproceedings{WingateEtAl2009-UncertaintyInArtificialIntelligence, author = {Wingate, David and Goodman, Noah D and Roy, Daniel M and Tenenbaum, Joshua B}, booktitle = {Proceedings of the Twenty-Fifth Conference on Uncertainty in Artificial Intelligence}, organization = {AUAI Press}, pages = {607--614}, title = {The infinite latent events model}, website = {papers/WingateEtAl2009-UncertaintyInArtificialIntelligence.pdf}, year = {2009} }

Preprints

2008

• A Rational Analysis of Rule-based Concept Learning. [Abstract] [PDF] [BibTeX]
  N. D. Goodman, J. B. Tenenbaum, J. Feldman, & T. L. Griffiths. (2008). Cognitive Science.

  Abstract: We propose a new model of human concept learning that provides a rational analysis of learning feature-based concepts. This model is built upon Bayesian inference for a grammatically structured hypothesis space – a concept language of logical rules. We compare the model predictions to human generalization judgments in several well-known category learning experiments, and find good agreement for both average and individual participants generalizations. We further investigate judgments for a broad set of seven-feature concepts – a more natural setting in several ways – and again find that the model explains human performance.

  BibTeX: @article{GoodmanTenenbaum2008-CognitiveScience, author = {Goodman, Noah D. and Tenenbaum, Joshua B. and Feldman, Jacob and Griffiths, Thomas L.}, journal = {Cognitive Science}, number = {1}, pages = {108---154}, title = {A Rational Analysis of Rule-based Concept Learning}, volume = {32}, website = {papers/GoodmanTenenbaum2008-CognitiveScience.pdf}, year = {2008} }

• Church: a language for generative models. [Abstract] [PDF] [BibTeX]
  N. D. Goodman, V. K. Mansinghka, D. M. Roy, K. Bonawitz, & J. B. Tenenbaum. (2008). In Uncertainty in Artificial Intelligence.

  Abstract: Formal languages for probabilistic modeling enable re-use, modularity, and descriptive clarity, and can foster generic inference techniques. We introduce Church, a universal language for describing stochastic generative processes. Church is based on the Lisp model of lambda calculus, containing a pure Lisp as its deterministic subset. The semantics of Church is defined in terms of evaluation histories and conditional distributions on such histories. Church also includes a novel language construct, the stochastic memoizer, which enables simple description of many complex non-parametric models. We illustrate language features through several examples, including: a generalized Bayes net in which parameters cluster over trials, infinite PCFGs, planning by inference, and various non-parametric clustering models. Finally, we show how to implement query on any Church program, exactly and approximately, using Monte Carlo techniques.

  BibTeX: @inproceedings{GoodmanEtAl2008-UncertaintyInArtificialIntelligence, author = {Goodman, N. D. and Mansinghka, V. K. and Roy, D. M. and Bonawitz, K. and Tenenbaum, J. B.}, journal = {Uncertainty in Artificial Intelligence}, title = {Church: a language for generative models}, website = {papers/GoodmanEtAl2008-UncertaintyInArtificialIntelligence.pdf}, year = {2008} }

• Teaching Games: Statistical Sampling Assumptions for Learning in Pedagogical Situations. [Abstract] [PDF] [BibTeX]
  P. Shafto, & N. D. Goodman. (2008). In Proceedings of the Thirtieth Annual Meeting of the Cognitive Science Society.

  Abstract: Much of learning and reasoning occurs in pedagogical situations – situations in which teachers choose examples with the goal of having a learner infer the concept the teacher has in mind. In this paper, we present a model of teaching and learning in pedagogical settings which predicts what examples teachers should choose and what learners should infer given a teachers’ examples. We present two experiments using a novel experimental paradigm called the rectangle game. The first experiment compares people’s inferences to qualitative model predictions. The second experiment tests people in a situation where pedagogical sampling is not appropriate, ruling out alternative explanations, and suggesting that people use contextappropriate sampling assumptions. We conclude by discussing connections to broader work in inductive reasoning and cognitive development, and outline areas of future work.

  BibTeX: @inproceedings{ShaftoGoodman2008-Cogsci, author = {Shafto, Patrick and Goodman, Noah D.}, booktitle = {Proceedings of the Thirtieth Annual Meeting of the {C}ognitive {S}cience {S}ociety}, title = {Teaching Games: Statistical Sampling Assumptions for Learning in Pedagogical Situations}, website = {papers/ShaftoGoodman2008-Cogsci.pdf}, year = {2008} }

• A Bayesian Model of the Acquisition of Compositional Semantics. [Abstract] [PDF] [BibTeX]
  S. T. Piantadosi, N. D. Goodman, B. A. Ellis, & J. B. Tenenbaum. (2008). In Proceedings of Thirtieth Annual Meeting of the Cognitive Science Society.

  Abstract: We present an unsupervised, cross-situational Bayesian learning model for the acquisition of compositional semantics. We show that the model acquires the correct grammar for a toy version of English using a psychologically-plausible amount of data, over a wide range of possible learning environments. By assuming that speakers typically produce sentences which are true in the world, the model learns the semantic representation of content and function words, using only positive evidence in the form of sentences and world contexts. We argue that the model can adequately solve both the problem of referential uncertainty and the subset problem in this domain, and show that the model makes mistakes analogous to those made by children. Keywords: Compositional semantics; language acquisition; Bayesian learning; Combinatory Categorial Grammar.

  BibTeX: @inproceedings{PiantadosiEtAl2008-Cogsci, author = {Piantadosi, S. T. and Goodman, N. D. and Ellis, B. A. and Tenenbaum, J. B.}, booktitle = {Proceedings of Thirtieth Annual Meeting of the {C}ognitive {S}cience {S}ociety}, title = {A Bayesian Model of the Acquisition of Compositional Semantics}, website = {papers/PiantadosiEtAl2008-Cogsci.pdf}, year = {2008} }

• Theory acquisition and the language of thought. [Abstract] [PDF] [BibTeX]
  C. Kemp, N. D. Goodman, & J. B. Tenenbaum. (2008). In Proceedings of Thirtieth Annual Meeting of the Cognitive Science Society.

  Abstract: Everyday knowledge about living things, physical objects and the beliefs and desires of other people appears to be organized into sophisticated systems that are often called intuitive theories. Two long term goals for psychological research are to understand how these theories are mentally represented, and how they are acquired. We argue that the language of thought hypothesis can help to address both questions. First, compositional languages can capture the content of intuitive theories. Second, any compositional language will generate an account of theory learning which predicts that theories with short descriptions tend to be preferred. We describe a computational framework that captures both ideas, and compare its predictions to behavioral data from a simple theory learning task.

  BibTeX: @inproceedings{KempEtAl2008-Cogsci, author = {Kemp, C. and Goodman, N. D. and Tenenbaum, J. B.}, booktitle = {Proceedings of Thirtieth Annual Meeting of the {C}ognitive {S}cience {S}ociety}, title = {Theory acquisition and the language of thought}, website = {papers/KempEtAl2008-Cogsci.pdf}, year = {2008} }

• Theory-based Social Goal Inference. [Abstract] [PDF] [BibTeX]
  C. L. Baker, N. D. Goodman, & J. B. Tenenbaum. (2008). In Proceedings of Thirtieth Annual Meeting of the Cognitive Science Society.

  Abstract: Everyday human interaction relies on making inferences about social goals: goals that an intentional agent adopts in relation to another agent, such as "chasing", "fleeing", "approaching", "avoiding", "helping" or "hindering". We present a computational model of social goal inference that takes as input observations of multiple agents moving in some environmental context. The model infers a social goal for each agent that is most likely to have given rise to that agent’s observed actions, under an intuitive theory that expects agents to act approximately rationally. We provide evidence for our theory-based approach over a simpler bottom-up motion cue-based approach in a behavioral experiment designed to distinguish the two accounts.

  BibTeX: @inproceedings{BakerEtAl2008-Cogsci, author = {Baker, C. L. and Goodman, N. D. and Tenenbaum, J. B.}, booktitle = {Proceedings of Thirtieth Annual Meeting of the {C}ognitive {S}cience {S}ociety}, title = {Theory-based Social Goal Inference}, website = {papers/BakerEtAl2008-Cogsci.pdf}, year = {2008} }

• Structured Correlation from the Causal Background. [Abstract] [PDF] [BibTeX]
  R. Mayrhofer, N. D. Goodman, M. R. Waldmann, & J. B. Tenenbaum. (2008). In Proceedings of the Thirtieth Annual Conference of the Cognitive Science Society.

  Abstract: Previous research has cast doubt on whether the Markov condition is a default assumption of human causal reasoning—as causal Bayes net approaches suggest. Human subjects often seem to violate the Markov condition in common-cause reasoning tasks. While this might be treated as evidence that humans are inefficient causal reasoners, we propose that the underlying human intuitions reflect abstract causal knowledge that is sensitive to a great deal of contextual information— knowledge of the "causal background". In this paper, we introduce a hierarchical Bayesian model of causal background knowledge which explains Markov violations and makes additional, more fine-grained predictions on the basis of causally relevant category membership. We confirm these predictions using an experimental paradigm which extends that used in previous studies of "Markov violation."

  BibTeX: @inproceedings{MayrhoferEtAl2008-Cogsci, author = {Mayrhofer, R. and Goodman, N. D. and Waldmann, M. R. and Tenenbaum, J. B.}, booktitle = {Proceedings of the Thirtieth Annual Conference of the {C}ognitive {S}cience {S}ociety}, title = {Structured Correlation from the Causal Background}, website = {papers/MayrhoferEtAl2008-Cogsci.pdf}, year = {2008} }

• Compositionality in rational analysis: Grammar-based induction for concept learning. [Abstract] [PDF] [BibTeX]
  N. D. Goodman, J. B. Tenenbaum, T. L. Griffiths, & J. Feldman. (2008). In The probabilistic mind: Prospects for rational models of cognition , M. Oaksford & N. Chater (Eds.).

  Abstract: Rational analysis attempts to explain aspects of human cognition as an adaptive response to the environment (Marr, 1982; Anderson, 1990; Chater, Tenenbaum, & Yuille, 2006). The dominant approach to rational analysis today takes an ecologically reasonable specification of a problem facing an organism, given in statistical terms, then seeks an optimal solution, usually using Bayesian methods. This approach has proven very successful in cognitive science; it has predicted perceptual phenomena (Geisler & Kersten, 2002; Feldman, 2001), illuminated puzzling effects in reasoning (Chater & Oaksford, 1999; Griffiths & Tenenbaum, 2006), and, especially, explained how human learning can succeed despite sparse input and endemic uncertainty (Tenenbaum, 1999; Tenenbaum & Griffiths, 2001). However, there were earlier notions of the "rational" analysis of cognition that emphasized very different ideas. One of the central ideas behind logical and computational approaches, which previously dominated notions of rationality, is that meaning can be captured in the structure of representations, but that compositional semantics are needed for these representations to provide a coherent account of thought. In this chapter we attempt to reconcile the modern approach to rational analysis with some aspects of this older, logico-computational approach. We do this via a model—offered as an extended example—of human concept learning. In the current chapter we are primarily concerned with formal aspects of this approach; in other work (Goodman, Tenenbaum, Feldman, & Griffiths, in press) we more carefully study a variant of this model as a psychological model of human concept learning.

  BibTeX: @inbook{GoodmanEtAl2008-Chapter, author = {Goodman, Noah D. and Tenenbaum, Joshua B. and Griffiths, Thomas L. and Feldman, Jacob}, booktitle = {The probabilistic mind: Prospects for rational models of cognition}, editor = {Oaksford, Michael and Chater, Nick}, publisher = {Oxford University Press}, title = {Compositionality in rational analysis: Grammar-based induction for concept learning}, website = {papers/GoodmanEtAl2008-Chapter.pdf}, year = {2008} }

• Modeling Semantic Cognition as Logical Dimensionality Reduction. [Abstract] [PDF] [BibTeX]
  Y. Katz, N. D. Goodman, K. Kersting, C. Kemp, & J. B. Tenenbaum. (2008). In Proceedings of the Thirtieth Annual Conference of the Cognitive Science Society.

  Abstract: Semantic knowledge is often expressed in the form of intuitive theories, which organize, predict and explain our observations of the world. How are these powerful knowledge structures represented and acquired? We present a framework, logical dimensionality reduction, that treats theories as compressive probabilistic models, attempting to express observed data as a sample from the logical consequences of the theory’s underlying laws and a small number of core facts. By performing Bayesian learning and inference on these models we combine important features of more familiar connectionist and symbolic approaches to semantic cognition: an ability to handle graded, uncertain inferences, together with systematicity and compositionality that support appropriate inferences from sparse observations in novel contexts.

  BibTeX: @inproceedings{KatzEtAl2008-Cogsci, author = {Katz, Y. and Goodman, N. D. and Kersting, K. and Kemp, C. and Tenenbaum, J. B.}, journal = {Proceedings of the Thirtieth Annual Conference of the {C}ognitive {S}cience {S}ociety}, title = {Modeling Semantic Cognition as Logical Dimensionality Reduction}, website = {papers/KatzEtAl2008-Cogsci.pdf}, year = {2008} }

• A stochastic programming perspective on nonparametric Bayes. [Abstract] [PDF] [BibTeX]
  D. M. Roy, V. K. Mansinghka, N. D. Goodman, & J. B. Tenenbaum. (2008). Nonparametric Bayesian Workshop, Int. Conf. on Machine Learning.

  Abstract: We use Church, a Turing-universal language for stochastic generative processes and the probability distributions they induce, to study and extend several objects in nonparametric Bayesian statistics. We connect exchangeability and de Finetti measures with notions of purity and closures from functional programming. We exploit delayed evaluation to provide finite, machine-executable representations for various nonparametric Bayesian objects. We relate common uses of the Dirichlet process to a stochastic generalization of memoization, and use this abstraction to compactly describe and extend several nonparametric models. Finally, we briefly discuss issues of computability and inference.

  BibTeX: @misc{roy2008stochastic, author = {Roy, DM and Mansinghka, VK and Goodman, ND and Tenenbaum, JB}, journal = {Nonparametric Bayesian Workshop, Int. Conf. on Machine Learning}, pages = {26}, title = {A stochastic programming perspective on nonparametric Bayes}, volume = {22}, website = {http://danroy.org/papers/RoyManGooTen-ICMLNPB-2008.pdf}, year = {2008} }

• Going beyond the evidence: abstract laws and preschoolers’ responses to anomalous data.. [Abstract] [PDF] [BibTeX]
  L. E. Schulz, N. D. Goodman, J. B. Tenenbaum, & A. C. Jenkins. (2008). Cognition. doi:n.2008.07.017

  Abstract: Given minimal evidence about novel objects, children might learn only relationships among the specific entities, or they might make a more abstract inference, positing classes of entities and the relations that hold among those classes. Here we show that preschoolers (mean: 57 months) can use sparse data about perceptually unique objects to infer abstract physical causal laws. These newly inferred abstract laws were robust to potentially anomalous evidence; in the face of apparent counter-evidence, children (correctly) posited the existence of an unobserved object rather than revise the abstract laws. This suggests that children’s ability to learn robust, abstract principles does not depend on extensive prior experience but can occur rapidly, on-line, and in tandem with inferences about specific relations.

  BibTeX: @article{Schulz2008, author = {Schulz, Laura E. and Goodman, Noah D. and Tenenbaum, Joshua B. and Jenkins, Adrianna C.}, doi = {n.2008.07.017}, journal = {Cognition}, month = nov, number = {2}, owner = {noah}, pages = {211--223}, pii = {S0010-0277(08)00179-0}, pmid = {18930186}, timestamp = {2009.01.09}, title = {Going beyond the evidence: abstract laws and preschoolers' responses to anomalous data.}, volume = {109}, website = {https://pubmed.ncbi.nlm.nih.gov/18930186/}, year = {2008}, bdsk-url-1 = {https://doi.org/n.2008.07.017} }

• Random-World Semantics and Syntactic Independence for Expressive Languages. [Abstract] [PDF] [BibTeX]
  D. McAllester, B. Milch, & N. D. Goodman. (2008).

  Abstract: We consider three desiderata for a language combining logic and probability: logical expressivity, random-world semantics, and the existence of a useful syntactic condition for probabilistic independence. Achieving these three desiderata simultaneously is nontrivial. Expressivity can be achieved by using a formalism similar to a programming language, but standard approaches to combining programming languages with probabilities sacrifice random-world semantics. Naive approaches to restoring random-world semantics undermine syntactic independence criteria. Our main result is a syntactic independence criterion that holds for a broad class of highly expressive logics under random-world semantics. We explore various examples including Bayesian networks, probabilistic context-free grammars, and an example from Mendelian genetics. Our independence criterion supports a case-factor inference technique that reproduces both variable elimination for BNs and the inside algorithm for PCFGs.

  BibTeX: @techreport{McAllesterEtAl2008-CSAIL, author = {McAllester, D. and Milch, B. and Goodman, N. D.}, institution = {Massachusetts Institute of Technology}, number = {MIT-CSAIL-TR-2008-025}, title = {Random-World Semantics and Syntactic Independence for Expressive Languages}, website = {papers/McAllesterEtAl2008-CSAIL.pdf}, year = {2008} }

Preprints

2007

• A bayesian framework for crosssituational word-learning. [Abstract] [PDF] [BibTeX]
  M. C. Frank, N. D. Goodman, & J. B. Tenenbaum. (2007). In Advances in Neural Information Processing Systems (NIPS 2007).

  Abstract: For infants, early word learning is a chicken-and-egg problem. One way to learn a word is to observe that it co-occurs with a particular referent across different situations. Another way is to use the social context of an utterance to infer the intended referent of a word. Here we present a Bayesian model of cross-situational word learning, and an extension of this model that also learns which social cues are relevant to determining reference. We test our model on a small corpus of mother-infant interaction and find it performs better than competing models. Finally, we show that our model accounts for experimental phenomena including mutual exclusivity, fast-mapping, and generalization from social cues.

  BibTeX: @inproceedings{FrankEtAl2007-NIPS, author = {Frank, M. C. and Goodman, N. D. and Tenenbaum, J. B.}, journal = {Advances in Neural Information Processing Systems (NIPS 2007)}, publisher = {MIT Press}, title = {A bayesian framework for crosssituational word-learning}, volume = {20}, website = {papers/FrankEtAl2007-NIPS.pdf}, year = {2007} }

• Learning grounded causal models. [Abstract] [PDF] [BibTeX]
  N. D. Goodman, V. Mansinghka, & J. B. Tenenbaum. (2007). In Proceedings of the Twenty-Ninth Annual Conference of the Cognitive Science Society. [Winner of the 1007 Cognitive Science Society computational modeling prize for Perception and Action.]

  Abstract: We address the problem of learning grounded causal models: systems of concepts that are connected by causal relations and explicitly grounded in perception. We present a Bayesian framework for learning these models – both a causal Bayesian network structure over variables and the consequential region of each variable in perceptual space – from dynamic perceptual evidence. Using a novel experimental paradigm we show that humans are able to learn grounded causal models, and that the Bayesian model accounts well for human performance.

  BibTeX: @inproceedings{GoodmanEtAl2007-Cogsci, annote = {[Winner of the 1007 Cognitive Science Society computational modeling prize for Perception and Action.]}, author = {Goodman, Noah D. and Mansinghka, Vikash and Tenenbaum, Joshua B.}, booktitle = {Proceedings of the Twenty-Ninth Annual Conference of the {C}ognitive {S}cience {S}ociety}, title = {Learning grounded causal models}, website = {papers/GoodmanEtAl2007-Cogsci.pdf}, year = {2007} }

• Learning causal schemata. [Abstract] [PDF] [BibTeX]
  C. Kemp, N. D. Goodman, & J. B. Tenenbaum. (2007). In Proceedings of the Twenty-ninth Annual Meeting of the Cognitive Science Society. [Winner of the 2007 Cognitive Science Society computational modeling prize for Higher-level Cognition.]

  Abstract: Causal inferences about sparsely observed objects are often supported by causal schemata, or systems of abstract causal knowledge. We present a hierarchical Bayesian framework that learns simple causal schemata given only raw data as input. Given a set of objects and observations of causal events involving some of these objects, our framework simultaneously discovers the causal type of each object, the causal powers of these types, the characteristic features of these types, and the characteristic interactions between these types. Previous behavioral studies confirm that humans are able to discover causal schemata, and we show that our framework accounts for data collected by Lien and Cheng and Shanks and Darby.

  BibTeX: @inproceedings{KempEtAl2007-Cogsci, annote = {[Winner of the 2007 Cognitive Science Society computational modeling prize for Higher-level Cognition.]}, author = {Kemp, Charles and Goodman, Noah D. and Tenenbaum, Joshua B.}, booktitle = {Proceedings of the Twenty-ninth Annual Meeting of the {C}ognitive {S}cience {S}ociety}, title = {Learning causal schemata}, website = {papers/KempEtAl2007-Cogsci.pdf}, year = {2007} }

• Learning and using relational theories.. [Abstract] [PDF] [BibTeX]
  C. Kemp, N. D. Goodman, & J. B. Tenenbaum. (2007). In Advances in Neural Information Processing Systems (NIPS 2007).

  Abstract: Much of human knowledge is organized into sophisticated systems that are often called intuitive theories. We propose that intuitive theories are mentally represented in a logical language, and that the subjective complexity of a theory is determined by the length of its representation in this language. This complexity measure helps to explain how theories are learned from relational data, and how they support inductive inferences about unobserved relations. We describe two experiments that test our approach, and show that it provides a better account of human learning and reasoning than an approach developed by Goodman [1].

  BibTeX: @inproceedings{KempEtAl2007-NIPS, author = {Kemp, Charles and Goodman, Noah D and Tenenbaum, Joshua B}, booktitle = {Advances in Neural Information Processing Systems (NIPS 2007)}, title = {Learning and using relational theories.}, website = {papers/KempEtAl2007-NIPS.pdf}, year = {2007} }

Preprints

2006

• Intuitive theories of mind: a rational approach to false belief. [Abstract] [PDF] [BibTeX]
  N. D. Goodman, C. L. Baker, E. Baraff-Bonawitz, V. K. Mansinghka, A. Gopnik, H. Wellman, L. Schulz, & J. B. Tenenbaum. (2006). In Proceedings of the Twenty-Eight Annual Conference of the Cognitive Science Society.

  Abstract: We propose a rational analysis of children’s false belief reasoning. Our analysis realizes a continuous, evidencedriven transition between two causal Bayesian models of false belief. Both models support prediction and explanation; however, one model is less complex while the other has greater explanatory resources. Because of this explanatory asymmetry, unexpected outcomes weigh more heavily against the simpler model. We test this account empirically by showing children the standard outcome of the false belief task and a novel "psychic" outcome. As expected, we find children whose explanations and predictions are consistent with each model, and an interaction between prediction and explanation. Critically, we find unexpected outcomes only induce children to move from predictions consistent with the simpler model to those consistent with the more complex one, never the reverse.

  BibTeX: @inproceedings{GoodmanEtAl2006-Cogsci, author = {Goodman, Noah D. and Baker, Chris L. and Baraff-Bonawitz, Elizabeth and Mansinghka, Vikash K. and Gopnik, Alison and Wellman, Henry and Schulz, Laura and Tenenbaum, Joshua B.}, booktitle = {Proceedings of the Twenty-Eight Annual Conference of the {C}ognitive {S}cience {S}ociety}, title = {Intuitive theories of mind: a rational approach to false belief}, website = {papers/GoodmanEtAl2006-Cogsci.pdf}, year = {2006} }

Preprints