{
  "title": "The Compression Library: Ilya Sutskever’s reading list, shelved",
  "publisher": "Context Jamming / ACRA Insight",
  "author": "Bret Kerr",
  "published": "2026-11-04",
  "license": "CC BY 4.0 (https://creativecommons.org/licenses/by/4.0/). Cite as: Context Jamming, “The Compression Library,” 2026.",
  "page": "https://www.contextjamming.com/founder-files/sutskever/reading-list",
  "article": "https://www.contextjamming.com/founder-files/sutskever/reading-list/the-missing-elements",
  "provenance": [
    "John Carmack recalled receiving roughly 40 papers from Sutskever around 2020, with the promise that learning them would cover “90% of what matters.” The email was later lost.",
    "The 27 items that survive trace to an arc.net folder shared in May 2024.",
    "Sutskever has never confirmed any version of the list.",
    "The shelf assignments are Context Jamming’s classification, not his."
  ],
  "classification_note": "Shelf assignments are Context Jamming’s classification. Sutskever never sorted the list and has never confirmed any version of it.",
  "shelves": [
    {
      "id": "represent",
      "label": "Represent",
      "code": "REP",
      "definition": "How a network turns raw input into features: convolution and depth.",
      "count": 5
    },
    {
      "id": "route",
      "label": "Route",
      "code": "RTE",
      "definition": "Deciding which information flows where: attention and pointers.",
      "count": 4
    },
    {
      "id": "remember",
      "label": "Remember",
      "code": "MEM",
      "definition": "Holding state across time: gates and external memory.",
      "count": 4
    },
    {
      "id": "relate",
      "label": "Relate",
      "code": "REL",
      "definition": "Reasoning over entities and the relations between them.",
      "count": 3
    },
    {
      "id": "compress",
      "label": "Compress",
      "code": "CMP",
      "definition": "The theory: MDL, Kolmogorov complexity and regularization, or why the shortest description generalizes.",
      "count": 7
    },
    {
      "id": "scale",
      "label": "Scale",
      "code": "SCL",
      "definition": "Making models bigger, and measuring what bigger buys.",
      "count": 3
    },
    {
      "id": "evaluate",
      "label": "Evaluate",
      "code": "EVL",
      "definition": "Judging an action before its outcome arrives: value and reward.",
      "count": 1
    },
    {
      "id": "adapt",
      "label": "Adapt",
      "code": "ADP",
      "definition": "Learning continually after deployment.",
      "count": 0
    }
  ],
  "volumes": [
    {
      "call": "REP-01",
      "shelf": "represent",
      "kind": "ScholarlyArticle",
      "coauthor": true,
      "title": "ImageNet Classification with Deep Convolutional Neural Networks",
      "short": "AlexNet",
      "authors": "Krizhevsky, Sutskever, Hinton",
      "year": "2012",
      "url": "https://proceedings.neurips.cc/paper/2012/hash/c399862d3b9d6b76c8436e924a68c45b-Abstract.html",
      "significance": "GPUs, ReLU and dropout together; the start of the age of research.",
      "why": "Learned features beat hand-built ones: the network finds its own short description of an image instead of being handed one.",
      "code_length_bits_lz78": 1020
    },
    {
      "call": "REP-02",
      "shelf": "represent",
      "kind": "ScholarlyArticle",
      "title": "Multi-Scale Context Aggregation by Dilated Convolutions",
      "short": "Dilated Convolutions",
      "authors": "Yu & Koltun",
      "year": "2015",
      "url": "https://arxiv.org/abs/1511.07122",
      "significance": "An exponentially larger receptive field without losing resolution.",
      "why": "More context per parameter: the model sees farther for the same number of bits.",
      "code_length_bits_lz78": 630
    },
    {
      "call": "REP-03",
      "shelf": "represent",
      "kind": "ScholarlyArticle",
      "title": "Deep Residual Learning for Image Recognition",
      "short": "ResNet",
      "authors": "He, Zhang, Ren, Sun",
      "year": "2015",
      "url": "https://arxiv.org/abs/1512.03385",
      "significance": "Skip connections make very deep networks trainable.",
      "why": "Each block only has to encode what the identity map gets wrong, which is a cheaper thing to learn.",
      "code_length_bits_lz78": 784
    },
    {
      "call": "REP-04",
      "shelf": "represent",
      "kind": "ScholarlyArticle",
      "title": "Identity Mappings in Deep Residual Networks",
      "short": "Identity Mappings",
      "authors": "He, Zhang, Ren, Sun",
      "year": "2016",
      "url": "https://arxiv.org/abs/1603.05027",
      "significance": "Clean identity paths for signal propagation.",
      "why": "Keeping the shortcut clean lets depth add description without scrambling the signal underneath it.",
      "code_length_bits_lz78": 770
    },
    {
      "call": "REP-05",
      "shelf": "represent",
      "kind": "Course",
      "title": "CS231n: Convolutional Neural Networks for Visual Recognition",
      "short": "CS231n",
      "authors": "Stanford University",
      "year": "Course",
      "url": "https://cs231n.github.io/",
      "significance": "Backprop, CNNs and the practice of training, taught end to end.",
      "why": "The working toolkit: how to fit and regularize a model before any theory of why it generalizes.",
      "code_length_bits_lz78": 784
    },
    {
      "call": "RTE-01",
      "shelf": "route",
      "kind": "ScholarlyArticle",
      "title": "Neural Machine Translation by Jointly Learning to Align and Translate",
      "short": "Bahdanau Attention",
      "authors": "Bahdanau, Cho, Bengio",
      "year": "2014",
      "url": "https://arxiv.org/abs/1409.0473",
      "significance": "The original soft attention.",
      "why": "Instead of squeezing a sentence into one fixed vector, the decoder looks up only what it needs: a smarter bottleneck.",
      "code_length_bits_lz78": 990
    },
    {
      "call": "RTE-02",
      "shelf": "route",
      "kind": "ScholarlyArticle",
      "title": "Pointer Networks",
      "authors": "Vinyals, Fortunato, Jaitly",
      "year": "2015",
      "url": "https://arxiv.org/abs/1506.03134",
      "significance": "Attention used as the output: pointing at positions in the input.",
      "why": "Output by reference: pointing at something already in the input is a far shorter code than writing it out again.",
      "code_length_bits_lz78": 840
    },
    {
      "call": "RTE-03",
      "shelf": "route",
      "kind": "ScholarlyArticle",
      "title": "Attention Is All You Need",
      "short": "The Transformer",
      "authors": "Vaswani et al.",
      "year": "2017",
      "url": "https://arxiv.org/abs/1706.03762",
      "significance": "Attention, residuals and normalization as one general-purpose block.",
      "why": "One reusable routing block, later scaled into the best text compressors anyone has built.",
      "code_length_bits_lz78": 742
    },
    {
      "call": "RTE-04",
      "shelf": "route",
      "kind": "TechArticle",
      "title": "The Annotated Transformer",
      "authors": "Rush",
      "year": "2018",
      "url": "https://nlp.seas.harvard.edu/annotated-transformer/",
      "significance": "A line-by-line, executable Transformer; the list’s only learn-by-building item.",
      "why": "The theory made runnable: you learn the compressor by writing it yourself.",
      "code_length_bits_lz78": 644
    },
    {
      "call": "MEM-01",
      "shelf": "remember",
      "kind": "BlogPosting",
      "title": "The Unreasonable Effectiveness of Recurrent Neural Networks",
      "short": "Unreasonable Effectiveness of RNNs",
      "authors": "Karpathy",
      "year": "2015",
      "url": "https://karpathy.github.io/2015/05/21/rnn-effectiveness/",
      "significance": "Character-level prediction produces structure.",
      "why": "Predicting the next character is compressing the text, and structure shows up because predicting well requires modeling it.",
      "code_length_bits_lz78": 1005
    },
    {
      "call": "MEM-02",
      "shelf": "remember",
      "kind": "BlogPosting",
      "title": "Understanding LSTM Networks",
      "authors": "Olah",
      "year": "2015",
      "url": "https://colah.github.io/posts/2015-08-Understanding-LSTMs/",
      "significance": "Gating as learned memory control.",
      "why": "Gates decide what to keep and what to forget: learned control over a finite memory budget.",
      "code_length_bits_lz78": 742
    },
    {
      "call": "MEM-03",
      "shelf": "remember",
      "kind": "ScholarlyArticle",
      "title": "Neural Turing Machines",
      "authors": "Graves, Wayne, Danihelka",
      "year": "2014",
      "url": "https://arxiv.org/abs/1410.5401",
      "significance": "Differentiable external memory with content-based addressing.",
      "why": "Separating the program from its storage lets a short controller work over a long memory.",
      "code_length_bits_lz78": 714
    },
    {
      "call": "MEM-04",
      "shelf": "remember",
      "kind": "ScholarlyArticle",
      "title": "Relational Recurrent Neural Networks",
      "authors": "Santoro et al.",
      "year": "2018",
      "url": "https://arxiv.org/abs/1806.01822",
      "significance": "Memory slots that interact through multi-head attention.",
      "why": "Memories that attend to one another, so stored facts can be compared and not just recalled.",
      "code_length_bits_lz78": 728
    },
    {
      "call": "REL-01",
      "shelf": "relate",
      "kind": "ScholarlyArticle",
      "title": "Order Matters: Sequence to Sequence for Sets",
      "short": "Order Matters",
      "authors": "Vinyals, Bengio, Kudlur",
      "year": "2015",
      "url": "https://arxiv.org/abs/1511.06391",
      "significance": "Ordering changes what seq2seq can learn; attention over sets.",
      "why": "A set has no natural order, and the order you pick changes how hard the sequence is to model.",
      "code_length_bits_lz78": 742
    },
    {
      "call": "REL-02",
      "shelf": "relate",
      "kind": "ScholarlyArticle",
      "title": "A Simple Neural Network Module for Relational Reasoning",
      "short": "Relation Networks",
      "authors": "Santoro et al.",
      "year": "2017",
      "url": "https://arxiv.org/abs/1706.01427",
      "significance": "One pairwise relation function, summed over every pair of objects.",
      "why": "A single shared rule reused across all pairs: a compact law in place of a lookup table.",
      "code_length_bits_lz78": 686
    },
    {
      "call": "REL-03",
      "shelf": "relate",
      "kind": "ScholarlyArticle",
      "title": "Neural Message Passing for Quantum Chemistry",
      "short": "Message Passing",
      "authors": "Gilmer et al.",
      "year": "2017",
      "url": "https://arxiv.org/abs/1704.01212",
      "significance": "Unifies graph networks as message passing.",
      "why": "Many graph models turn out to be one model; the unification is itself a compression of the literature.",
      "code_length_bits_lz78": 784
    },
    {
      "call": "CMP-01",
      "shelf": "compress",
      "kind": "ScholarlyArticle",
      "title": "Keeping Neural Networks Simple by Minimizing the Description Length of the Weights",
      "short": "Description Length of the Weights",
      "authors": "Hinton & van Camp",
      "year": "1993",
      "url": "https://www.cs.toronto.edu/~hinton/absps/colt93.pdf",
      "significance": "MDL applied to network weights; the origin of bits-back coding. The list’s oldest paper.",
      "why": "The thesis, stated in 1993: the network that generalizes is the one whose weights take the fewest bits to send.",
      "code_length_bits_lz78": 826
    },
    {
      "call": "CMP-02",
      "shelf": "compress",
      "kind": "ScholarlyArticle",
      "title": "A Tutorial Introduction to the Minimum Description Length Principle",
      "short": "MDL Tutorial",
      "authors": "Grünwald",
      "year": "2004",
      "url": "https://arxiv.org/abs/math/0406077",
      "significance": "The best model is the best compressor.",
      "why": "The formal rule behind the whole list: choose the model that minimizes bits for the model plus bits for the data.",
      "code_length_bits_lz78": 840
    },
    {
      "call": "CMP-03",
      "shelf": "compress",
      "kind": "Book",
      "title": "Kolmogorov Complexity and Algorithmic Randomness (from p. 434)",
      "short": "Kolmogorov Complexity",
      "authors": "Shen, Uspensky, Vereshchagin",
      "year": "2017",
      "url": "https://www.lirmm.fr/~ashen/kolmbook-eng-scan.pdf",
      "significance": "The mathematical bedrock of the compression view.",
      "why": "The ideal compressor is the shortest program that prints the data; Sutskever’s 2023 Simons talk casts trained networks as small approximations of it.",
      "code_length_bits_lz78": 1200
    },
    {
      "call": "CMP-04",
      "shelf": "compress",
      "kind": "BlogPosting",
      "title": "The First Law of Complexodynamics",
      "authors": "Aaronson",
      "year": "2011",
      "url": "https://scottaaronson.blog/?p=762",
      "significance": "Why “interestingness” rises and then falls while entropy only rises.",
      "why": "It separates structure from noise: what is interesting is hard to compress without being random.",
      "code_length_bits_lz78": 756
    },
    {
      "call": "CMP-05",
      "shelf": "compress",
      "kind": "ScholarlyArticle",
      "title": "Quantifying the Rise and Fall of Complexity in Closed Systems: The Coffee Automaton",
      "short": "The Coffee Automaton",
      "authors": "Aaronson, Carroll, Ouellette",
      "year": "2014",
      "url": "https://arxiv.org/abs/1405.6903",
      "significance": "Complexity peaks at intermediate times.",
      "why": "It puts numbers on the blog post’s intuition, using compressed file size as the yardstick for complexity.",
      "code_length_bits_lz78": 854
    },
    {
      "call": "CMP-06",
      "shelf": "compress",
      "kind": "ScholarlyArticle",
      "coauthor": true,
      "title": "Variational Lossy Autoencoder",
      "authors": "Chen, Kingma, Salimans, Duan, Dhariwal, Schulman, Sutskever, Abbeel",
      "year": "2016",
      "url": "https://arxiv.org/abs/1611.02731",
      "significance": "A bits-back argument controls what the latent code stores. Sutskever co-authored it.",
      "why": "Bits-back reasoning decides what the latent keeps and what the decoder handles: Hinton’s 1993 idea, 23 years on.",
      "code_length_bits_lz78": 882
    },
    {
      "call": "CMP-07",
      "shelf": "compress",
      "kind": "ScholarlyArticle",
      "coauthor": true,
      "title": "Recurrent Neural Network Regularization",
      "short": "RNN Regularization",
      "authors": "Zaremba, Sutskever, Vinyals",
      "year": "2014",
      "url": "https://arxiv.org/abs/1409.2329",
      "significance": "Dropout on the non-recurrent connections; a generalization fix.",
      "why": "Regularization is a budget on description length, and this is the practical fix that let large LSTMs generalize.",
      "code_length_bits_lz78": 882
    },
    {
      "call": "SCL-01",
      "shelf": "scale",
      "kind": "ScholarlyArticle",
      "title": "GPipe: Efficient Training of Giant Neural Networks using Pipeline Parallelism",
      "short": "GPipe",
      "authors": "Huang et al.",
      "year": "2018",
      "url": "https://arxiv.org/abs/1811.06965",
      "significance": "Pipeline parallelism splits one giant model across many accelerators.",
      "why": "The engineering that lets a compressor grow past the memory of a single chip.",
      "code_length_bits_lz78": 644
    },
    {
      "call": "SCL-02",
      "shelf": "scale",
      "kind": "ScholarlyArticle",
      "title": "Deep Speech 2: End-to-End Speech Recognition in English and Mandarin",
      "short": "Deep Speech 2",
      "authors": "Amodei et al.",
      "year": "2015",
      "url": "https://arxiv.org/abs/1512.02595",
      "significance": "An early empirical case for scaling.",
      "why": "One end-to-end model that kept improving with more data and compute, before scaling laws had a name.",
      "code_length_bits_lz78": 826
    },
    {
      "call": "SCL-03",
      "shelf": "scale",
      "kind": "ScholarlyArticle",
      "title": "Scaling Laws for Neural Language Models",
      "short": "Scaling Laws",
      "authors": "Kaplan et al.",
      "year": "2020",
      "url": "https://arxiv.org/abs/2001.08361",
      "significance": "Loss follows power laws; the charter of the age of scaling.",
      "why": "Loss in nats is code length, so a scaling law is a law of how compression improves with resources.",
      "code_length_bits_lz78": 742
    },
    {
      "call": "EVL-01",
      "shelf": "evaluate",
      "kind": "Thesis",
      "title": "Machine Super Intelligence (PhD thesis)",
      "short": "Machine Super Intelligence",
      "authors": "Legg",
      "year": "2008",
      "url": "https://www.vetta.org/documents/Machine_Super_Intelligence.pdf",
      "significance": "A formal measure of universal intelligence with a simplicity prior; the list’s only reward and agent framework.",
      "why": "Intelligence as reward earned across all environments, weighted toward the simple ones. The only volume on this shelf.",
      "code_length_bits_lz78": 975
    }
  ],
  "vacancies": [
    {
      "id": "evl-v1",
      "call": "— VACANT —",
      "shelf": "evaluate",
      "subject": "Value functions",
      "note": "Not on the list. In his Nov 2025 Dwarkesh interview, Sutskever put value functions and continual learning at the center of SSI’s problem.",
      "source": "https://www.dwarkesh.com/p/ilya-sutskever-2"
    },
    {
      "id": "evl-v2",
      "call": "— VACANT —",
      "shelf": "evaluate",
      "subject": "Reinforcement learning from experience",
      "note": "Not on the list. In his Nov 2025 Dwarkesh interview, Sutskever put value functions and continual learning at the center of SSI’s problem.",
      "source": "https://www.dwarkesh.com/p/ilya-sutskever-2"
    },
    {
      "id": "adp-v1",
      "call": "— VACANT —",
      "shelf": "adapt",
      "subject": "Continual learning",
      "note": "Not on the list. In his Nov 2025 Dwarkesh interview, Sutskever put value functions and continual learning at the center of SSI’s problem.",
      "source": "https://www.dwarkesh.com/p/ilya-sutskever-2"
    },
    {
      "id": "adp-v2",
      "call": "— VACANT —",
      "shelf": "adapt",
      "subject": "Learning after deployment",
      "note": "Not on the list. In his Nov 2025 Dwarkesh interview, Sutskever put value functions and continual learning at the center of SSI’s problem.",
      "source": "https://www.dwarkesh.com/p/ilya-sutskever-2"
    },
    {
      "id": "adp-v3",
      "call": "— VACANT —",
      "shelf": "adapt",
      "subject": "Alignment to sentient life",
      "note": "Not on the list. In his Nov 2025 Dwarkesh interview, Sutskever put value functions and continual learning at the center of SSI’s problem.",
      "source": "https://www.dwarkesh.com/p/ilya-sutskever-2"
    }
  ],
  "community_additions": {
    "note": "Added by the PaperTrace “top 30” version; not on the 27-item list. At least two postdate any plausible handoff to Carmack.",
    "source": "https://yayajjiang.github.io/PaperTrace/papers/top30",
    "items": [
      {
        "call": "ADD-01",
        "title": "Better & Faster Large Language Models via Multi-token Prediction",
        "authors": "Gloeckle et al.",
        "year": "2024",
        "url": "https://arxiv.org/abs/2404.19737",
        "significance": "Trains a model to predict several future tokens at once.",
        "note": "Published in 2024, years after any plausible handoff to Carmack.",
        "code_length_bits_lz78": 616
      },
      {
        "call": "ADD-02",
        "title": "Dense Passage Retrieval for Open-Domain Question Answering",
        "authors": "Karpukhin et al.",
        "year": "2020",
        "url": "https://arxiv.org/abs/2004.04906",
        "significance": "Dense vector retrieval for open-domain question answering.",
        "note": "A 2020 paper: possible by date, but absent from the surviving 27.",
        "code_length_bits_lz78": 602
      },
      {
        "call": "ADD-03",
        "title": "Retrieval-Augmented Generation for Knowledge-Intensive NLP Tasks",
        "authors": "Lewis et al.",
        "year": "2020",
        "url": "https://arxiv.org/abs/2005.11401",
        "significance": "Generation conditioned on retrieved documents (RAG).",
        "note": "A 2020 paper: possible by date, but absent from the surviving 27.",
        "code_length_bits_lz78": 602
      },
      {
        "call": "ADD-04",
        "title": "Zephyr: Direct Distillation of LM Alignment",
        "authors": "Tunstall et al.",
        "year": "2023",
        "url": "https://arxiv.org/abs/2310.16944",
        "significance": "Distilled preference alignment for a small chat model.",
        "note": "Published in 2023, after any plausible handoff to Carmack.",
        "code_length_bits_lz78": 560
      }
    ]
  }
}