/19 mins read
Kimi K3Engineering GuidePart 2/6
#attention
Things I've written about #attention
- #kafka(23)
- #machine-learning(19)
- #distributed-systems(17)
- #performance(14)
- #nanochat(12)
- #llm(10)
- #architecture(8)
- #redpanda(8)
- #zero-copy(8)
- #ai(7)
- #reliability(7)
- #tutorial(7)
- #linux(7)
- #quantization(6)
- #kimi-k3(6)
- #practical-guide(6)
- #optimization(6)
- #ai-engineering(5)
- #deep-learning(5)
- #streaming(4)
- #model-distillation(4)
- #distillation(4)
- #networking(4)
- #inference(4)
- #protocol(4)
- #pytorch(4)
- #transformers(4)
- #training(4)
- #leadership(3)
- #safety(3)
- #engineering(3)
- #decision-making(3)
- #vendor-selection(3)
- #production-systems(3)
- #nextjs(3)
- #protocols(3)
- #compatibility(3)
- #model-compression(3)
- #lora(3)
- #mixture-of-experts(3)
- #benchmarking(3)
- #idempotency(3)
- #consumers(3)
- #benchmarks(3)
- #systems(3)
- #security(2)
- #react(2)
- #consumer-groups(2)
- #platform-engineering(2)
- #pruning(2)
- #cost-optimization(2)
- #seastar(2)
- #raft(2)
- #latency(2)
- #replication(2)
- #serialization(2)
- #storage(2)
- #model-audit(2)
- #reproducibility(2)
- #observability(2)
- #distributed-inference(2)
- #migration(2)
- #gpu(2)
- #attention(2)
- #production(2)
- #mermaid(2)
- #pragmatic-engineering(2)
- #evaluation(2)
- #fine-tuning(2)
- #tokenization(2)
- #edge-computing(2)
- #io-uring(2)
- #rce(1)
- #cve(1)
- #javascript(1)
- #web-security(1)
- #debugging(1)
- #rust(1)
- #ibm(1)
- #confluent(1)
- #infrastructure(1)
- #migrations(1)
- #introduction(1)
- #scalability(1)
- #knowledge-distillation(1)
- #ai-safety(1)
- #mlops(1)
- #synthetic-data(1)
- #data-quality(1)
- #python(1)
- #enterprise-ai(1)
- #cluster-management(1)
- #controller(1)
- #consensus(1)
- #rpc(1)
- #c(1)
- #log-structured-storage(1)
- #tiered-storage(1)
- #object-storage(1)
- #retention(1)
- #wasm(1)
- #transforms(1)
- #data-processing(1)
- #open-weights(1)
- #supply-chain(1)
- #ai-agents(1)
- #authorization(1)
- #long-context(1)
- #kv-cache(1)
- #retrieval(1)
- #multi-gpu(1)
- #tensor-parallelism(1)
- #expert-parallelism(1)
- #gpu-topology(1)
- #kraft(1)
- #operations(1)
- #internals(1)
- #idempotence(1)
- #transactions(1)
- #compression(1)
- #zstd(1)
- #lz4(1)
- #multi-tenant(1)
- #concurrency(1)
- #outbox(1)
- #retries(1)
- #recovery(1)
- #queues(1)
- #hardware(1)
- #api(1)
- #tool-calling(1)
- #context-caching(1)
- #mxfp4(1)
- #model-weights(1)
- #checkpoint(1)
- #api-pricing(1)
- #self-hosting(1)
- #experiments(1)
- #ablation(1)
- #model-architecture(1)
- #case-study(1)
- #build-optimization(1)
- #static-site-generation(1)
- #remark-plugins(1)
- #custom-hooks(1)
- #pan-zoom(1)
- #svg-rendering(1)
- #over-engineering(1)
- #feature-creep(1)
- #css(1)
- #simplification(1)
- #refactoring(1)
- #mermaid-tutorial(1)
- #lessons-learned(1)
- #core(1)
- #testing(1)
- #distributed-training(1)
- #zero(1)
- #sft(1)
- #chat(1)
- #scaling-laws(1)
- #memory-optimization(1)
- #rlhf(1)
- #reinforcement-learning(1)
- #grpo(1)
- #policy-gradient(1)
- #bpe(1)
- #tiktoken(1)
- #data-engineering(1)
- #beginner(1)
- #tiny-llm(1)
- #efficient-models(1)
- #mobile-ai(1)
- #mathematics(1)
- #comparison(1)
- #deployment-patterns(1)
- #deployment(1)
- #raspberry-pi(1)
- #mobile(1)
- #qlora(1)
- #domain-adaptation(1)
- #tls(1)
- #dpdk(1)
- #spdk(1)
- #rdma(1)
- #gpudirect(1)
- #memory(1)
/26 mins readTiny Language ModelsFoundations & ArchitecturePart 4/10Efficient Attention Mechanisms for Tiny Language Models