js-tiktoken count and encode: working versions (tested 2026-10-18)

{
  "name": "js-tiktoken-count-encode",
  "private": true,
  "type": "module",
  "scripts": {
    "test": "node --import tsx --test test/*.test.ts"
  },
  "dependencies": {
    "js-tiktoken": "1.0.21"
  },
  "devDependencies": {
    "@types/node": "26.6.5",
    "tsx": "4.23.15"
  }
}

import { Tiktoken } from "js-tiktoken/lite";
import cl100k_base from "js-tiktoken/ranks/cl100k_base";
import o200k_base from "js-tiktoken/ranks/o200k_base";

export type EncodingName = "cl100k_base" | "o200k_base";

const RANKS: Record<EncodingName, Record<string, number>> = {
  cl100k_base,
  o200k_base,
};

export function createEncoding(name: EncodingName): Tiktoken {
  return new Tiktoken(RANKS[name]);
}

export type ChatMessage = {
  role: string;
  content: string;
  name?: string;
};

export function countChatTokens(enc: Tiktoken, messages: ChatMessage[]): number {
  const tokensPerMessage = 3;
  const tokensPerName = 1;
  let total = 0;
  for (const message of messages) {
    total += tokensPerMessage;
    total += enc.encode(message.role).length;
    total += enc.encode(message.content).length;
    if (message.name !== undefined) {
      total += enc.encode(message.name).length;
      total += tokensPerName;
    }
  }
  total += 3;
  return total;
}

export function truncateToTokenBudget(enc: Tiktoken, text: string, budget: number): string {
  const tokens = enc.encode(text);
  if (tokens.length <= budget) {
    return text;
  }
  return enc.decode(tokens.slice(0, Math.max(0, budget)));
}

import { test } from "node:test";
import assert from "node:assert/strict";
import { createEncoding, countChatTokens, truncateToTokenBudget } from "../src/tokenizer.ts";

test("encode/decode roundtrip is lossless for bundled ranks", () => {
  for (const name of ["cl100k_base", "o200k_base"] as const) {
    const enc = createEncoding(name);
    const text = "ProvenPin recipes run offline with deterministic output.";
    const tokens = enc.encode(text);
    assert.equal(enc.decode(tokens), text);
  }
});

test("token counts are deterministic across instances", () => {
  const text = "count these tokens exactly once";
  const first = createEncoding("cl100k_base").encode(text);
  const second = createEncoding("cl100k_base").encode(text);
  assert.deepEqual(first, second);
});

test("counts chat messages deterministically", () => {
  const enc = createEncoding("cl100k_base");
  const messages = [
    { role: "system", content: "You are a helpful assistant." },
    { role: "user", content: "Count my tokens." },
  ];
  const first = countChatTokens(enc, messages);
  const second = countChatTokens(enc, messages);
  assert.equal(first, second);
  const contentOnly = messages.reduce((sum, m) => sum + enc.encode(m.content).length, 0);
  assert.ok(first > contentOnly);
});

test("truncates text to a token budget at a token boundary", () => {
  const enc = createEncoding("o200k_base");
  const text = "The quick brown fox jumps over the lazy dog and keeps on running.";
  const budget = 5;
  const truncated = truncateToTokenBudget(enc, text, budget);
  assert.ok(truncated.length < text.length);
  assert.ok(enc.encode(truncated).length <= budget);
});

test("keeps text unchanged when it fits the budget", () => {
  const enc = createEncoding("cl100k_base");
  const text = "short text";
  assert.equal(truncateToTokenBudget(enc, text, 100), text);
});