Reads like

import NaturalLanguage
 
let text = "Angela Merkel met Emmanuel Macron in Paris on 5 March 2026."
 
let recognizer = NLLanguageRecognizer()
recognizer.processString(text)
let language = recognizer.dominantLanguage                 // .english
 
let words = NLTokenizer(unit: .word)
words.string = text
words.enumerateTokens(in: text.startIndex..<text.endIndex) { range, _ in
    print(text[range]); return true
}
 
let tagger = NLTagger(tagSchemes: [.nameType])
tagger.string = text
tagger.enumerateTags(in: text.startIndex..<text.endIndex, unit: .word, scheme: .nameType,
                     options: [.omitWhitespace, .omitPunctuation, .joinNames]) { tag, range in
    if let tag, tag != .otherWord { print(text[range], tag) }
    return true
}
 
let vector = NLEmbedding.sentenceEmbedding(for: .english)?.vector(for: text)

string text = "Angela Merkel met Emmanuel Macron in Paris on 5 March 2026. The deal cost $5m.";
 
Language language = Nlp.DetectLanguage(text);                    // Language.English
IReadOnlyList<Span> sentences = Nlp.Sentences(text);             // sentence spans, offsets into text
Token[] words = Nlp.Tokens(text);                                // word tokens
string[] lemmas = Nlp.Lemmas("The cats were running.");          // "the", "cat", "be", "run", "."
IReadOnlyList<Span> entities = Nlp.Entities(text);               // "Angela Merkel" PER, "Paris" LOC, ...
 
DateTimeOffset today = new(2026, 3, 18, 10, 30, 0, TimeSpan.Zero);
DateRange? when = Nlp.ParseDate("the first Monday after Easter 2027", today);
IEnumerable<Quantity> money = Nlp.Quantities(text);              // 5000000 USD

float[] question = Nlp.Embed("how do I reset my password");        // 384 floats, length 1 (normalized)
float[] related = Nlp.Embed("I forgot my login credentials");
float[] unrelated = Nlp.Embed("the best way to bake sourdough bread");
 
static float Dot(float[] a, float[] b) => a.Zip(b, (x, y) => x * y).Sum();

import spacy
from spacy.matcher import Matcher
 
nlp = spacy.blank("en")
ruler = nlp.add_pipe("entity_ruler", config={"overwrite_ents": True})
ruler.add_patterns([
    {"label": "CASE_NO", "pattern": [{"SHAPE": "dd-dddd"}]},
    {"label": "ORG", "pattern": "Recursive AI"},
])
 
doc = nlp("Recursive AI filed case 21-1234 on Monday.")
for ent in doc.ents:
    print(ent.label_, ent.text, ent.start_char, ent.end_char)
 
matcher = Matcher(nlp.vocab)
matcher.add("GREETING", [[{"LOWER": "hello"}, {"IS_PUNCT": True, "OP": "?"}, {"LOWER": "world"}]])
matches = matcher(nlp("Hello, world! hello world"))
 
for d in nlp.pipe(["First text.", "Second text."]):
    print(len(list(d.sents)))

// A blank English pipeline: the Unicode word tokenizer, then the components in the order they are added.
using Pipeline nlp = Pipeline.Blank(Language.English);
 
// Rule entities, from token patterns (the vocabulary of spaCy's rule-based matching) or from phrases.
EntityRuler ruler = new() { OverwriteEnts = true };
ruler.Add("CASE_NO", Pattern.Shape("dd-dddd"));
ruler.AddPhrase("ORG", "Recursive AI");
nlp.Add(ruler);
 
Doc doc = nlp.Process("Recursive AI filed case 21-1234 on Monday.");
foreach (Span e in doc.Ents)
{
    Console.WriteLine($"{e.Label} {e} {e.StartChar}..{e.EndChar}");
}
 
// The Matcher finds token patterns in any Doc.
Matcher matcher = new();
matcher.Add("GREETING", Pattern.Token(new TokenSpec().Lower("hello"), new TokenSpec().Is(TokenProperty.IsPunct, true).Optional(), new TokenSpec().Lower("world")));
IReadOnlyList<PatternMatch> matches = matcher.Match(nlp.Process("Hello, world! hello world"));
 
// Many texts at once, lazily, in batches.
foreach (Doc d in nlp.ProcessMany(["First text.", "Second text."]))
{
    Console.WriteLine(d.Sentences.Count);
}

TrainResult result = nlp.Train(examples, new TrainOptions { Epochs = 30, Seed = 7, Dropout = 0f, DevExamples = examples });
PipelineFile.Save(nlp, path);                                  // one file: tokenizer, encoder, heads, labels

from tokenizers import Tokenizer
 
tok = Tokenizer.from_file("tokenizer.json")
enc = tok.encode("Hello, world! Tokenizers count in UTF-16 units.")
for i, t in enumerate(enc.tokens):
    print(enc.ids[i], t, enc.offsets[i])
 
count = len(tok.encode("How long is this text, in tokens?").ids)
text = tok.decode(enc.ids)                       # skip_special_tokens=True by default

Tokenizer tok = Tokenizer.FromFile(tokenizerJson);                  // tokenizer.json, SentencePiece .model, vocab.txt, .tiktoken
 
// Ids, tokens and character offsets into the input, as the Hugging Face Encoding has them.
TokenEncoding enc = tok.Encode("Hello, world! Tokenizers count in UTF-16 units.");
for (int i = 0; i < enc.Length; i++)
{
    Console.WriteLine($"{enc.Ids[i],6} {enc.Tokens[i],-12} {enc.Offsets[i].Start}..{enc.Offsets[i].End}");
}
 
// The span form: ids go into your buffer, nothing is allocated, and the status says whether it all fit.
int[] buffer = new int[8];
OperationStatus status = tok.EncodeToIds("Hello, world! Tokenizers count in UTF-16 units.", buffer, out int written, out int consumed);
 
int count = tok.CountTokens("How long is this text, in tokens?");
string back = tok.Decode(enc.Ids);                                  // special tokens skipped
ReadOnlySpan<char> budget = tok.TrimToTokenBudget("one two three four five six seven eight", 4);