Web scraping in C#

dotnet new console -o BookScraper
cd BookScraper
dotnet add package RecursiveAI.OneHtml --version 1.0.0

using RecursiveAI.OneHtml;
 
using var http = new HttpClient();
http.DefaultRequestHeaders.UserAgent.ParseAdd("BookPriceBot/1.0 (+https://example.com/bot)");
 
var pageUrl = new Uri("https://books.toscrape.com/");
using HttpResponseMessage response = await http.GetAsync(pageUrl);
response.EnsureSuccessStatusCode();
 
// Pass the bytes and the Content-Type: the parser picks the encoding as a browser would
byte[] body = await response.Content.ReadAsByteArrayAsync();
Document doc = Document.Parse(body, new ParseOptions { ContentType = response.Content.Headers.ContentType?.ToString() });
 
List<Node> books = doc.Root.QuerySelectorAll("article.product_pod");
Console.WriteLine($"{books.Count} books on {pageUrl}");
 
foreach (Node book in books.Take(5))
{
    Node link = book.QuerySelector("h3 > a");
    string title = link.GetAttribute("title")!;                          // the link text is truncated, the title isn't
    string price = book.QuerySelector(".price_color").TextContent!;
    string rating = book.QuerySelector("p.star-rating").GetAttribute("class")!.Split(' ')[1];
    string stock = book.QuerySelector(".availability").TextContent!.Trim();
    var url = new Uri(pageUrl, link.GetAttribute("href"));                // resolve the relative link
    Console.WriteLine($"{price,7}  {rating,-5}  {stock}  {title}");
    Console.WriteLine($"         {url}");
}
 
// :has() filters on what an element contains, so no loop is needed
int fiveStars = doc.Root.QuerySelectorAll("article.product_pod:has(p.star-rating.Five)").Count;
Console.WriteLine($"Five-star books on this page: {fiveStars}");

20 books on https://books.toscrape.com/
 £51.77  Three  In stock  A Light in the Attic
         https://books.toscrape.com/catalogue/a-light-in-the-attic_1000/index.html
 £53.74  One    In stock  Tipping the Velvet
         https://books.toscrape.com/catalogue/tipping-the-velvet_999/index.html
 £50.10  One    In stock  Soumission
         https://books.toscrape.com/catalogue/soumission_998/index.html
 £47.82  Four   In stock  Sharp Objects
         https://books.toscrape.com/catalogue/sharp-objects_997/index.html
 £54.23  Five   In stock  Sapiens: A Brief History of Humankind
         https://books.toscrape.com/catalogue/sapiens-a-brief-history-of-humankind_996/index.html
Five-star books on this page: 4

using System.Globalization;
using RecursiveAI.OneHtml;
 
using var http = new HttpClient();
http.DefaultRequestHeaders.UserAgent.ParseAdd("BookPriceBot/1.0 (+https://example.com/bot)");
 
var rows = new List<string> { "title,price_gbp,url" };
Uri? next = new Uri("https://books.toscrape.com/catalogue/category/books/mystery_3/index.html");
int pages = 0;
 
while (next is not null && pages < 10)
{
    Document doc = await FetchAsync(next);
    pages++;
 
    foreach (Node book in doc.Root.QuerySelectorAll("article.product_pod"))
    {
        Node link = book.QuerySelector("h3 > a");
        decimal price = decimal.Parse(book.QuerySelector(".price_color").TextContent!.TrimStart('£'), CultureInfo.InvariantCulture);
        var url = new Uri(next, link.GetAttribute("href"));
        rows.Add($"\"{link.GetAttribute("title")!.Replace("\"", "\"\"")}\",{price.ToString(CultureInfo.InvariantCulture)},{url}");
    }
 
    // Follow the "next" link until there isn't one
    Node nextLink = doc.Root.QuerySelector("li.next > a");
    Console.WriteLine($"Page {pages}: {next.Segments[^1]}, {rows.Count - 1} books so far");
    next = nextLink.IsNull ? null : new Uri(next, nextLink.GetAttribute("href"));
 
    await Task.Delay(TimeSpan.FromSeconds(1));   // one request a second keeps the load on the site low
}
 
File.WriteAllLines("mystery.csv", rows);
Console.WriteLine($"Wrote {rows.Count - 1} books to mystery.csv");
 
async Task<Document> FetchAsync(Uri url)
{
    using HttpResponseMessage response = await http.GetAsync(url);
    response.EnsureSuccessStatusCode();
    byte[] body = await response.Content.ReadAsByteArrayAsync();
    return Document.Parse(body, new ParseOptions { ContentType = response.Content.Headers.ContentType?.ToString() });
}

Page 1: index.html, 20 books so far
Page 2: page-2.html, 32 books so far
Wrote 32 books to mystery.csv

title,price_gbp,url
"Sharp Objects",47.82,https://books.toscrape.com/catalogue/sharp-objects_997/index.html
"In a Dark, Dark Wood",19.63,https://books.toscrape.com/catalogue/in-a-dark-dark-wood_963/index.html
"The Past Never Ends",56.50,https://books.toscrape.com/catalogue/the-past-never-ends_942/index.html

dotnet add package Microsoft.Playwright --version 1.64.1
dotnet build
pwsh bin/Debug/net10.0/playwright.ps1 install chromium

using Microsoft.Playwright;
using RecursiveAI.OneHtml;
 
const string url = "https://quotes.toscrape.com/js/";
 
// The HTML the server sends has no quotes in it: a script builds them in the browser
using var http = new HttpClient();
Document raw = Document.Parse(await http.GetByteArrayAsync(url));
Console.WriteLine($"Quotes in the server's HTML: {raw.Root.QuerySelectorAll(".quote").Count}");
 
// Let a real browser run the script, then parse the DOM it built
using IPlaywright playwright = await Playwright.CreateAsync();
await using IBrowser browser = await playwright.Chromium.LaunchAsync();
IPage page = await browser.NewPageAsync();
await page.GotoAsync(url);
await page.WaitForSelectorAsync(".quote");
Document rendered = Document.Parse(await page.ContentAsync());
 
List<Node> quotes = rendered.Root.QuerySelectorAll(".quote");
Console.WriteLine($"Quotes in the rendered DOM:  {quotes.Count}");
foreach (Node quote in quotes.Take(3))
{
    string author = quote.QuerySelector(".author").TextContent!;
    string tags = string.Join(", ", quote.QuerySelectorAll(".tag").Select(t => t.TextContent));
    Console.WriteLine($"{author}: {tags}");
}

Quotes in the server's HTML: 0
Quotes in the rendered DOM:  10
Albert Einstein: change, deep-thoughts, thinking, world
J.K. Rowling: abilities, choices
Albert Einstein: inspirational, life, live, miracle, miracles