Web scraping in C#

dotnet new console -o BookScraper && cd BookScraper
dotnet add package RecursiveAI.OneHtml --version 1.5.1

using RecursiveAI.OneHtml;
 
using var http = new HttpClient();
http.DefaultRequestHeaders.UserAgent
    .ParseAdd("BookPriceBot/1.0 (+https://example.com/bot)");
 
var pageUrl = new Uri("https://books.toscrape.com/");
using HttpResponseMessage response = await http.GetAsync(pageUrl);
response.EnsureSuccessStatusCode();
 
// Pass the bytes and the Content-Type: the parser picks the encoding as a browser does
byte[] body = await response.Content.ReadAsByteArrayAsync();
string? type = response.Content.Headers.ContentType?.ToString();
var options = new ParseOptions { ContentType = type };
Document doc = Document.Parse(body, options);
 
List<Node> books = doc.Root.QuerySelectorAll("article.product_pod");
Console.WriteLine($"{books.Count} books on {pageUrl}");
 
foreach (Node book in books.Take(4))
{
    Node link = book.QuerySelector("h3 > a");
    string title = link.GetAttribute("title")!;   // the link text is cut short
    string price = book.QuerySelector(".price_color").TextContent!;
    string stock = book.QuerySelector(".availability").TextContent!.Trim();
    var url = new Uri(pageUrl, link.GetAttribute("href"));   // a relative link
    Console.WriteLine($"{price,7}  {stock}  {title}");
    Console.WriteLine($"         {url}");
}
 
// :has() filters on what an element contains, so no loop is needed
int fiveStars = doc.Root
    .QuerySelectorAll("article.product_pod:has(.star-rating.Five)").Count;
Console.WriteLine($"Five-star books on this page: {fiveStars}");

20 books on https://books.toscrape.com/
 £51.77  In stock  A Light in the Attic
         https://books.toscrape.com/catalogue/a-light-in-the-attic_1000/index.html
 £53.74  In stock  Tipping the Velvet
         https://books.toscrape.com/catalogue/tipping-the-velvet_999/index.html
 £50.10  In stock  Soumission
         https://books.toscrape.com/catalogue/soumission_998/index.html
 £47.82  In stock  Sharp Objects
         https://books.toscrape.com/catalogue/sharp-objects_997/index.html
Five-star books on this page: 4

using System.Globalization;
using RecursiveAI.OneHtml;
 
using var http = new HttpClient();
http.DefaultRequestHeaders.UserAgent
    .ParseAdd("BookPriceBot/1.0 (+https://example.com/bot)");
 
var rows = new List<string> { "title,price_gbp,url" };
Uri? next = new Uri(
    "https://books.toscrape.com/catalogue/category/books/mystery_3/index.html");
int pages = 0;
 
while (next is not null && pages < 10)
{
    Document doc = await FetchAsync(next);
    pages++;
 
    foreach (Node book in doc.Root.QuerySelectorAll("article.product_pod"))
    {
        Node link = book.QuerySelector("h3 > a");
        string price = book.QuerySelector(".price_color").TextContent!.TrimStart('£');
        decimal gbp = decimal.Parse(price, CultureInfo.InvariantCulture);
        string title = link.GetAttribute("title")!.Replace("\"", "\"\"");
        var url = new Uri(next, link.GetAttribute("href"));
        rows.Add($"\"{title}\",{gbp.ToString(CultureInfo.InvariantCulture)},{url}");
    }
 
    Console.WriteLine($"Page {pages}: {next.Segments[^1]}, {rows.Count - 1} books");
    Node nextLink = doc.Root.QuerySelector("li.next > a");
    next = nextLink.IsNull ? null : new Uri(next, nextLink.GetAttribute("href"));
 
    await Task.Delay(TimeSpan.FromSeconds(1));   // one request a second
}
 
File.WriteAllLines("mystery.csv", rows);
Console.WriteLine($"Wrote {rows.Count - 1} books to mystery.csv");
Console.WriteLine(rows[1]);
 
async Task<Document> FetchAsync(Uri url)
{
    using HttpResponseMessage response = await http.GetAsync(url);
    response.EnsureSuccessStatusCode();
    byte[] body = await response.Content.ReadAsByteArrayAsync();
    string? type = response.Content.Headers.ContentType?.ToString();
    return Document.Parse(body, new ParseOptions { ContentType = type });
}

Page 1: index.html, 20 books
Page 2: page-2.html, 32 books
Wrote 32 books to mystery.csv
"Sharp Objects",47.82,https://books.toscrape.com/catalogue/sharp-objects_997/index.html

dotnet add package Microsoft.Playwright --version 1.64.1

using Microsoft.Playwright;
using RecursiveAI.OneHtml;
 
const string url = "https://quotes.toscrape.com/js/";
 
// The HTML the server sends: a script builds the quotes later, in the browser
using var http = new HttpClient();
Document raw = Document.Parse(await http.GetByteArrayAsync(url));
int inHtml = raw.Root.QuerySelectorAll(".quote").Count;
Console.WriteLine($"Quotes in the server's HTML: {inHtml}");
 
// Downloads Chromium the first time; does nothing once it is there
Microsoft.Playwright.Program.Main(["install", "chromium"]);
using IPlaywright playwright = await Playwright.CreateAsync();
await using IBrowser browser = await playwright.Chromium.LaunchAsync();
IPage page = await browser.NewPageAsync();
await page.GotoAsync(url);
await page.WaitForSelectorAsync(".quote");
 
Document rendered = Document.Parse(await page.ContentAsync());
List<Node> quotes = rendered.Root.QuerySelectorAll(".quote");
Console.WriteLine($"Quotes in the rendered DOM:  {quotes.Count}");
foreach (Node quote in quotes.Take(3))
{
    string author = quote.QuerySelector(".author").TextContent!;
    var tags = quote.QuerySelectorAll(".tag").Select(t => t.TextContent);
    Console.WriteLine($"{author}: {string.Join(", ", tags)}");
}

Quotes in the server's HTML: 0
Quotes in the rendered DOM:  10
Albert Einstein: change, deep-thoughts, thinking, world
J.K. Rowling: abilities, choices
Albert Einstein: inspirational, life, live, miracle, miracles