Stock counts in batches: one transaction, a failed batch skipped

Runs the sample. Generated for this page; it reads the connection string from the environment.

using ArxSamples.MSSQL;
using ArxSamples.MSSQL.Operations.UpdateMultiple.AdvancedCases;

// The connection string comes from the environment, never from the code (see "Configure the connection").
var connectionString = Environment.GetEnvironmentVariable("ARX_SAMPLES_SQLSERVER")
    ?? throw new InvalidOperationException("Set the ARX_SAMPLES_SQLSERVER environment variable to your SQL Server connection string.");

await new Advanced1_ImportUpdatesInBatches().RunAsync(new SampleContext(connectionString));

The sample, as it is in the repository.

using ARX.SDK;
using ARX.SDK.Services;
using static ArxSamples.MSSQL.SampleOutput;

namespace ArxSamples.MSSQL.Operations.UpdateMultiple.AdvancedCases;

/// <summary>
/// A stock count file updates products in batches, in two scenarios: every batch in one transaction with
/// automatic commit; and a batch that fails, skipped while the others are kept (savepoints, manual commit).
/// </summary>
public sealed class Advanced1_ImportUpdatesInBatches : ISample
{
    private const int PRODUCTS = 2_000;
    private const int BATCH = 500;

    public string Name => "update-multiple-advanced-1-import-updates-in-batches";

    public string Title => "Stock counts in batches: one transaction, a failed batch skipped";

    public async Task RunAsync(SampleContext dbContext)
    {
        // Two warehouses, A and B, one per scenario, so each scenario's check holds after the whole sample has run.
        foreach (var warehouse in new[] { "A", "B" })
        {
            var products = Enumerable.Range(1, PRODUCTS).Select(i =>
            {
                var product = new ArxEntity("product");
                product["sku"] = $"SKU-U9{warehouse}{i:00000}";
                product["name"] = $"Counted product {warehouse}{i}";
                product["unit_price"] = 5.00m;
                return product;
            }).ToList();
            await dbContext.Service.CreateMultipleAsync(products);
        }

        await Scenario1_AutomaticCommitAsync(dbContext);

        await Scenario2_AFailedBatchIsSkippedAsync(dbContext);
    }

    /// <summary>The stock count file, as batches of product updates found by SKU. One line can be broken on purpose.</summary>
    private static IEnumerable<List<ArxEntity>> ReadStockFile(string warehouse, int count, int? brokenBatch = null)
    {
        for (var start = 1; start <= PRODUCTS; start += BATCH)
        {
            var batch = Enumerable.Range(start, BATCH).Select(i =>
            {
                var line = new ArxEntity("product");
                line["sku"] = $"SKU-U9{warehouse}{i:00000}";
                line["stock_quantity"] = count;
                return line;
            }).ToList();

            if ((start - 1) / BATCH == brokenBatch)
            {
                batch[3]["sku"] = "SKU-NOT-COUNTED";
            }

            yield return batch;
        }
    }

    /// <summary>1. RunBatchAsync: four batches of 500, one commit when the code returns.</summary>
    private static async Task Scenario1_AutomaticCommitAsync(SampleContext dbContext)
    {
        Step("1. Four batches, one transaction");

        var options = new ArxUpdateOptions { Match = ArxMatch.Key("sku") };

        await dbContext.Service.RunBatchAsync(async scope =>
        {
            foreach (var batch in ReadStockFile("A", count: 12))
            {
                await scope.UpdateMultipleAsync(batch, options);
            }

            Show($"{scope.RowsWritten} products counted in {scope.BatchCount} batches");
        });

        /* ===========================================================
         * [DIRECT QUERY CHECK]
         * -----------------------------------------------------------
         * SELECT count(*)
         * FROM product
         * WHERE sku LIKE 'SKU-U9A%' AND stock_quantity = 12;
         * -- Expect: 2000.
         * ===========================================================
         */
    }

    /// <summary>2. Batch 3 has a SKU the catalogue doesn't know: that batch is undone (its savepoint), the others are kept.</summary>
    private static async Task Scenario2_AFailedBatchIsSkippedAsync(SampleContext dbContext)
    {
        Step("2. A failed batch is skipped, the rest committed");

        var options = new ArxUpdateOptions { Match = ArxMatch.Key("sku") };

        await using (var scope = await dbContext.Service.BeginBatchAsync(new ArxBatchOptions { Savepoints = true }))
        {
            foreach (var batch in ReadStockFile("B", count: 30, brokenBatch: 2))
            {
                try
                {
                    await scope.UpdateMultipleAsync(batch, options);
                }
                catch (ArxException error) when (error.Code == ArxErrorCode.RecordNotFound)
                {
                    Show($"batch {error.BatchNumber} skipped: {error.Message}");
                }
            }

            await scope.CommitAsync();
            Show($"{scope.RowsWritten} products counted; failed batches: {string.Join(", ", scope.FailedBatches)}");
        }

        /* ===========================================================
         * [DIRECT QUERY CHECK]
         * -----------------------------------------------------------
         * SELECT stock_quantity, count(*)
         * FROM product
         * WHERE sku LIKE 'SKU-U9B%'
         * GROUP BY stock_quantity
         * ORDER BY stock_quantity;
         * -- Expect: 0 for 500 products (batch 3, skipped: their stock stays at the default 0), 30 for 1500.
         * ===========================================================
         */
    }
}

What every sample is given: the connection string (and the ISample interface the samples implement).

namespace ArxSamples.MSSQL;

/// <summary>One documentation sample: a scenario a real application has, shown end to end.</summary>
public interface ISample
{
    /// <summary>The name to run it alone: <c>dotnet run -- &lt;name&gt;</c>.</summary>
    string Name { get; }

    /// <summary>One line on what it shows.</summary>
    string Title { get; }

    Task RunAsync(SampleContext dbContext);
}

/// <summary>
/// What a sample is given. Every sample gets the connection string; the application's service and catalog
/// (SampleContext.Application.cs) and the plain SqlClient read-back helper (SampleContext.Direct.cs) live in their own
/// files, so a sample needs only the files it uses.
/// </summary>
public sealed partial class SampleContext
{
    public SampleContext(string connectionString)
    {
        ConnectionString = connectionString;
    }

    public string ConnectionString { get; }
}

How the samples print: Step for each scenario, Show for what it did, ShowError for an error caught on purpose.

using ARX.SDK;

namespace ArxSamples.MSSQL;

/// <summary>How the samples print: a numbered step for each scenario, then what it did.</summary>
public static class SampleOutput
{
    /// <summary>The start of a scenario: <c>Step("1. By the id Create returned")</c>.</summary>
    public static void Step(string text) => Console.WriteLine($"  > {text}");

    /// <summary>What the scenario did.</summary>
    public static void Show(string text) => Console.WriteLine($"      {text}");

    /// <summary>An ARX error the scenario caught on purpose.</summary>
    public static void ShowError(ArxException exception)
        => Console.WriteLine($"      caught {exception.GetType().Name}: {exception.Message}");
}

The application's ARX service and its schema catalog, created once and shared by the samples.

using ARX.SDK.SQL.Services;
using ARX.SDK.Services;

namespace ArxSamples.MSSQL;

/// <summary>The application's ARX service and its schema catalog, shared by the real and advanced samples.</summary>
public sealed partial class SampleContext
{
    private ArxSchemaCatalog? _catalog;
    private ArxMsSqlDbService? _service;

    /// <summary>The application's schema catalog: table ids, keys and selection sets (SampleCatalog.cs).</summary>
    public ArxSchemaCatalog Catalog => _catalog ??= SampleCatalog.Build();

    /// <summary>
    /// One service for the whole application. It holds no connection: every call opens one from the SqlClient pool and
    /// closes it, unless you pass your own connection or transaction.
    /// </summary>
    public ArxMsSqlDbService Service => _service ??= new ArxMsSqlDbService(ConnectionString, Catalog);
}

The application's schema catalog: every table's id column and keys, and its selection sets.

using ARX.SDK.Services;
using System.Text.Json;
using System.Text.Json.Serialization;

namespace ArxSamples.MSSQL;

/// <summary>An account's industry. Stored in account.industry (smallint) as the enum's number.</summary>
public enum Industry : short
{
    Retail = 1,
    Manufacturing = 2,
    Healthcare = 3,
    Finance = 4,
    Technology = 5
}

/// <summary>
/// The application's schema catalog: every table's id column and keys, and every picklist (selection set),
/// defined once.
/// </summary>
/// <remarks>
/// <para>
/// <b>Ids.</b> Every table the application writes is declared with its one id column: its name, its .NET type, and
/// whether the database generates it (<c>ArxIdColumn.Generated</c>: identity, sequence, default) or the caller supplies
/// it (<c>ArxIdColumn.Supplied</c>: a natural id, a client-generated Guid). A table with no id column, such as a log or a
/// table identified by several columns together, is declared <c>ArxIdColumn.NoIdColumn</c>. ARX never reads ids from
/// the database: the developer knows the schema, and an undeclared table is refused (ARX1016).
/// </para>
/// <para>
/// <b>Keys.</b> Optional business keys: one or more columns whose values identify a record, such as an account number,
/// or a region and a SKU together. Create doesn't use them (the database's unique constraints do the enforcing); Update
/// and Delete will be able to find rows by them. <c>ValidateSchemaAsync</c> checks ids and keys against the database
/// when you ask it to.
/// </para>
/// <para>
/// <b>Selection sets.</b> /// A selection set (picklist, option set) is a column that only accepts a fixed list of codes, each with a display
/// text: an account's status (ACTIVE, ON_HOLD, CLOSED), a contact's preferred channel (E, P, S). The catalog tells ARX,
/// for each set:
/// </para>
/// <list type="bullet">
/// <item><b>Name</b>: what an <see cref="ArxSelectionSetValue"/> refers to, for example <c>new ArxSelectionSetValue("AccountStatus", "ON_HOLD")</c>.</item>
/// <item><b>Kind</b>: what the codes are, matching the column: Integer, String, Character, Boolean, Decimal or Guid.
/// (SQL Server has no enum types: a fixed list of text values is a String set, and a CHECK can guard the column.)</item>
/// <item><b>Multiplicity</b>: Single (one code), or several codes stored as bit flags (Flags), a delimited string
/// (Delimited) or a JSON array (Json). SQL Server has no array columns.</item>
/// <item><b>Options</b>: each code with its display text and order. A retired option (<c>IsActive: false</c>) stays
/// readable in old rows, but can't be written any more.</item>
/// </list>
/// <para>
/// Build the catalog once at startup and give it to the service: <c>new ArxMsSqlDbService(connectionString, SampleCatalog.Build())</c>.
/// ARX checks every <see cref="ArxSelectionSetValue"/> against it in memory, before any SQL is sent, and stores the code
/// itself in the column: no lookup table, no extra query. A plain value (a string or number that isn't an
/// ArxSelectionSetValue) isn't checked.
/// </para>
/// <para>
/// The catalog is plain code, so the options can come from wherever you keep them: written out here, taken from a .NET
/// enum (<c>AddEnum</c>), loaded from JSON (<see cref="SetsJson"/> and <see cref="FromJson"/> below), or loaded at
/// startup from your own option table and passed to <c>new ArxSelectionSetDefinition(name, kind, options)</c>. ARX
/// never reads option tables or files itself.
/// </para>
/// </remarks>
public static class SampleCatalog
{
    /// <summary>
    /// Selection sets kept as JSON, as an application might keep them in a file, a settings store or a database column.
    /// Here the JSON is in the class itself; in an application, read it from wherever you keep it
    /// (<c>File.ReadAllText("selection-sets.json")</c>, configuration, an HTTP call) and pass it to <see cref="FromJson"/>.
    /// "multiplicity" and "active" are optional (Single and true).
    /// </summary>
    public const string SetsJson = """
        [
          {
            "name": "LeadSource",
            "kind": "String",
            "options": [
              { "value": "WEB",       "text": "Website",    "order": 0 },
              { "value": "REFERRAL",  "text": "Referral",   "order": 1 },
              { "value": "EVENT",     "text": "Trade show", "order": 2 },
              { "value": "COLD_CALL", "text": "Cold call",  "order": 3, "active": false }
            ]
          },
          {
            "name": "ContactRating",
            "kind": "Integer",
            "multiplicity": "Single",
            "options": [
              { "value": 1, "text": "Poor" },
              { "value": 2, "text": "Fair" },
              { "value": 3, "text": "Good" },
              { "value": 4, "text": "Very good" },
              { "value": 5, "text": "Excellent" }
            ]
          }
        ]
        """;

    public static ArxSchemaCatalog Build()
    {
        var builder = ArxSchemaCatalog.Create()

            // ---------- tables: id column and keys ----------
            // Generated bigint ids (IDENTITY): the database fills them in. The keys are the unique columns that also
            // identify a record.
            .Table("account", ArxIdColumn.Generated<long>("id")).Key("account", "account_number")
            .Table("contact", ArxIdColumn.Generated<long>("id")).Key("contact", "email")

            // Retrieve: a contact's account_id reads back as a reference to its account, named by the account's
            // name; a contact is named by its first and last names.
            .Lookup("contact", "account_id", "account")
            .DisplayName("account", "name")
            .DisplayName("contact", new[] { "first_name", "last_name" }, " ")

            .Table("product_category", ArxIdColumn.Generated<long>("id")).Key("product_category", "name")
            .Table("product", ArxIdColumn.Generated<long>("id")).Key("product", "sku")
            .Table("sales_order", ArxIdColumn.Generated<long>("id")).Key("sales_order", "order_number")
            .Table("order_line", ArxIdColumn.Generated<long>("id"))
            .Table("data_type_showcase", ArxIdColumn.Generated<long>("id"))

            // A natural id: the caller gives the ISO country code.
            .Table("country", ArxIdColumn.Supplied<string>("code"))

            // A generated uuid id (DEFAULT gen_random_uuid()).
            .Table("device", ArxIdColumn.Generated<Guid>("id")).Key("device", "serial_number")

            // No id column: a price is identified by its region and SKU together, a compound key.
            .Table("price_list", ArxIdColumn.NoIdColumn).Key("price_list", "region", "sku")

            // No primary key at all: the same action can be logged twice.
            .Table("activity_log", ArxIdColumn.NoIdColumn)

            // A partitioned table without an id: a visit is a fact, never updated (Retrieve Advanced8).
            .Table("web_visit", ArxIdColumn.NoIdColumn)

            // ---------- selection sets ----------

            // From a .NET enum: the set is named after the enum ("Industry"), the codes are its numbers (Retail = 1, …),
            // the display texts are its member names. Column: account.industry smallint.
            .AddEnum<Industry>()

            // Written out: String codes with display texts, single choice. Column: account.status varchar(10).
            .Add("AccountStatus", ArxSelectionSetKind.String, ("ACTIVE", "Active"), ("ON_HOLD", "On hold"), ("CLOSED", "Closed"))

            // Multiple choice, stored as delimited text ("ENTERPRISE,PUBLIC"). RESELLER is retired: rows that have it still
            // read back, but new writes can't use it. Column: account.segments nvarchar(200).
            .Add(new ArxSelectionSetDefinition("CustomerSegment", ArxSelectionSetKind.String, new[]
            {
                new ArxSelectionSetOption("SMB", "Small business", 0),
                new ArxSelectionSetOption("ENTERPRISE", "Enterprise", 1),
                new ArxSelectionSetOption("PUBLIC", "Public sector", 2),
                new ArxSelectionSetOption("RESELLER", "Reseller", 3, IsActive: false)
            }, ArxSelectionSetMultiplicity.Delimited))

            // Single-character codes. Column: contact.preferred_channel char(1).
            .Add("ContactChannel", ArxSelectionSetKind.Character, ('E', "Email"), ('P', "Phone"), ('S', "Text message"))

            // A fixed list of text values the column's CHECK also guards (SQL Server has no enum types).
            // Column: data_type_showcase.mood nvarchar(10).
            .Add(new ArxSelectionSetDefinition("Mood", ArxSelectionSetKind.String, new[]
            {
                new ArxSelectionSetOption("happy", "Happy", 0),
                new ArxSelectionSetOption("neutral", "Neutral", 1),
                new ArxSelectionSetOption("sad", "Sad", 2)
            }));

        // Loaded from JSON: LeadSource (contact.lead_source text) and ContactRating (contact.rating smallint).
        foreach (var set in FromJson(SetsJson))
        {
            builder.Add(set);
        }

        return builder.Build();
    }

    /// <summary>Reads selection set definitions from JSON (the shape of <see cref="SetsJson"/>).</summary>
    public static IEnumerable<ArxSelectionSetDefinition> FromJson(string json)
    {
        var options = new JsonSerializerOptions
        {
            PropertyNameCaseInsensitive = true,
            Converters = { new JsonStringEnumConverter() }   // "String", "Integer", "Array", … as names
        };

        var sets = JsonSerializer.Deserialize<List<JsonSet>>(json, options)
            ?? throw new InvalidOperationException("The selection set JSON is empty.");

        foreach (var set in sets)
        {
            yield return new ArxSelectionSetDefinition(
                set.Name,
                set.Kind,
                set.Options.Select(x => new ArxSelectionSetOption(Code(x.Value, set.Kind), x.Text, x.Order, x.Active)),
                set.Multiplicity,
                set.DatabaseTypeName);
        }
    }

    /// <summary>A JSON value as a code of the set's kind: JSON has no char, Guid or 64-bit integer of its own.</summary>
    private static object Code(JsonElement value, ArxSelectionSetKind kind) => kind switch
    {
        ArxSelectionSetKind.Integer => value.GetInt64(),
        ArxSelectionSetKind.Decimal => value.GetDecimal(),
        ArxSelectionSetKind.Boolean => value.GetBoolean(),
        ArxSelectionSetKind.Guid => value.GetGuid(),
        ArxSelectionSetKind.Character => value.GetString()![0],
        _ => value.GetString()!   // String and DatabaseEnum
    };

    private sealed record JsonSet(
        string Name,
        ArxSelectionSetKind Kind,
        List<JsonOption> Options,
        ArxSelectionSetMultiplicity Multiplicity = ArxSelectionSetMultiplicity.Single,
        string? DatabaseTypeName = null);

    private sealed record JsonOption(JsonElement Value, string Text, int Order = 0, bool Active = true);
}

  > 1. Four batches, one transaction
      2000 products counted in 4 batches
  > 2. A failed batch is skipped, the rest committed
      batch 3 skipped: ARX2002 RecordNotFound: UpdateMultiple on 'product': 1 of 500 entities matched no row. Nothing was changed.
      1500 products counted; failed batches: 3