Product catalog sync

async Task<Dictionary<int, string?>> LoadNamesAsync(string list)
{
    var rows = await GetAsync<List<NamedRow>>($"/api/item-lists/{list}");
    return rows.ToDictionary(row => row.Id, row => row.Name);
}

var colors = await LoadNamesAsync("colors");
var sizes = await LoadNamesAsync("sizes");
var lengths = await LoadNamesAsync("lengths");
var dimensions = await LoadNamesAsync("dimensions");

record NamedRow(int Id, string? Name);

const int priceLevel = 1; // from your configuration

string[] fields =
[
    "productId", "productCode", "productName", "barcode", "groupName", "unitName", "vatRate",
    "isColorAndSize", "isInactive", "lastModify", $"price{priceLevel}", $"price{priceLevel}WoVat",
];

// The upper bound of a run: one second before the newest change on the instance.
async Task<DateTime?> GetUpperBoundAsync()
{
    var page = await PostAsync<Page<JsonObject>>("/api/products/filter-paginated", new
    {
        filter = new { basic = new { includeInactive = true } },
        pageNumber = 1,
        pageSize = 1,
        orderBy = new[] { "lastModify desc" },
        limitFields = new[] { "lastModify" },
    });

    return page.Data.FirstOrDefault()?["lastModify"]?.GetValue<DateTime>().AddSeconds(-1);
}

async Task<List<JsonObject>> LoadProductsAsync(DateTime? modifiedFrom, DateTime modifiedTo)
{
    while (true)
    {
        var products = new List<JsonObject>();
        int? expectedCount = null;
        var shifted = false;

        for (var pageNumber = 1; ; pageNumber++)
        {
            var page = await PostAsync<Page<JsonObject>>("/api/products/filter-paginated", new
            {
                filter = new
                {
                    basic = new { includeInactive = true },
                    additional = new { lastModifyFrom = modifiedFrom, lastModifyTo = modifiedTo },
                },
                pageNumber,
                pageSize = 500,
                orderBy = new[] { "lastModify", "productId" },
                limitFields = fields,
            });

            expectedCount ??= page.TotalCount;
            if (page.TotalCount != expectedCount)
            {
                shifted = true; // a product left the window: read the window again
                break;
            }

            products.AddRange(page.Data);
            if (!page.HasNextPage) break;
        }

        if (!shifted) return products;
    }
}

// the first full load: no lower bound
if (await GetUpperBoundAsync() is { } upperBound)
{
    var products = await LoadProductsAsync(modifiedFrom: null, upperBound);

    foreach (var product in products)
    {
        var id = product["productId"]!.GetValue<int>();
        var price = product[$"price{priceLevel}"]!.GetValue<decimal>();
        // upsert into your system by id
    }

    await SaveWatermarkAsync(upperBound);
}

async Task<List<ProductDetail>> LoadDetailsAsync(IEnumerable<int> productIds)
{
    var details = new List<ProductDetail>();

    foreach (var batch in productIds.Chunk(50))
    {
        // arrays repeat the key: productIds=1&productIds=2
        var query = string.Join("&", batch.Select(id => $"productIds={id}"));

        details.AddRange(await GetAsync<List<ProductDetail>>(
            $"/api/products/product-details?{query}&details=ColorSizeVariants&details=Barcodes"));
    }

    return details;
}

async Task StoreDetailsAsync(IEnumerable<int> productIds)
{
    foreach (var detail in await LoadDetailsAsync(productIds))
    {
        foreach (var variant in detail.ColorSizeVariants ?? [])
        {
            var color = variant.ColorId is { } colorId ? colors.GetValueOrDefault(colorId) : null;
            var size = variant.SizeId is { } sizeId ? sizes.GetValueOrDefault(sizeId) : null;
            // store the variant with its names
        }

        foreach (var barcode in (detail.Barcodes ?? []).Where(barcode => barcode.IsInvalid != true))
        {
            // store barcode.Barcode for the product, or for its variant when ColorId/SizeId are set
        }
    }
}

record ProductDetail(int ProductId, List<Variant>? ColorSizeVariants, List<ProductBarcode>? Barcodes);

record Variant(int Id, int? ColorId, int? SizeId, int? LengthId, int? DimensionId);

record ProductBarcode(int Id, string? Barcode, int? ColorId, int? SizeId, int? LengthId, int? DimensionId,
    bool? IsMain, bool? IsInvalid);

async Task SyncChangesAsync()
{
    if (await GetUpperBoundAsync() is not { } upperBound) return; // empty catalog

    DateTime? watermark = await ReadWatermarkAsync(); // null before the first full load
    var changed = await LoadProductsAsync(watermark?.AddMinutes(-1), upperBound);

    foreach (var product in changed)
    {
        // upsert by productId; hide the product when isInactive is true
    }

    // replaces the variants and barcodes of the changed products
    await StoreDetailsAsync(changed.Select(product => product["productId"]!.GetValue<int>()));

    // move the watermark only after everything above succeeded
    await SaveWatermarkAsync(upperBound);
}

async Task<HashSet<int>> LoadAllProductIdsAsync()
{
    while (true)
    {
        var ids = new HashSet<int>();
        int? expectedCount = null;
        var shifted = false;

        for (var pageNumber = 1; ; pageNumber++)
        {
            var page = await PostAsync<Page<JsonObject>>("/api/products/filter-paginated", new
            {
                filter = new { basic = new { includeInactive = true } },
                pageNumber,
                pageSize = 5000,
                orderBy = new[] { "productId" },
                limitFields = new[] { "productId" },
            });

            expectedCount ??= page.TotalCount;
            if (page.TotalCount != expectedCount)
            {
                shifted = true; // products were created or deleted: read again
                break;
            }

            foreach (var row in page.Data) ids.Add(row["productId"]!.GetValue<int>());
            if (!page.HasNextPage) break;
        }

        if (!shifted) return ids;
    }
}

async Task ReconcileAsync(HashSet<int> knownIds)
{
    var existing = await LoadAllProductIdsAsync();

    foreach (var deletedId in knownIds.Except(existing))
    {
        // remove the product once it was missing in two runs in a row
    }

    foreach (var batch in existing.Except(knownIds).Chunk(500))
    {
        var page = await PostAsync<Page<JsonObject>>("/api/products/filter-paginated", new
        {
            filter = new { basic = new { includeInactive = true, productId = batch } },
            pageNumber = 1,
            pageSize = 500,
            orderBy = new[] { "productId" },
            limitFields = fields,
        });

        // upsert page.Data, then load their variants and barcodes
        await StoreDetailsAsync(batch);
    }
}