C#

Pete O'Hanlon wrote:

he documentation is all over the place as the examples you tend to find relate to Rx as it was about 6 years ago and it's moved on so much

var buttonClick = Observable.FromEventPattern<RoutedEventHandler, RoutedEventArgs>(
    h => btnMainWindow.Click += h,
    h => btnMainWindow.Click -= h);

var EnumerableTest = Enumerable
    .Range(1, 10)
    .ToObservable()
    .ObserveOnDispatcher()
    .SubscribeOn(NewThreadScheduler.Default);

var DisposableEnumerable = EnumerableTest
    .Subscribe(evt =>
txtMainWindow.Text += evt + Environment.NewLine
);

buttonClick.Subscribe(evt =>
{
    DisposableEnumerable.Dispose();
});

.SubscribeOn(NewThreadScheduler.Default)

.SubscribeOnDispatcher()

List<object> entityBatch = new List<object>(batchSize);

public static class EnumerableExtensions
{
    private static IEnumerable<IEnumerable<TSource>> BatchIterator<TSource>(IEnumerable<TSource> source, int size)
    {
        int count = 0;
        var bucket = new TSource[size];
        foreach (var item in source)
        {
            bucket[count] = item;

            checked
            {
                count++;
            }

            if (count == size)
            {
                yield return bucket.Select(x => x);
                bucket = new TSource[size];
                count = 0;
            }
        }
        if (count != 0)
        {
            yield return bucket.Take(count);
        }

        // ReSharper disable once RedundantAssignment
        bucket = null;
    }

    public static IEnumerable<IEnumerable<TSource>> Batch<TSource>(this IEnumerable<TSource> source, int size)
    {
        if (source == null) throw new ArgumentNullException(nameof(source));
        if (size < 1) throw new ArgumentOutOfRangeException(nameof(size));
        return BatchIterator(source, size);
    }
}

Richard Deeming wrote:

Don't measure the performance of your code using DateTime; use the Stopwatch class[^] instead.

Richard Deeming wrote:

Did you make sure to "warm up" the code before you timed it, and run the test over many iterations?

Richard Deeming wrote:

Since you know the maximum number of items you'll be adding to the list, you should set its capacity when you create it:

C#

List<object> entityBatch = new List<object>(batchSize);

If you don't set the capacity, it starts at 4 and doubles each time it runs out of space.

Richard Deeming wrote:

You need to check the value returned from enumerator.MoveNext; if it returns false, you've reached the end of the sequence. Your code currently continues adding the final items from the sequence to the list until you reach the batch size.

Richard Deeming wrote:

You should consider using proper generic types, rather than a List<object>; that way, you avoid having to cast the items back to the correct type when you read the batches.

Richard Deeming wrote:

It's probably best to wrap this sort of thing up as a extension method

RichardGrimmer wrote:

not possible in this instance - it's in an abstract base class, which operates over several types of similar but not related types and exists in a (necessarily) singleton class....

public abstract class MyBaseClass
{
    public void ProcessList<T>(IEnumerable<T> listToProcess) where T : MyBaseClass
    {
        ...
    }
}

public class ChildItem
{
  public ChildItem()
  {
    Count = int.MaxValue;
    LongCount = long.MaxValue;
    Nullable2 = int.MaxValue - 3;
    Nullable3 = Nullable2.Value - 2;
  }
  public int Count { get; set; }
  public long LongCount { get; set; }
  public int? Nullable1 { get; set; }
  public int? Nullable2 { get; set; }
  public int? Nullable3 { get; set; }
}
public class MasterList
{
  private List<ChildItem> children = new List<ChildItem>();
  public MasterList()
  {
    for (int i = 0; i < 6; i++)
    {
      children.Add(new ChildItem());
    }
    Master1 = "Hi";
    Master2 = "Hi";
    Master3 = "Hi";
    Master4 = "Hi";
    Master5 = "Hi";
    Master6 = "Hi";
    Master7 = "Hi";
    Master8 = "Hi";
    Master9 = "Hi";
    Master10 = "Hi";
    Master11 = "Hi";
    Master12 = "Hi";
  }

  public IEnumerable<ChildItem> Children => children;

  public string Master1 { get; set; }
  public string Master2 { get; set; }
  public string Master3 { get; set; }
  public string Master4 { get; set; }
  public string Master5 { get; set; }
  public string Master6 { get; set; }
  public string Master7 { get; set; }
  public string Master8 { get; set; }
  public string Master9 { get; set; }
  public string Master10 { get; set; }
  public string Master11 { get; set; }
  public string Master12 { get; set; }
}
public class TestIt
{
  private List<MasterList> items = new List<MasterList>(700000);

  public TestIt()
  {
    for (int i = 0; i < 700000; i++)
    {
      items.Add(new MasterList());
    }
    Console.WriteLine("Ready");
  }

  public void StartTest(string text = "Dummy run")
  {
    var enumerator = items.GetEnumerator();

    enumerator.MoveNext();

    List<object> entityBatch = new List<object>();

    Stopwatch sw = new Stopwatch();
    sw.Start();

    for (int iterStep = 0; iterStep < 700000; iterStep++)
    {
      entityBatch.Add(enumerator.Current);
      enumerator.MoveNext();
    }
    sw.Stop();

    Console.WriteLine(text);
    Console.WriteLine("Total time {0}", sw.ElapsedMilliseconds);
  }
}

public partial class DateOfBirth
    {
        public int DateOfBirthId { get; set; }
        public Nullable<int> Day { get; set; }
        public Nullable<int> Month { get; set; }
        public Nullable<int> Year { get; set; }
        public long PepDeskRecordId { get; set; }
    }

public class ExpandedFoo
    {
        
        /// <summary>
        /// Gets or sets a value indicating the Foo Id associated with this person
        /// </summary>
        public string FooId { get; set; }

        /// <summary>
        /// Gets or sets a value indicating the Bar Id (Primary Key) associated with this person
        /// </summary>
        public string BarId { get; set; }

        /// <summary>
        /// Gets or sets a value indicating the title of this person
        /// </summary>
        public string Title { get; set; }

        /// <summary>
        /// Gets or sets a value indicating the gender of this person
        /// </summary>
        public string Gender { get; set; }

        /// <summary>
        /// Gets or sets a value indicating the first name of this person
        /// </summary>
        public string FirstName { get; set; }

        /// <summary>
        /// Gets or sets a value indicating the last name of this person
        /// </summary>
        public string LastName { get; set; }

        /// <summary>
        /// Gets or sets a value indicating the full name of this person
        /// </summary>
        public string FullName { get; set; }

        /// <summary>
        /// Gets or sets a value indicating the other names associated with this person
        /// </summary>
        public string OtherNames { get; set; }

        /// <summary>
        /// Gets or sets a value indicating the Country associated with this person
        /// </summary>
        public string Country { get; set; }

        /// <summary>
        /// Gets or sets a value indicating the Blah Id associated with this person
        /// </summary>
        public string BlahId{ get; set; }

        /// <summary>
        /// Gets or sets a value indicating the Combined name associated with this person
        /// </summary>
        public string CombinedName { get; set; }

        /// <summary>
        /// Gets or sets a value indicating the date of births associated with the Foo
        /// </summary>
        public IEnumerable<DateOfBirth> DateOfBirths { get; set; }
        
        /// <summary>
        /// Initialises a new instance of the <see cref="ExpandedFoo"/> class
        /// </summary>
        public ExpandedFoo()
        {
            this.DateOfBirths = new List<DateOfBirth>();
        }

    }

var expandedEntities = Foontities.Select(entity => new ExpandedFoo
            {
                FooId = entity.fooId.ToString(CultureInfo.InvariantCulture),
                BarId = entity.BarId.ToString(CultureInfo.InvariantCulture),

                Title = entity.Title,
                Gender = entity.Gender,
                FirstName = entity.FirstName,
                LastName = entity.LastName,
                FullName = entity.FullName,

                OtherNames = entity.OtherNames,
                CombinedName = entity.CombinedName,

                Country = entity.Country,

                BlahId=  entity.BlahId.ToString(),

                DateOfBirths = (from DateOfBirth dob in dateOfBirths
                                where dob.FooId== entity.FooId
                                select new FooDateOfBirth()
                                {
                                    DateOfBirthId = dob.DateOfBirthId,
                                    Year = dob.Year,
                                    Month = dob.Month,
                                    Day = dob.Day
                                }).ToList()
            });

DateOfBirths = (from DateOfBirth dob in dateOfBirths
                                where dob.PooId == entity.FooId
                                select new FooDateOfBirth()
                                {
                                    DateOfBirthId = dob.DateOfBirthId,
                                    Year = dob.Year,
                                    Month = dob.Month,
                                    Day = dob.Day
                                })

DateOfBirths = (from DateOfBirth dob in dateOfBirths
                                where dob.FooId == entity.FooId
                                select new FooDateOfBirth()
                                {
                                    DateOfBirthId = dob.DateOfBirthId,
                                    Year = dob.Year,
                                    Month = dob.Month,
                                    Day = dob.Day
                                }) as List<DateOfBirth>

Pete O'Hanlon wrote:

I can make an awful lot of assumptions here