Visual Basic

' "lbSelFiles" is the listbox containing paths to the files to be processed.
    ' the "sr" streamreader reads each line of each file. If the line isn't already
    'in the collection (addList), it adds it.
    ' The "getLines" array's only purpose is to provide a max value for the progress bar (progBar).

    addList.Clear()

    For l = 0 To lbSelFiles.Items.Count - 1
      If File.Exists(lbSelFiles.Items.Item(l).ToString) Then
        Dim getLines() As String = File.ReadAllLines(lbSelFiles.Items.Item(l).ToString)
        progBar.Maximum = getLines.Length
        progBar.Value = 0
        Dim sr As New StreamReader(lbSelFiles.Items.Item(l).ToString)

        Dim line As String
        Do
          line = sr.ReadLine()
          'this appears to be the bottle neck
          If Not addList.Contains(line) Then
            addList.Add(line)
          End If
          progBar.Increment(1)
        Loop Until line Is Nothing
        sr.Close()
      End If

    Next l

    'this copies addList to an array, then sends the array to a richtextbox
    'the form containing this code then closes
    Dim arr(addList.Count) As String
    addList.CopyTo(arr, 0)
    Dim newRTF As RichTextBox = CType(main.ActiveMdiChild.Controls(0), RichTextBox)
    newRTF.Lines = arr
    Me.Close()