Issue #4615. Document how to use itertools for de-duping.

1 files changed, 24 insertions, 0 deletions

diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst
index e0a55bd..2bd18d0 100644
--- a/Doc/library/itertools.rst
+++ b/Doc/library/itertools.rst
@@ -687,3 +687,27 @@ which incur interpreter overhead.
                return
            indices[i:] = [indices[i] + 1] * (r - i)
            yield tuple(pool[i] for i in indices)
+
+    def unique_everseen(iterable, key=None):
+        "List unique elements, preserving order. Remember all elements ever seen."
+        # unique_everseen('AAAABBBCCDAABBB') --> A B C D
+        # unique_everseen('ABBCcAD', str.lower) --> A B C D    
+        seen = set()
+        seen_add = seen.add
+        if key is None:
+            for element in iterable:
+                if element not in seen:
+                    seen_add(element)
+                    yield element
+        else:
+            for element in iterable:
+                k = key(element)
+                if k not in seen:
+                    seen_add(k)
+                    yield element
+
+    def unique_justseen(iterable, key=None):
+        "List unique elements, preserving order. Remember only the element just seen."
+        # unique_justseen('AAAABBBCCDAABBB') --> A B C D A B
+        # unique_justseen('ABBCcAD', str.lower) --> A B C A D
+        return imap(next, imap(itemgetter(1), groupby(iterable, key)))