Generators & Iterators

Generators are a way to create iterators using yield. They produce values lazily, one at a time, saving memory compared to building complete lists.

Generator Basics

A generator is a function that uses yield to produce values:

def count_up_to(n):
    i = 1
    while i <= n:
        yield i
        i += 1

# Creating a generator object
gen = count_up_to(5)
print(next(gen))  # 1
print(next(gen))  # 2
print(next(gen))  # 3

# Or iterate directly
for num in count_up_to(3):
    print(num)

yield from

Delegate to another generator with yield from:

def chain(*iterables):
    for it in iterables:
        yield from it

# Equivalent to:
def chain(*iterables):
    for it in iterables:
        for item in it:
            yield item

# Usage
for item in chain([1, 2], [3, 4], [5]):
    print(item)  # 1, 2, 3, 4, 5

Infinite Generators

Generators can be infinite - memory stays constant:

def natural_numbers():
    n = 1
    while True:
        yield n
        n += 1

def fibonacci():
    a, b = 0, 1
    while True:
        yield a
        a, b = b, a + b

def cycle_through(items):
    while True:
        yield from items

# Use with islice to limit
from itertools import islice
print(list(islice(fibonacci(), 10)))  # First 10 fib numbers

Generator Methods: send() & throw()

Communicate back into generators:

def coroutine():
    while True:
        value = yield
        print(f"Received: {value}")

# Using send() to pass values in
gen = coroutine()
next(gen)  # Start the generator
gen.send("Hello")  # Received: Hello
gen.send("World")  # Received: World

# Using throw() to raise exceptions
def error_prone_gen():
    try:
        yield 1
    except ValueError:
        print("Caught ValueError!")

gen = error_prone_gen()
print(next(gen))  # 1
gen.throw(ValueError)  # Caught ValueError!

itertools Module

Powerful iterator functions:

from itertools import (
    count, cycle, repeat,  # Infinite iterators
    chain, zip_longest,    # Combinators
    islice, takewhile, dropwhile, filterfalse,  # Filtering
    accumulate, groupby,  # Grouping
    combinations, permutations, product  # Permutations
)

# count(start, step) - infinite counter
for i, n in zip(count(1), range(5)):
    print(i)  # 1, 2, 3, 4, 5

# islice(iterable, stop) or (start, stop, step)
print(list(islice(range(100), 10)))           # [0, 1, 2, ..., 9]
print(list(islice(range(100), 5, 15)))      # [5, 6, 7, ..., 14]
print(list(islice(range(100), 0, 20, 2)))  # [0, 2, 4, ..., 18]

# takewhile(predicate, iterable) - stop when false
print(list(takewhile(lambda x: x < 5, range(10))))  # [0, 1, 2, 3, 4]

# dropwhile(predicate, iterable) - skip while true
print(list(dropwhile(lambda x: x < 5, range(10))))  # [5, 6, 7, 8, 9]

# groupby - group consecutive items
for key, group in groupby("AAABBBCCCAAA"):
    print(key, list(group))

Custom Iterators

Create iterators with __iter__ and __next__:

class RangeIterator:
    def __init__(self, start, stop, step=1):
        self.current = start
        self.stop = stop
        self.step = step
    
    def __iter__(self):
        return self
    
    def __next__(self):
        if self.current >= self.stop:
            raise StopIteration
        value = self.current
        self.current += self.step
        return

# Usage
for i in RangeIterator(0, 5, 2):
    print(i)  # 0, 2, 4

Generator Expressions

Comprehensions as generators (memory efficient):

# List comprehension - creates entire list in memory
squares = [x**2 for x in range(1000000)]

# Generator expression - yields one at a time
squares = (x**2 for x in range(1000000))

# Can be passed directly to functions
print(sum(x**2 for x in range(100)))  # 328350
print(max(x for x in range(100) if x % 2 == 0))  # 98