Generators & Iterators

What is a Generator?

A generator is a function that returns an iterator. Instead of returning all values at once, it yields them one at a time.

# Regular function (returns all at once)
def get_squares_list(n):
    result = []
    for i in range(n):
        result.append(i ** 2)
    return result  # Returns entire list

# Generator function (yields one at a time)
def get_squares_gen(n):
    for i in range(n):
        yield i ** 2  # Pauses here, resumes when next() called

# Usage
squares_list = get_squares_list(1000000)  # Creates list in memory
squares_gen = get_squares_gen(1000000)    # Creates generator object

# Iterating
for square in squares_gen:
    print(square)  # One at a time, memory efficient

How Generators Work

def simple_generator():
    print('First yield')
    yield 1
    print('Second yield')
    yield 2
    print('Third yield')
    yield 3
    print('Done')

# Create generator
gen = simple_generator()

# Each call to next() resumes execution
print(next(gen))  # First yield, then 1
print(next(gen))  # Second yield, then 2
print(next(gen))  # Third yield, then 3
# next(gen)  # StopIteration exception

# Or use for loop
for value in simple_generator():
    print(value)

Generator vs List

import sys

# List comprehension
list_comp = [x ** 2 for x in range(1000)]
print(sys.getsizeof(list_comp))  # ~8856 bytes

# Generator expression
gen_exp = (x ** 2 for x in range(1000))
print(sys.getsizeof(gen_exp))    # ~200 bytes

# Generator is much smaller!

Benefits of Generators

1. Memory efficient: Don't store all values in memory
2. Lazy evaluation: Compute values on demand
3. Infinite sequences: Can represent infinite data
4. Pipeline processing: Chain generators for data processing
5. Early termination: Can stop iteration early

Basic Syntax

# List comprehension: []
list_comp = [x ** 2 for x in range(10)]

# Generator expression: ()
gen_exp = (x ** 2 for x in range(10))

# Usage
for square in gen_exp:
    print(square)

# Convert to list if needed
squares = list(gen_exp)

With Conditions

# Filter
evens = (x for x in range(20) if x % 2 == 0)

# Transform and filter
long_words = (word.upper() for word in words if len(word) > 5)

Nested Generator Expressions

# Flatten matrix
matrix = [[1, 2, 3], [4, 5, 6], [7, 8, 9]]
flat = (num for row in matrix for num in row)

print(list(flat))  # [1, 2, 3, 4, 5, 6, 7, 8, 9]

Generator Functions

# Fibonacci generator
def fibonacci():
    a, b = 0, 1
    while True:
        yield a
        a, b = b, a + b

# Take first 10 Fibonacci numbers
fib = fibonacci()
fib_10 = [next(fib) for _ in range(10)]
print(fib_10)  # [0, 1, 1, 2, 3, 5, 8, 13, 21, 34]

# Infinite sequence
def count_from(n):
    while True:
        yield n
        n += 1

# File reading generator
def read_lines(filename):
    with open(filename) as f:
        for line in f:
            yield line.strip()

Pipeline Pattern

def read_data(filename):
    with open(filename) as f:
        for line in f:
            yield line.strip()

def parse_csv(lines):
    for line in lines:
        yield line.split(',')

def filter_valid(rows):
    for row in rows:
        if len(row) >= 3:
            yield row

# Pipeline - processes one item at a time
data = read_data('data.csv')
parsed = parse_csv(data)
valid = filter_valid(parsed)

for row in valid:
    print(row)

Sending Values to Generators

def accumulator():
    total = 0
    while True:
        value = yield total
        if value is None:
            break
        total += value

acc = accumulator()
next(acc)  # Initialize (must call next first)
print(acc.send(10))  # 10
print(acc.send(20))  # 30
print(acc.send(30))  # 60

Iterator Protocol

class Countdown:
    def __init__(self, start):
        self.start = start
    
    def __iter__(self):
        return self
    
    def __next__(self):
        if self.start <= 0:
            raise StopIteration
        self.start -= 1
        return self.start + 1

# Usage
countdown = Countdown(5)
for num in countdown:
    print(num)  # 5, 4, 3, 2, 1

Iterator vs Generator

# Generator (simpler)
def countdown_gen(start):
    while start > 0:
        yield start
        start -= 1

# Iterator class (more control)
class CountdownIter:
    def __init__(self, start):
        self.start = start
        self.current = start
    
    def __iter__(self):
        return self
    
    def __next__(self):
        if self.current <= 0:
            raise StopIteration
        value = self.current
        self.current -= 1
        return value
    
    def reset(self):
        self.current = self.start

# Generator is usually preferred
cd = CountdownGen(5)
list(cd)  # [5, 4, 3, 2, 1]

# But iterator class allows reset
cd = CountdownIter(5)
list(cd)  # [5, 4, 3, 2, 1]
cd.reset()
list(cd)  # [5, 4, 3, 2, 1]

Infinite Iterator

class InfiniteCounter:
    def __init__(self, start=0, step=1):
        self.current = start
        self.step = step
    
    def __iter__(self):
        return self
    
    def __next__(self):
        value = self.current
        self.current += self.step
        return value

# Use with itertools.islice for finite usage
from itertools import islice
counter = InfiniteCounter(1, 2)  # 1, 3, 5, 7, ...
first_10 = list(islice(counter, 10))
print(first_10)  # [1, 3, 5, 7, 9, 11, 13, 15, 17, 19]

Practical Example: File Reader

class FileReader:
    def __init__(self, filename, chunk_size=1024):
        self.filename = filename
        self.chunk_size = chunk_size
    
    def __iter__(self):
        return self
    
    def __next__(self):
        if not hasattr(self, '_file'):
            self._file = open(self.filename, 'rb')
        
        chunk = self._file.read(self.chunk_size)
        if not chunk:
            self._file.close()
            raise StopIteration
        return chunk

# Memory-efficient file reading
for chunk in FileReader('large_file.bin'):
    process(chunk)