⚡️ Speed up method IntParseTable.from_ParseTable by 18%

[codeflash-ai]

📄 18% (0.18x) speedup for IntParseTable.from_ParseTable in python/ccxt/static_dependencies/lark/parsers/lalr_analysis.py

⏱️ Runtime : 993 microseconds → 839 microseconds (best of 250 runs)

📝 Explanation and details

The optimization achieves an 18% speedup by reducing dictionary lookup overhead and intermediate object allocations in the critical loop that transforms parse table states.

Key optimizations applied:

1. Local variable caching: Stores Shift and state_to_idx as local variables (shift, state_lookup) to avoid repeated attribute/global lookups in the hot loop processing each state's lookahead actions.

2. Explicit loop transformation: Replaces the nested dictionary comprehension that was creating intermediate objects with explicit loops that build the transformed lookahead dictionary (la_new) incrementally, reducing memory pressure and improving cache locality.

3. Iterator reuse: Captures parse_table.states.items() once as states_items to avoid repeated method calls.

4. Start/end state processing: Converts dictionary comprehensions to explicit loops for start_states and end_states, which reduces temporary object creation overhead.

Why this leads to speedup:

• The nested dictionary comprehension in the original code was the performance bottleneck (38.8% of total time), creating many temporary tuples and performing repeated dictionary lookups
• Local variable binding eliminates global/attribute lookup overhead in tight loops
• Explicit loops provide better memory access patterns and reduce garbage collection pressure

Impact on workloads:
Based on the function reference, this optimization occurs in the LALR parser construction phase (compute_lalr1_states), which is called during grammar compilation. The test results show consistent 4-25% improvements across various parse table sizes, with larger improvements (19-24%) for complex grammars with many states or actions. This benefits any application using Lark parsers, particularly those processing large or complex grammars during initialization.

✅ Correctness verification report:

Test	Status
⚙️ Existing Unit Tests	🔘 None Found
🌀 Generated Regression Tests	✅ 56 Passed
⏪ Replay Tests	🔘 None Found
🔎 Concolic Coverage Tests	🔘 None Found
📊 Tests Coverage	100.0%

🌀 Generated Regression Tests and Runtime

from typing import Any, Dict

# imports
import pytest
from ccxt.static_dependencies.lark.parsers.lalr_analysis import IntParseTable


# Minimal mock definitions to support the function
class Action:
    def __init__(self, name):
        self.name = name
    def __eq__(self, other):
        return isinstance(other, Action) and self.name == other.name
    def __repr__(self):
        return f"Action({self.name!r})"

Shift = Action('Shift')

class State:
    def __init__(self, id):
        self.id = id
    def __eq__(self, other):
        return isinstance(other, State) and self.id == other.id
    def __hash__(self):
        return hash(self.id)
    def __repr__(self):
        return f"State({self.id!r})"

class ParseTable:
    def __init__(self, states: Dict[State, Dict[Any, Any]], start_states: Dict[str, State], end_states: Dict[str, State]):
        self.states = states
        self.start_states = start_states
        self.end_states = end_states

class ParseTableBase(dict):
    def __init__(self, states, start_states, end_states):
        super().__init__(states)
        self.start_states = start_states
        self.end_states = end_states
from ccxt.static_dependencies.lark.parsers.lalr_analysis import IntParseTable

# unit tests

# --- Basic Test Cases ---

def test_single_state_no_actions():
    # One state, no actions, start/end states set
    s1 = State('A')
    pt = ParseTable(
        states={s1: {}},
        start_states={'main': s1},
        end_states={'main': s1}
    )
    codeflash_output = IntParseTable.from_ParseTable(pt); ipt = codeflash_output # 4.16μs -> 3.98μs (4.70% faster)

def test_two_states_shift_action():
    # Two states, one shift action from s1 to s2
    s1 = State('A')
    s2 = State('B')
    pt = ParseTable(
        states={
            s1: {'x': (Shift, s2)},
            s2: {}
        },
        start_states={'main': s1},
        end_states={'main': s2}
    )
    codeflash_output = IntParseTable.from_ParseTable(pt); ipt = codeflash_output # 4.74μs -> 4.55μs (4.15% faster)

def test_multiple_actions_mixed():
    # State with multiple actions, including Shift and non-Shift
    s1 = State('A')
    s2 = State('B')
    s3 = State('C')
    reduce_action = Action('Reduce')
    pt = ParseTable(
        states={
            s1: {'x': (Shift, s2), 'y': (reduce_action, s3)},
            s2: {},
            s3: {}
        },
        start_states={'main': s1},
        end_states={'main': s3}
    )
    codeflash_output = IntParseTable.from_ParseTable(pt); ipt = codeflash_output # 5.18μs -> 4.80μs (7.83% faster)

# --- Edge Test Cases ---

def test_empty_parse_table():
    # No states at all
    pt = ParseTable(states={}, start_states={}, end_states={})
    codeflash_output = IntParseTable.from_ParseTable(pt); ipt = codeflash_output # 2.66μs -> 2.52μs (5.44% faster)

def test_state_with_multiple_shift_targets():
    # State with multiple shift actions to different states
    s1 = State('A')
    s2 = State('B')
    s3 = State('C')
    pt = ParseTable(
        states={
            s1: {'x': (Shift, s2), 'y': (Shift, s3)},
            s2: {},
            s3: {}
        },
        start_states={'main': s1},
        end_states={'main': s3}
    )
    codeflash_output = IntParseTable.from_ParseTable(pt); ipt = codeflash_output # 5.09μs -> 4.57μs (11.4% faster)

def test_state_with_non_shift_action_tuple():
    # State with tuple action not being Shift
    s1 = State('A')
    s2 = State('B')
    custom_action = Action('Custom')
    pt = ParseTable(
        states={
            s1: {'x': (custom_action, s2)},
            s2: {}
        },
        start_states={'main': s1},
        end_states={'main': s2}
    )
    codeflash_output = IntParseTable.from_ParseTable(pt); ipt = codeflash_output # 4.66μs -> 4.45μs (4.63% faster)


def test_multiple_start_and_end_states():
    # ParseTable with multiple start and end states
    s1 = State('A')
    s2 = State('B')
    s3 = State('C')
    pt = ParseTable(
        states={s1: {}, s2: {}, s3: {}},
        start_states={'main': s1, 'aux': s2},
        end_states={'main': s3, 'aux': s2}
    )
    codeflash_output = IntParseTable.from_ParseTable(pt); ipt = codeflash_output # 6.01μs -> 5.62μs (7.01% faster)

def test_state_with_duplicate_states():
    # Two states with same id (should be treated as different objects)
    s1 = State('A')
    s2 = State('A')
    pt = ParseTable(
        states={s1: {}, s2: {}},
        start_states={'main': s1, 'aux': s2},
        end_states={'main': s2}
    )
    codeflash_output = IntParseTable.from_ParseTable(pt); ipt = codeflash_output # 4.61μs -> 4.32μs (6.69% faster)
    # Both states should have unique indices
    indices = set(ipt.start_states.values())

# --- Large Scale Test Cases ---

def test_large_number_of_states():
    # Many states, each with a shift to the next
    N = 500
    states = [State(str(i)) for i in range(N)]
    parse_states = {}
    for i in range(N-1):
        parse_states[states[i]] = {'go': (Shift, states[i+1])}
    parse_states[states[-1]] = {}  # Last state has no actions
    pt = ParseTable(
        states=parse_states,
        start_states={'main': states[0]},
        end_states={'main': states[-1]}
    )
    codeflash_output = IntParseTable.from_ParseTable(pt); ipt = codeflash_output # 225μs -> 187μs (19.9% faster)
    for i in range(N-1):
        pass

def test_large_number_of_actions_per_state():
    # One state with many actions to different states
    N = 500
    s0 = State('root')
    targets = [State(f't{i}') for i in range(N)]
    actions = {f'a{i}': (Shift, targets[i]) for i in range(N)}
    parse_states = {s0: actions}
    for t in targets:
        parse_states[t] = {}
    pt = ParseTable(
        states=parse_states,
        start_states={'main': s0},
        end_states={'main': targets[-1]}
    )
    codeflash_output = IntParseTable.from_ParseTable(pt); ipt = codeflash_output # 216μs -> 174μs (23.9% faster)
    # s0 should be index 0, targets should be 1..N
    for i in range(N):
        pass

def test_large_number_of_start_end_states():
    # Many start and end states
    N = 300
    states = [State(str(i)) for i in range(N)]
    parse_states = {s: {} for s in states}
    start_states = {f'start{i}': states[i] for i in range(N)}
    end_states = {f'end{i}': states[i] for i in range(N)}
    pt = ParseTable(
        states=parse_states,
        start_states=start_states,
        end_states=end_states
    )
    codeflash_output = IntParseTable.from_ParseTable(pt); ipt = codeflash_output # 187μs -> 166μs (12.9% faster)
    # All start/end states should be mapped correctly
    for i in range(N):
        pass
# codeflash_output is used to check that the output of the original code is the same as that of the optimized code.

from typing import Any, Dict

# imports
import pytest
from ccxt.static_dependencies.lark.parsers.lalr_analysis import IntParseTable

# --- Minimal stubs for dependencies ---

class Action(str):
    """A simple stub for Action."""
    def __new__(cls, name):
        obj = str.__new__(cls, name)
        obj.name = name
        return obj

Shift = Action('Shift')

class State:
    """A stub for the State class, uniquely identified by an integer id."""
    def __init__(self, id):
        self.id = id
    def __hash__(self):
        return hash(self.id)
    def __eq__(self, other):
        return isinstance(other, State) and self.id == other.id
    def __repr__(self):
        return f"State({self.id})"

class ParseTable:
    """A stub for the ParseTable class."""
    def __init__(self, states: Dict[State, Dict[Any, Any]], start_states: Dict[str, State], end_states: Dict[str, State]):
        self.states = states
        self.start_states = start_states
        self.end_states = end_states

class ParseTableBase(dict):
    """A stub for the ParseTableBase class."""
    def __init__(self, states, start_states, end_states):
        self.states = states
        self.start_states = start_states
        self.end_states = end_states
from ccxt.static_dependencies.lark.parsers.lalr_analysis import IntParseTable

# --- Unit tests ---

# 1. Basic Test Cases

def test_basic_single_state_no_actions():
    # One state, no lookahead actions, one start and end state
    s0 = State(0)
    pt = ParseTable(states={s0: {}}, start_states={'A': s0}, end_states={'A': s0})
    codeflash_output = IntParseTable.from_ParseTable(pt); ipt = codeflash_output # 4.16μs -> 3.82μs (8.98% faster)

def test_basic_single_shift_action():
    # One state, one lookahead with Shift action to itself
    s0 = State(0)
    pt = ParseTable(
        states={s0: {'a': (Shift, s0)}},
        start_states={'A': s0},
        end_states={'A': s0}
    )
    codeflash_output = IntParseTable.from_ParseTable(pt); ipt = codeflash_output # 4.13μs -> 3.86μs (7.10% faster)

def test_basic_two_states_shift_between():
    # Two states, shift from s0 to s1
    s0 = State(0)
    s1 = State(1)
    pt = ParseTable(
        states={s0: {'a': (Shift, s1)}, s1: {}},
        start_states={'A': s0},
        end_states={'A': s1}
    )
    codeflash_output = IntParseTable.from_ParseTable(pt); ipt = codeflash_output # 4.61μs -> 4.33μs (6.37% faster)
    # The mapping of states to indices may be {s0: 0, s1: 1} or {s1: 0, s0: 1}
    # But keys in start_states/end_states and shift targets must be consistent
    s0_idx = ipt.start_states['A']
    s1_idx = ipt.end_states['A']

def test_basic_non_shift_action():
    # One state, one lookahead with a non-shift action (e.g., 'Reduce')
    s0 = State(0)
    REDUCE = Action('Reduce')
    pt = ParseTable(
        states={s0: {'a': (REDUCE, s0)}},
        start_states={'A': s0},
        end_states={'A': s0}
    )
    codeflash_output = IntParseTable.from_ParseTable(pt); ipt = codeflash_output # 3.69μs -> 3.70μs (0.513% slower)

def test_basic_multiple_lookaheads():
    # One state, multiple lookaheads with different actions
    s0 = State(0)
    s1 = State(1)
    REDUCE = Action('Reduce')
    pt = ParseTable(
        states={s0: {'a': (Shift, s1), 'b': (REDUCE, s0)}, s1: {}},
        start_states={'A': s0},
        end_states={'A': s1}
    )
    codeflash_output = IntParseTable.from_ParseTable(pt); ipt = codeflash_output # 4.66μs -> 4.39μs (6.22% faster)
    s0_idx = ipt.start_states['A']
    s1_idx = ipt.end_states['A']

# 2. Edge Test Cases

def test_edge_empty_parse_table():
    # No states at all
    pt = ParseTable(states={}, start_states={}, end_states={})
    codeflash_output = IntParseTable.from_ParseTable(pt); ipt = codeflash_output # 2.58μs -> 2.45μs (5.23% faster)

def test_edge_unconnected_states():
    # States not reachable from start/end
    s0 = State(0)
    s1 = State(1)
    pt = ParseTable(
        states={s0: {}, s1: {}},
        start_states={'A': s0},
        end_states={'A': s0}
    )
    codeflash_output = IntParseTable.from_ParseTable(pt); ipt = codeflash_output # 4.40μs -> 4.30μs (2.16% faster)

def test_edge_duplicate_state_objects():
    # Two State objects with same id, but different objects
    s0a = State(0)
    s0b = State(0)
    pt = ParseTable(
        states={s0a: {'a': (Shift, s0a)}, s0b: {'b': (Shift, s0b)}},
        start_states={'A': s0a, 'B': s0b},
        end_states={'A': s0a, 'B': s0b}
    )
    codeflash_output = IntParseTable.from_ParseTable(pt); ipt = codeflash_output # 4.50μs -> 4.42μs (1.79% faster)
    idx = list(ipt.states.keys())[0]

def test_edge_multiple_start_and_end_states():
    # Multiple start/end states
    s0 = State(0)
    s1 = State(1)
    pt = ParseTable(
        states={s0: {}, s1: {}},
        start_states={'A': s0, 'B': s1},
        end_states={'A': s1, 'B': s0}
    )
    codeflash_output = IntParseTable.from_ParseTable(pt); ipt = codeflash_output # 4.72μs -> 4.48μs (5.20% faster)
    # Both mappings must be present and correct
    mapping = {s0: None, s1: None}
    for k, v in ipt.start_states.items():
        mapping[pt.start_states[k]] = v
    for k, v in ipt.end_states.items():
        pass

def test_edge_shift_to_self_and_other():
    # State with shift to self and to another
    s0 = State(0)
    s1 = State(1)
    pt = ParseTable(
        states={s0: {'a': (Shift, s0), 'b': (Shift, s1)}, s1: {'c': (Shift, s0)}},
        start_states={'A': s0},
        end_states={'A': s1}
    )
    codeflash_output = IntParseTable.from_ParseTable(pt); ipt = codeflash_output # 4.76μs -> 4.48μs (6.34% faster)
    s0_idx = ipt.start_states['A']
    s1_idx = ipt.end_states['A']

def test_edge_non_shift_action_with_state():
    # Non-shift action with a state as second tuple element
    s0 = State(0)
    REDUCE = Action('Reduce')
    pt = ParseTable(
        states={s0: {'a': (REDUCE, s0)}},
        start_states={'A': s0},
        end_states={'A': s0}
    )
    codeflash_output = IntParseTable.from_ParseTable(pt); ipt = codeflash_output # 3.71μs -> 3.67μs (1.09% faster)

def test_edge_non_shift_action_without_state():
    # Non-shift action with a non-state as second tuple element
    s0 = State(0)
    REDUCE = Action('Reduce')
    pt = ParseTable(
        states={s0: {'a': (REDUCE, 'not_a_state')}},
        start_states={'A': s0},
        end_states={'A': s0}
    )
    codeflash_output = IntParseTable.from_ParseTable(pt); ipt = codeflash_output # 3.69μs -> 3.56μs (3.71% faster)

def test_edge_state_with_no_lookaheads():
    # State with empty lookahead dict
    s0 = State(0)
    pt = ParseTable(
        states={s0: {}},
        start_states={'A': s0},
        end_states={'A': s0}
    )
    codeflash_output = IntParseTable.from_ParseTable(pt); ipt = codeflash_output # 3.68μs -> 3.44μs (6.98% faster)

# 3. Large Scale Test Cases

def test_large_scale_many_states_and_actions():
    # 500 states, each with one shift to the next
    N = 500
    states = [State(i) for i in range(N)]
    state_dict = {}
    for i in range(N-1):
        state_dict[states[i]] = {'a': (Shift, states[i+1])}
    state_dict[states[N-1]] = {}
    pt = ParseTable(
        states=state_dict,
        start_states={'A': states[0]},
        end_states={'A': states[N-1]}
    )
    codeflash_output = IntParseTable.from_ParseTable(pt); ipt = codeflash_output # 200μs -> 161μs (24.5% faster)
    # Check all states are present and shift actions point to correct indices
    indices = {s: idx for idx, s in enumerate(state_dict)}
    for i in range(N-1):
        idx = indices[states[i]]
        next_idx = indices[states[i+1]]

def test_large_scale_many_lookaheads():
    # 10 states, each with 50 lookahead actions (mix of shift and reduce)
    N = 10
    M = 50
    REDUCE = Action('Reduce')
    states = [State(i) for i in range(N)]
    state_dict = {}
    for i in range(N):
        la = {}
        for j in range(M):
            if j % 2 == 0:
                la[f'a{j}'] = (Shift, states[(i+1)%N])
            else:
                la[f'a{j}'] = (REDUCE, states[i])
        state_dict[states[i]] = la
    pt = ParseTable(
        states=state_dict,
        start_states={'A': states[0]},
        end_states={'A': states[N-1]}
    )
    codeflash_output = IntParseTable.from_ParseTable(pt); ipt = codeflash_output # 30.8μs -> 29.7μs (3.47% faster)
    indices = {s: idx for idx, s in enumerate(state_dict)}
    for i in range(N):
        idx = indices[states[i]]
        for j in range(M):
            if j % 2 == 0:
                next_idx = indices[states[(i+1)%N]]
            else:
                pass

def test_large_scale_multiple_start_end_states():
    # 100 states, 10 start/end states
    N = 100
    K = 10
    states = [State(i) for i in range(N)]
    state_dict = {s: {} for s in states}
    start_states = {f'START{i}': states[i] for i in range(K)}
    end_states = {f'END{i}': states[N-1-i] for i in range(K)}
    pt = ParseTable(
        states=state_dict,
        start_states=start_states,
        end_states=end_states
    )
    codeflash_output = IntParseTable.from_ParseTable(pt); ipt = codeflash_output # 37.9μs -> 29.8μs (27.2% faster)
    indices = {s: idx for idx, s in enumerate(state_dict)}
    for i in range(K):
        pass
    for v in ipt.states.values():
        pass

def test_large_scale_duplicate_state_ids():
    # 100 states, but all have the same id (should be treated as the same state)
    class MyState(State):
        def __init__(self, id):
            super().__init__(0)
    states = [MyState(0) for _ in range(100)]
    state_dict = {s: {'a': (Shift, s)} for s in states}
    pt = ParseTable(
        states=state_dict,
        start_states={'A': states[0]},
        end_states={'A': states[-1]}
    )
    codeflash_output = IntParseTable.from_ParseTable(pt); ipt = codeflash_output # 4.10μs -> 4.02μs (2.04% faster)
    idx = list(ipt.states.keys())[0]
# codeflash_output is used to check that the output of the original code is the same as that of the optimized code.

To edit these changes git checkout codeflash/optimize-IntParseTable.from_ParseTable-mhx82wtl and push.