Why does

.with_remote_executor(SynchronousLocalTaskExecutor())

execute more than once ~5/6 times? Given

len(accounts) == 1

.

dr = (
        driver.Builder()
        .enable_dynamic_execution(allow_experimental_mode=True)
        .with_config({
            "cob_date": datetime(2023,3,25),
            "portfolio_config": portfolio_config,
            "execution": "parallel",
            "accounts": ["account_1"] # the `Parallelizable` part
        })
        .with_remote_executor(SynchronousLocalTaskExecutor())
        .with_modules(
            data_loader, # accounts are broken up here to load each 
            transfrom, # performed after collect
        )
        .build()
    )

@Seth Stokes I can’t reproduce.

# debug_parallel.py
from hamilton import driver
import datetime
from hamilton.execution.executors import SynchronousLocalTaskExecutor
from hamilton.htypes import Parallelizable, Collect

def account_loop(accounts: list[str]) -> Parallelizable[str]:
    print("account loop")
    for account in accounts:
        yield account

def account_step(account_loop: str) -> str:
    print("in account step")
    return account_loop

def reduce(account_step: Collect[str]) -> list[str]:
    print("reducing")
    return list(account_step)


if __name__ == '__main__':

    import __main__ as debug_parallel
    dr = (
            driver.Builder()
            .enable_dynamic_execution(allow_experimental_mode=True)
            .with_config({
                "cob_date": datetime.datetime(2023,3,25),
                "portfolio_config": {},
                "execution": "parallel",
                "accounts": ["account_1"] # the `Parallelizable` part
            })
            .with_remote_executor(SynchronousLocalTaskExecutor())
            .with_modules(
                debug_parallel
            )
            .build()
        )
    dr.execute(["reduce"])

I tried to reproduce as well with a toy example but couldn’t either. The only difference was an unsuccessful response - which produced the odd behavior - where the task executor kept making calls.

What do you mean response? It's doing an API request?

That is correct

what step is the response?

In your dag, closet to `account_ste`p

do you have retrys? or something set?

I do not, but there’s a side effect of writing to a file. Not sure if that affects anything or not, but N calls is more than 1 even still

yeah even if

account_step

throws an error, it only is run once.