<@U06FW299H9T> some of the code we sketched for re...
# contribute
s
@Konstantin Tyapochkin some of the code we sketched for reference:
Copy code
dr = driver.Builder().with_modules(data_loading, feature_engineering, model_training, model_evaluation).with_adapter(...).build()

# one sagemaker job on small machine
data_set = dr.execute(["data_set_v1"], inputs={...})

# one sage maker job on large machine with GPU
model = dr.execute(["model_v1"], inputs={...}, override={"data_set_v1": data_set})

# one sagemaker job on small machine
evaluation = dr.execute(["evaluation_v1"], inputs={...}, override={"model_v1": model})

# some ideas on config structure?
config = {
    "tasks": [{"name": "data_set_v1", "sagemaker": ["machine.small"], "artifacts": ["data_set_v1"]},
              {"name": "model_v1", "sagemaker": ["machine.gpu"], "artifacts": ["model1"]},
              {"name": "model_v2", "sagemaker": ["machine.gpu"], "artifacts": ["model2"]},
              {"name": "evaluation_v1", "sagemaker": ["machine.small"]}]
}

sagemaker_pipeline_code = SageMakerPipelineBuilder(dr, config).compile()
airflow_pipeline_code = AirflowPipelineBuilder(dr, config).compile()
👍 1
👀 1