Stefan Krawczyk
04/02/2024, 5:23 PMdr = driver.Builder().with_modules(data_loading, feature_engineering, model_training, model_evaluation).with_adapter(...).build()
# one sagemaker job on small machine
data_set = dr.execute(["data_set_v1"], inputs={...})
# one sage maker job on large machine with GPU
model = dr.execute(["model_v1"], inputs={...}, override={"data_set_v1": data_set})
# one sagemaker job on small machine
evaluation = dr.execute(["evaluation_v1"], inputs={...}, override={"model_v1": model})
# some ideas on config structure?
config = {
"tasks": [{"name": "data_set_v1", "sagemaker": ["machine.small"], "artifacts": ["data_set_v1"]},
{"name": "model_v1", "sagemaker": ["machine.gpu"], "artifacts": ["model1"]},
{"name": "model_v2", "sagemaker": ["machine.gpu"], "artifacts": ["model2"]},
{"name": "evaluation_v1", "sagemaker": ["machine.small"]}]
}
sagemaker_pipeline_code = SageMakerPipelineBuilder(dr, config).compile()
airflow_pipeline_code = AirflowPipelineBuilder(dr, config).compile()