Paper: dagster-slurm: Dagster Assets on Slurm-Managed Scientific Workflows

papers/dagster_slurm/myst.yml

@@ -0,0 +1,80 @@
+version: 1
+extends: ../papers.yml
+project:
+  id: scipy-2026-dagster_slurm
+  title: 'dagster-slurm: Dagster Assets on Slurm-Managed Scientific Workflows'
+  description:
+    An open-source integration that runs Dagster asset source code across local,
+    CI, and Slurm-managed execution targets, demonstrated on a web-scale
+    European firm-discovery pipeline.
+  banner: asset_graph_tiers.png
+  thumbnail: asset_graph_tiers.png
+  authors:
+    - name: Hernan Picatto
+      email: h.picatto@gmail.com
+      orcid: 0000-0001-6662-464X
+      corresponding: true
+      affiliations:
+        - Austrian Supply Chain Intelligence Institute (ASCII)
+    - name: Maximilian Hess
+      orcid: 0009-0007-8093-7117
+      affiliations:
+        - Austrian Supply Chain Intelligence Institute (ASCII)
+    - name: Georg Heiler
+      orcid: 0000-0002-8684-1163
+      affiliations:
+        - Complexity Science Hub Vienna (CSH)
+        - Austrian Supply Chain Intelligence Institute (ASCII)
+    - name: Martin Pfister
+      affiliations:
+        - AI Factory Austria
+  keywords:
+    - Dagster
+    - Slurm
+    - HPC
+    - reproducible research
+    - workflow orchestration
+    - Pixi
+  abbreviations:
+    HPC: High-Performance Computing
+    RSE: Research Software Engineering
+    CI: Continuous Integration
+    LoRA: Low-Rank Adaptation
+    CC: Common Crawl
+    CPU: Central Processing Unit
+    GPU: Graphics Processing Unit
+    SSH: Secure Shell
+    QoS: Quality of Service
+    JSON: JavaScript Object Notation
+    URL: Uniform Resource Locator
+    RAM: Random Access Memory
+    S3: Simple Storage Service
+    API: Application Programming Interface
+    MFA: Multi-Factor Authentication
+    VSC: Vienna Scientific Cluster
+  # SciPy proceedings DOIs (10.25080/...) are not registered with Crossref until publication,
+  # so the doi-exists check is ignored for those keys.
+  error_rules:
+    - rule: doi-exists
+      severity: ignore
+      keys:
+        - dagster
+        - docker
+        - pixi
+        - dagsterpipes
+        - cwl_wiki
+        - commoncrawl
+        - daskjobqueue
+        - nuextract
+        - resiliparse
+        - feickert2025pixi
+        - turner2024flyte
+        - staneva2024echodataflow
+        - bednar2023pandata
+        - moritz2018ray
+  exports:
+    - id: pdf
+      format: typst
+      template: https://github.com/curvenote-templates/scipy.git
+      article: main.md
+      output: full_text.pdf

papers/dagster_slurm/references.bib

@@ -0,0 +1,265 @@
+@software{dagster,
+	title        = {{Dagster}: Data Orchestration Platform},
+	author       = {Elementl, Inc.},
+	year         = 2024,
+	url          = {https://dagster.io/},
+	note         = {Version 1.11}
+}
+@incollection{yoo2003slurm,
+	title        = {{SLURM}: Simple {Linux} Utility for Resource Management},
+	author       = {Yoo, Andy B. and Jette, Morris A. and Grondona, Mark},
+	year         = 2003,
+	booktitle    = {Job Scheduling Strategies for Parallel Processing},
+	publisher    = {Springer},
+	series       = {Lecture Notes in Computer Science},
+	volume       = 2862,
+	pages        = {44--60},
+	doi          = {10.1007/10968987_3},
+	editor       = {Feitelson, Dror G. and Rudolph, Larry and Schwiegelshohn, Uwe}
+}
+@software{pixi,
+	title        = {{Pixi}: Reproducible Package Environments},
+	author       = {prefix.dev},
+	year         = 2024,
+	url          = {https://github.com/prefix-dev/pixi},
+	note         = {Accessed 19 October 2025}
+}
+@manual{dagsterpipes,
+	title        = {{Dagster Pipes} Documentation},
+	author       = {Elementl, Inc.},
+	year         = 2024,
+	url          = {https://docs.dagster.io/concepts/dagster-pipes/pipes-overview},
+	note         = {Accessed 19 October 2025}
+}
+@software{docker,
+	title        = {{Docker}},
+	author       = {{Docker Inc.}},
+	year         = 2024,
+	url          = {https://www.docker.com/},
+	note         = {Accessed 19 June 2026}
+}
+@software{hettrick_2022_7015772,
+	title        = {International {RSE} Survey 2022},
+	author       = {Hettrick, Simon and Bast, Radovan and Crouch, Steve and Wyatt, Claire and Philippe, Olivier and others},
+	year         = 2022,
+	month        = aug,
+	publisher    = {Zenodo},
+	doi          = {10.5281/zenodo.7015772},
+	url          = {https://doi.org/10.5281/zenodo.7015772},
+	version      = {v0.9.3}
+}
+@article{ANTUNES2024100655,
+	title        = {Reproducibility, Replicability and Repeatability: A survey of reproducible research with a focus on high performance computing},
+	author       = {Antunes, Benjamin and Hill, David R. C.},
+	year         = 2024,
+	journal      = {Computer Science Review},
+	volume       = 53,
+	pages        = 100655,
+	doi          = {10.1016/j.cosrev.2024.100655},
+	issn         = {1574-0137}
+}
+@article{Courtes9882991,
+	title        = {Reproducibility and Performance: Why Choose?},
+	author       = {Court{\`e}s, Ludovic},
+	year         = 2022,
+	journal      = {Computing in Science \& Engineering},
+	volume       = 24,
+	number       = 3,
+	pages        = {77--80},
+	doi          = {10.1109/MCSE.2022.3165626}
+}
+@misc{hayotsasson2025address,
+	title        = {Addressing Reproducibility Challenges in {HPC} with Continuous Integration},
+	author       = {Hayot-Sasson, Val{\'e}rie and Hudson, Nathaniel and Bauer, Andr{\'e} and Gonthier, Maxime and Foster, Ian and Chard, Kyle},
+	year         = 2025,
+	doi          = {10.48550/arXiv.2508.21289},
+	eprint       = {2508.21289},
+	archiveprefix = {arXiv},
+	primaryclass = {cs.DC}
+}
+@misc{keahey_2025_15306610,
+	title        = {Report on Challenges of Practical Reproducibility for Systems and {HPC} Computer Science},
+	author       = {Keahey, Kate and Richardson, Marc and Tolosana Calasanz, Rafael and Hunold, Sascha and Lofstead, Jay and Malik, Tanu and Perez, Christian},
+	year         = 2025,
+	month        = apr,
+	publisher    = {Zenodo},
+	doi          = {10.5281/zenodo.15306610}
+}
+@misc{eisty2025,
+	title        = {Ten Essential Guidelines for Building High-Quality Research Software},
+	author       = {Eisty, Nasir U. and Bernholdt, David E. and Koufos, Alex and Luet, David J. and Mundt, Miranda},
+	year         = 2025,
+	doi          = {10.48550/arXiv.2507.16166},
+	eprint       = {2507.16166},
+	archiveprefix = {arXiv},
+	primaryclass = {cs.SE}
+}
+@inproceedings{parsl,
+	title        = {{Parsl}: Pervasive Parallel Programming in {Python}},
+	author       = {Babuji, Yadu and Woodard, Anna and Li, Zhuozhao and Katz, Daniel S. and Clifford, Ben and Kumar, Rohan and Lacinski, Lukasz and Chard, Ryan and Wozniak, Justin M. and Foster, Ian and Wilde, Michael and Chard, Kyle},
+	year         = 2019,
+	booktitle    = {Proceedings of the 28th International Symposium on High-Performance Parallel and Distributed Computing},
+	publisher    = {ACM},
+	series       = {HPDC '19},
+	pages        = {25--36},
+	doi          = {10.1145/3307681.3325400}
+}
+@article{executorlib,
+	title        = {{executorlib} -- Up-scaling {Python} workflows for hierarchical heterogeneous high-performance computing},
+	author       = {Janssen, Jan and Taylor, Michael Gilbert and Yang, Ping and Neugebauer, Joerg and Perez, Danny},
+	year         = 2025,
+	journal      = {Journal of Open Source Software},
+	volume       = 10,
+	number       = 108,
+	pages        = 7782,
+	doi          = {10.21105/joss.07782}
+}
+@article{jobflow,
+	title        = {{jobflow}: Computational Workflows Made Simple},
+	author       = {Rosen, Andrew S. and Gallant, Max and George, Janine and Riebesell, Janosh and Saez, Hrushikesh and Ganose, Alex M. and Jain, Anubhav},
+	year         = 2024,
+	journal      = {Journal of Open Source Software},
+	volume       = 9,
+	number       = 93,
+	pages        = 5995,
+	doi          = {10.21105/joss.05995}
+}
+@inproceedings{psij,
+	title        = {{PSI/J}: A Portable Interface for Submitting, Monitoring, and Managing Jobs},
+	author       = {Hategan-Marandiuc, Mihael and Merzky, Andre and Collier, Nicholson and Maheshwari, Ketan and Ozik, Jonathan and Turilli, Matteo and Wilke, Andreas and Wozniak, Justin M. and Chard, Kyle and Foster, Ian and Ferreira da Silva, Rafael and Jha, Shantenu and Laney, Daniel},
+	year         = 2023,
+	booktitle    = {2023 IEEE 19th International Conference on e-Science (e-Science)},
+	publisher    = {IEEE},
+	pages        = {1--10},
+	doi          = {10.1109/e-Science58273.2023.10254912}
+}
+@misc{cwl_wiki,
+	title        = {Existing Workflow Systems},
+	author       = {{Common Workflow Language Community}},
+	year         = 2024,
+	url          = {https://s.apache.org/existing-workflow-systems},
+	note         = {Accessed 17 February 2026}
+}
+@article{koster2012snakemake,
+	title        = {{Snakemake} -- a scalable bioinformatics workflow engine},
+	author       = {K{\"o}ster, Johannes and Rahmann, Sven},
+	year         = 2012,
+	journal      = {Bioinformatics},
+	volume       = 28,
+	number       = 19,
+	pages        = {2520--2522},
+	doi          = {10.1093/bioinformatics/bts480}
+}
+@article{ditommaso2017nextflow,
+	title        = {{Nextflow} enables reproducible computational workflows},
+	author       = {Di Tommaso, Paolo and Chatzou, Maria and Floden, Evan W. and Barja, Pablo Prieto and Palumbo, Emilio and Notredame, Cedric},
+	year         = 2017,
+	journal      = {Nature Biotechnology},
+	volume       = 35,
+	number       = 4,
+	pages        = {316--319},
+	doi          = {10.1038/nbt.3820}
+}
+@inproceedings{rocklin2015dask,
+	title        = {{Dask}: Parallel Computation with Blocked Algorithms and Task Scheduling},
+	author       = {Rocklin, Matthew},
+	year         = 2015,
+	booktitle    = {Proceedings of the 14th Python in Science Conference},
+	pages        = {126--132},
+	doi          = {10.25080/Majora-7b98e3ed-013}
+}
+@software{daskjobqueue,
+	title        = {{Dask-Jobqueue}: Deploy {Dask} on Job Queueing Systems},
+	author       = {Dask Developers},
+	year         = 2024,
+	url          = {https://jobqueue.dask.org/},
+	note         = {Accessed 19 June 2026}
+}
+@misc{airflow,
+	title        = {{Apache Airflow}},
+	author       = {{Apache Software Foundation}},
+	year         = 2024,
+	url          = {https://airflow.apache.org/},
+	note         = {Accessed 19 June 2026}
+}
+@inproceedings{feickert2025pixi,
+	title        = {Reproducible Machine Learning Workflows for Scientists with {Pixi}},
+	author       = {Feickert, Matthew and Arts, Ruben and Kirkham, John},
+	year         = 2025,
+	booktitle    = {Proceedings of the 24th Python in Science Conference},
+	doi          = {10.25080/nwuf8465}
+}
+@inproceedings{turner2024flyte,
+	title        = {Orchestrating Bioinformatics Workflows Across a Heterogeneous Toolset with {Flyte}},
+	author       = {Turner, Pryce},
+	year         = 2024,
+	booktitle    = {Proceedings of the 23rd Python in Science Conference},
+	doi          = {10.25080/DDJJ4932}
+}
+@inproceedings{staneva2024echodataflow,
+	title        = {{Echodataflow}: Recipe-based Fisheries Acoustics Workflow Orchestration},
+	author       = {Staneva, Valentina and Butala, Soham and Setiawan, Landung and Lee, Wu-Jung},
+	year         = 2024,
+	booktitle    = {Proceedings of the 23rd Python in Science Conference},
+	doi          = {10.25080/JXDK4427}
+}
+@inproceedings{bednar2023pandata,
+	title        = {The {Pandata} Scalable Open-Source Analysis Stack},
+	author       = {Bednar, James A. and Durant, Martin},
+	year         = 2023,
+	booktitle    = {Proceedings of the 22nd Python in Science Conference},
+	doi          = {10.25080/gerudo-f2bc6f59-00b}
+}
+@inproceedings{moritz2018ray,
+	title        = {{Ray}: A Distributed Framework for Emerging {AI} Applications},
+	author       = {Moritz, Philipp and Nishihara, Robert and Wang, Stephanie and Tumanov, Alexey and Liaw, Richard and Liang, Eric and Elibol, Melih and Yang, Zongheng and Paul, William and Jordan, Michael I. and Stoica, Ion},
+	year         = 2018,
+	booktitle    = {13th USENIX Symposium on Operating Systems Design and Implementation (OSDI 18)},
+	pages        = {561--577}
+}
+@inproceedings{kwon2023vllm,
+	title        = {Efficient Memory Management for Large Language Model Serving with {PagedAttention}},
+	author       = {Kwon, Woosuk and Li, Zhuohan and Zhuang, Siyuan and Sheng, Ying and Zheng, Lianmin and Yu, Cody Hao and Gonzalez, Joseph E. and Zhang, Hao and Stoica, Ion},
+	year         = 2023,
+	booktitle    = {Proceedings of the 29th Symposium on Operating Systems Principles},
+	doi          = {10.1145/3600006.3613165}
+}
+@software{nuextract,
+	title        = {{NuExtract-2.0}},
+	author       = {{Numind Team}},
+	year         = 2024,
+	url          = {https://huggingface.co/numind/NuExtract-2.0-8B},
+	note         = {Accessed 19 June 2026}
+}
+@software{resiliparse,
+	title        = {{Resiliparse}: Robust Web Archive Parsing and Processing},
+	author       = {Bevendorff, Janek},
+	year         = 2024,
+	url          = {https://github.com/chatnoir-eu/chatnoir-resiliparse},
+	note         = {Accessed 19 June 2026}
+}
+@misc{hu2021lora,
+	title        = {{LoRA}: Low-Rank Adaptation of Large Language Models},
+	author       = {Hu, Edward J. and Shen, Yelong and Wallis, Phillip and Allen-Zhu, Zeyuan and Li, Yuanzhi and Wang, Shean and Wang, Lu and Chen, Weizhu},
+	year         = 2021,
+	doi          = {10.48550/arXiv.2106.09685},
+	eprint       = {2106.09685},
+	archiveprefix = {arXiv},
+	primaryclass = {cs.CL}
+}
+@inproceedings{raasveldt2019duckdb,
+	title        = {{DuckDB}: an Embeddable Analytical Database},
+	author       = {Raasveldt, Mark and M{\"u}hleisen, Hannes},
+	year         = 2019,
+	booktitle    = {Proceedings of the 2019 International Conference on Management of Data (SIGMOD)},
+	publisher    = {ACM},
+	pages        = {1981--1984},
+	doi          = {10.1145/3299869.3320212}
+}
+@misc{commoncrawl,
+	title        = {{Common Crawl}},
+	author       = {{Common Crawl Foundation}},
+	year         = 2024,
+	url          = {https://commoncrawl.org/},
+	note         = {Accessed 19 June 2026}
+}