@conference {,
	title = {O(N) distributed direct factorization of structured dense matrices using runtime systems},
	booktitle = {52nd International Conference on Parallel Processing (ICPP 2023)},
	year = {2023},
	month = {2023-08},
	publisher = {ACM},
	organization = {ACM},
	address = {Salt Lake City, Utah},
	isbn = {9798400708435},
	doi = {10.1145/3605573.3605606},
	url = {https://dl.acm.org/doi/proceedings/10.1145/3605573},
	author = {Sameer Deshmukh and Rio Yokota and George Bosilca and Qinxiang Ma}
}
@conference {,
	title = {Improving the Scaling of an Asynchronous Many-Task Runtime with a Lightweight Communication Engine},
	booktitle = {52nd International Conference on Parallel Processing (ICPP 2023)},
	year = {2023},
	month = {2023-09},
	publisher = {ACM},
	organization = {ACM},
	address = {Salt Lake City, Utah},
	abstract = {There is a growing interest in Asynchronous Many-Task (AMT) runtimes as an efficient way to map irregular and dynamic parallel applications onto heterogeneous computing resources. In this work, we show that AMTs nonetheless struggle with communication bottlenecks when scaling computations strongly and that the design of commonly-used communication libraries such as MPI contribute to these bottlenecks. We replace MPI with LCI, a Lightweight Communication Interface that is designed for dynamic, asynchronous frameworks, as the communication layer for the PaRSEC runtime. The result is a significant reduction of end-to-end latency in communication microbenchmarks and a reduction of overall time-tosolution by up to 12\% in HiCMA, a tile-based low-rank Cholesky factorization package.},
	keywords = {asynchronous many-task, dynamic runtime, lightweight communication, low-rank Cholesky, message-passing, MPI, strong scaling},
	doi = {10.1145/3605573.3605642},
	url = {http://snir.cs.illinois.edu/listed/icpp2023-69.pdf},
	author = {Omri Mor and George Bosilca and Marc Snir}
}
@conference {,
	title = {MPI Continuations And How To Invoke Them},
	booktitle = {Sustained Simulation Performance 2021},
	year = {2023},
	month = {2023-02},
	pages = {67 - 83},
	publisher = {Springer International Publishing},
	organization = {Springer International Publishing},
	address = {Cham},
	abstract = {Asynchronous programming models (APM) are gaining more and more traction, allowing applications to expose the available concurrency to a runtime system tasked with coordinating the execution. While MPI has long provided support for multi-threaded communication and non-blocking operations, it falls short of adequately supporting the asynchrony of separate but dependent parts of an application coupled by the start and completion of a communication operation. Correctly and efficiently handling MPI communication in differentAPMmodels is still a challenge. We have previously proposed an extension to the MPI standard providing operation completion notifications using callbacks, so-called MPI Continuations. This interface is flexible enough to accommodate a wide range of different APMs. In this paper, we discuss different variations of the callback signature and how to best pass data from the code starting the communication operation to the code reacting to its completion. We establish three requirements (efficiency, usability, safety) and evaluate different variations against them. Finally, we find that the current choice is not the best design in terms of both efficiency and safety and propose a simpler, possibly more efficient and safe interface. We also show how the transfer of information into the continuation callback can be largely automated using C++ lambda captures.},
	isbn = {978-3-031-18045-3},
	doi = {10.1007/978-3-031-18046-010.1007/978-3-031-18046-0_5},
	url = {https://link.springer.com/10.1007/978-3-031-18046-0},
	author = {Schuchart, Joseph and George Bosilca}
}
@article {,
	title = {Accelerating Geostatistical Modeling and Prediction With Mixed-Precision Computations: A High-Productivity Approach With PaRSEC},
	journal = {IEEE Transactions on Parallel and Distributed Systems},
	volume = {33},
	year = {2022},
	month = {2022-04},
	pages = {964 - 976},
	abstract = {Geostatistical modeling, one of the prime motivating applications for exascale computing, is a technique for predicting desired quantities from geographically distributed data, based on statistical models and optimization of parameters. Spatial data are assumed to possess properties of stationarity or non-stationarity via a kernel fitted to a covariance matrix. A primary workhorse of stationary spatial statistics is Gaussian maximum log-likelihood estimation (MLE), whose central data structure is a dense, symmetric positive definite covariance matrix of the dimension of the number of correlated observations. Two essential operations in MLE are the application of the inverse and evaluation of the determinant of the covariance matrix. These can be rendered through the Cholesky decomposition and triangular solution. In this contribution, we reduce the precision of weakly correlated locations to single- or half- precision based on distance. We thus exploit mathematical structure to migrate MLE to a three-precision approximation that takes advantage of contemporary architectures offering BLAS3-like operations in a single instruction that are extremely fast for reduced precision. We illustrate application-expected accuracy worthy of double-precision from a majority half-precision computation, in a context where uniform single-precision is by itself insufficient. In tackling the complexity and imbalance caused by the mixing of three precisions, we deploy the PaRSEC runtime system. PaRSEC delivers on-demand casting of precisions while orchestrating tasks and data movement in a multi-GPU distributed-memory environment within a tile-based Cholesky factorization. Application-expected accuracy is maintained while achieving up to 1.59X by mixing FP64/FP32 operations on 1536 nodes of HAWK or 4096 nodes of Shaheen II , and up to 2.64X by mixing FP64/FP32/FP16 operations on 128 nodes of Summit , relative to FP64-only operations. This translates into up to 4.5, 4.7, ...},
	keywords = {Computational modeling, Covariance matrices, Data models, Maximum likelihood estimation, Predictive models, runtime, Task analysis},
	issn = {1045-9219},
	doi = {10.1109/TPDS.2021.3084071},
	url = {https://ieeexplore.ieee.org/document/9442267/https://ieeexplore.ieee.org/ielam/71/9575177/9442267-aam.pdfhttp://xplorestaging.ieee.org/ielx7/71/9575177/09442267.pdf?arnumber=9442267},
	author = {Abdulah, Sameh and Qinglei Cao and Pei, Yu and George Bosilca and Jack Dongarra and Genton, Marc G. and Keyes, David E. and Ltaief, Hatem and Sun, Ying}
}
@article {,
	title = {Comparing Distributed Termination Detection Algorithms for Modern HPC Platforms},
	journal = {International Journal of Networking and Computing},
	volume = {12},
	year = {2022},
	month = {2022-01},
	pages = {26 - 46},
	abstract = {This paper revisits distributed termination detection algorithms in the context of High-Performance Computing (HPC) applications. We introduce an efficient variant of the Credit Distribution Algorithm (CDA) and compare it to the original algorithm (HCDA) as well as to its two primary competitors: the Four Counters algorithm (4C) and the Efficient Delay-Optimal Distributed algorithm (EDOD). We analyze the behavior of each algorithm for some simplified task-based kernels and show the superiority of CDA in terms of the number of control messages. We then compare the implementation of these algorithms over a task-based runtime system, PaRSEC and show the advantages and limitations of each approach in a real implementation.},
	issn = {2185-2839},
	doi = {10.15803/ijnc.12.1_26},
	url = {https://www.jstage.jst.go.jp/article/ijnc/12/1/12_26/_article},
	author = {George Bosilca and Bouteiller, Aur{\'e}lien and Herault, Thomas and Le F{\`e}vre, Valentin and Robert, Yves and Dongarra, Jack}
}
@conference {,
	title = {Composition of Algorithmic Building Blocks in Template Task Graphs},
	booktitle = {2022 IEEE/ACM Parallel Applications Workshop: Alternatives To MPI+X (PAW-ATM)},
	year = {2022},
	month = {2023-01},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Dallas, TX, USA},
	doi = {10.1109/PAW-ATM56565.2022.00008},
	url = {https://ieeexplore.ieee.org/document/10024647/},
	author = {Herault, Thomas and Schuchart, Joseph and Valeev, Edward F. and George Bosilca}
}
@article {,
	title = {Evaluating Data Redistribution in PaRSEC},
	journal = {IEEE Transactions on Parallel and Distributed Systems},
	volume = {33},
	number = {8},
	year = {2022},
	month = {2022-08},
	pages = {1856-1872},
	doi = {10.1109/TPDS.2021.3131657},
	author = {Qinglei Cao and George Bosilca and Losada, Nuria and Wu, Wei and Zhong, Dong and Jack Dongarra}
}
@article {,
	title = {Evaluations of molecular modeling and machine learning for predictive capabilities in binding of lanthanum and actinium with carboxylic acids},
	journal = {Journal of Radioanalytical and Nuclear Chemistry},
	year = {2022},
	month = {2022-12},
	issn = {0236-5731},
	doi = {10.1007/s10967-022-08620-7},
	url = {https://rdcu.be/c2lGj},
	author = {Penchoff, Deborah A. and Peterson, Charles C. and Wrancher, Eleigha M. and George Bosilca and Harrison, Robert J. and Valeev, Edward F. and Benny, Paul D.}
}
@conference {,
	title = {A Framework to Exploit Data Sparsity in Tile Low-Rank Cholesky Factorization},
	booktitle = {IEEE International Parallel and Distributed Processing Symposium (IPDPS)},
	year = {2022},
	month = {2022-07},
	doi = {10.1109/IPDPS53621.2022.00047},
	url = {https://ieeexplore.ieee.org/stamp/stamp.jsp?tp=\&arnumber=9820680\&isnumber=9820610},
	author = {Qinglei Cao and Rabab Alomairy and Yu Pei and George Bosilca and Hatem Ltaief and David Keyes and Jack Dongarra}
}
@conference {,
	title = {Generalized Flow-Graph Programming Using Template Task-Graphs: Initial Implementation and Assessment},
	booktitle = {2022 IEEE International Parallel and Distributed Processing Symposium (IPDPS)},
	year = {2022},
	month = {2022-07},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Lyon, France},
	abstract = {We present and evaluate TTG, a novel programming model and its C++ implementation that by marrying the ideas of control and data flowgraph programming supports compact specification and efficient distributed execution of dynamic and irregular applications. Programming interfaces that support task-based execution often only support shared memory parallel environments; a few support distributed memory environments, either by discovering the entire DAG of tasks on all processes, or by introducing explicit communications. The first approach limits scalability, while the second increases the complexity of programming. We demonstrate how TTG can address these issues without sacrificing scalability or programmability by providing higher-level abstractions than conventionally provided by task-centric programming systems, without impeding the ability of these runtimes to manage task creation and execution as well as data and resource management efficiently. TTG supports distributed memory execution over 2 different task runtimes, PaRSEC and MADNESS. Performance of four paradigmatic applications (in graph analytics, dense and block-sparse linear algebra, and numerical integrodifferential calculus) with various degrees of irregularity implemented in TTG is illustrated on large distributed-memory platforms and compared to the state-of-the-art implementations.},
	doi = {10.1109/IPDPS53621.2022.00086},
	url = {https://ieeexplore.ieee.org/abstract/document/9820613},
	author = {Schuchart, Joseph and Nookala, Poornima and Javanmard, Mohammad Mahdi and Herault, Thomas and Valeev, Edward F. and George Bosilca and Harrison, Robert J.}
}
@conference {,
	title = {Implicit Actions and Non-blocking Failure Recovery with MPI},
	booktitle = {2022 IEEE/ACM 12th Workshop on Fault Tolerance for HPC at eXtreme Scale (FTXS)},
	year = {2022},
	month = {2023-01},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Dallas, TX, USA},
	abstract = {Scientific applications have long embraced the MPI as the environment of choice to execute on large distributed systems. The User-Level Failure Mitigation (ULFM) specification extends the MPI standard to address resilience and enable MPI applications to restore their communication capability after a failure. This works builds upon the wide body of experience gained in the field to eliminate a gap between current practice and the ideal, more asynchronous, recovery model in which the fault tolerance activities of multiple components can be carried out simultaneously and overlap. This work proposes to: (1) provide the required consistency in fault reporting to applications (i.e., enable an application to assess the success of a computational phase without incurring an unacceptable performance hit); (2) bring forward the building blocks that permit the effective scoping of fault recovery in an application, so that independent components in an application can recover without interfering with each other, and separate groups of processes in the application can recover independently or in unison; and (3) overlap recovery activities necessary to restore the consistency of the system (e.g., eviction of faulty processes from the communication group) with application recovery activities (e.g., dataset restoration from checkpoints).},
	doi = {10.1109/FTXS56515.2022.00009},
	url = {https://ieeexplore.ieee.org/document/10024038/},
	author = {Bouteiller, Aur{\'e}lien and George Bosilca}
}
@inproceedings {,
	title = {{Integrating process, control-flow, and data resiliency layers using a hybrid Fenix/Kokkos approach}},
	journal = {2022 IEEE International Conference on Cluster Computing (CLUSTER 2022)},
	year = {2022},
	month = {2022-09},
	address = {Heidelberg, Germany},
	keywords = {checkpointing, Fault tolerance, Fenix, HPC, Kokkos, MPI-ULFM, resilience},
	url = {https://hal.archives-ouvertes.fr/hal-03772536},
	author = {Whitlock, Matthew and Morales, Nicolas and George Bosilca and Bouteiller, Aur{\'e}lien and Nicolae, Bogdan and Teranishi, Keita and Giem, Elisabeth and Sarkar, Vivek}
}
@inproceedings {,
	title = {Lossy all-to-all exchange for accelerating parallel 3-D FFTs on hybrid architectures with GPUs},
	journal = {2022 IEEE International Conference on Cluster Computing (CLUSTER)},
	year = {2022},
	month = {2022-09},
	pages = {152-160},
	abstract = {In the context of parallel applications, communication is a critical part of the infrastructure and a potential bottleneck. The traditional approach to tackle communication challenges consists of redesigning algorithms so that the complexity or the communication volume is reduced. However, there are algorithms like the Fast Fourier Transform (FFT) where reducing the volume of communication is very challenging yet can reap large benefit in terms of time-to-completion. In this paper, we revisit the implementation of the MPI all-to-all routine at the core of 3D FFTs by using advanced MPI features, such as One-Sided Communication, and integrate data compression during communication to reduce the volume of data exchanged. Since some compression techniques are {\textquoteleft}lossy{\textquoteright} in the sense that they involve a loss of accuracy, we study the impact of lossy compression in heFFTe, the state-of-the-art FFT library for large scale 3D FFTs on hybrid architectures with GPUs. Consequently, we design an approximate FFT algorithm that trades off user-controlled accuracy for speed. We show that we speedup the 3D FFTs proportionally to the compression rate. In terms of accuracy, comparing our approach with a reduced precision execution, where both the data and the computation are in reduced precision, we show that when the volume of communication is compressed to the size of the reduced precision data, the approximate FFT algorithm is as fast as the one in reduced precision while the accuracy is one order of magnitude better.},
	doi = {10.1109/CLUSTER51413.2022.00029},
	author = {Cayrols, Sebastien and Li, Jiali and George Bosilca and Stanimire Tomov and Ayala, Alan and Dongarra, Jack}
}
@techreport {,
	title = {Mixed precision and approximate 3D FFTs: Speed for accuracy trade-off with GPU-aware MPI and run-time data compression},
	journal = {ICL Technical Report},
	number = {ICL-UT-22-04},
	year = {2022},
	month = {2022-05},
	keywords = {All-to-all, Approximate FFTs, ECP, heFFTe, Lossy compression, mixed-precision algorithms, MPI},
	author = {Sebastien Cayrols and Jiali Li and George Bosilca and Stanimire Tomov and Alan Ayala and Jack Dongarra}
}
@conference {,
	title = {Pushing the Boundaries of Small Tasks: Scalable Low-Overhead Data-Flow Programming in TTG},
	booktitle = {2022 IEEE International Conference on Cluster Computing (CLUSTER)},
	year = {2022},
	month = {2022-09},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Heidelberg, Germany},
	abstract = {Shared memory parallel programming models strive to provide low-overhead execution environments. Task-based programming models, in particular, are well-suited to cope with the ubiquitous multi- and many-core systems since they allow applications to express all available concurrency to a scheduler, which is tasked with exploiting the available hardware resources. It is general consensus that atomic operations should be preferred over locks and mutexes to avoid inter-thread serialization and the resulting loss in efficiency. However, even atomic operations may serialize threads if not used judiciously. In this work, we will discuss several optimizations applied to TTG and the underlying PaRSEC runtime system aiming at removing contentious atomic operations to reduce the overhead of task management to a few hundred clock cycles. The result is an optimized data-flow programming system that seamlessly scales from a single node to distributed execution and which is able to compete with OpenMP in shared memory.},
	keywords = {Dataflow graph, Hardware, Instruction sets, Memory management, PaR-SEC, parallel programming, runtime, scalability, Task analysis, task-based programming, Template Task Graph, TTG},
	doi = {10.1109/CLUSTER51413.2022.00026},
	url = {https://ieeexplore.ieee.org/document/9912704/},
	author = {Schuchart, Joseph and Nookala, Poornima and Herault, Thomas and Valeev, Edward F. and George Bosilca}
}
@inproceedings {,
	title = {Reshaping Geostatistical Modeling and Prediction for Extreme-Scale Environmental Applications},
	journal = {2022 International Conference for High Performance Computing, Networking, Storage and Analysis (SC22)},
	year = {2022},
	month = {2022-11},
	publisher = {IEEE Press},
	address = {Dallas, TX},
	abstract = {We extend the capability of space-time geostatistical modeling using algebraic approximations, illustrating application-expected accuracy worthy of double precision from majority low-precision computations and low-rank matrix approximations. We exploit the mathematical structure of the dense covariance matrix whose inverse action and determinant are repeatedly required in Gaussian log-likelihood optimization. Geostatistics augments first-principles modeling approaches for the prediction of environmental phenomena given the availability of measurements at a large number of locations; however, traditional Cholesky-based approaches grow cubically in complexity, gating practical extension to continental and global datasets now available. We combine the linear algebraic contributions of mixed-precision and low-rank computations within a tilebased Cholesky solver with on-demand casting of precisions and dynamic runtime support from PaRSEC to orchestrate tasks and data movement. Our adaptive approach scales on various systems and leverages the Fujitsu A64FX nodes of Fugaku to achieve up to 12X performance speedup against the highly optimized dense Cholesky implementation.},
	keywords = {climate/weather prediction, dynamic runtime systems, high performance computing., low- rank matrix approximations, mixed-precision computations, space-time geospatial statistics, Task-based programming models},
	isbn = {9784665454445},
	url = {https://dl.acm.org/doi/abs/10.5555/3571885.3571888},
	author = {Cao, Qinglei and Abdulah, Sameh and Rabab Alomairy and Pei, Yu and Pratik Nag and George Bosilca and Dongarra, Jack and Genton, Marc G. and Keyes, David and Ltaief, Hatem and Sun, Ying}
}
@article {,
	title = {Using long vector extensions for MPI reductions},
	journal = {Parallel Computing},
	volume = {109},
	year = {2022},
	month = {2022-03},
	pages = {102871},
	abstract = {The modern CPU{\textquoteright}s design, including the deep memory hierarchies and SIMD/vectorization capability have a more significant impact on algorithms{\textquoteright} efficiency than the modest frequency increase observed recently. The current introduction of wide vector instruction set extensions (AVX and SVE) motivated vectorization to become a critical software component to increase efficiency and close the gap to peak performance.

In this paper, we investigate the impact of the vectorization of MPI reduction operations. We propose an implementation of predefined MPI reduction operations using vector intrinsics (AVX and SVE) to improve the time-to-solution of the predefined MPI reduction operations. The evaluation of the resulting software stack under different scenarios demonstrates that the approach is not only efficient but also generalizable to many vector architectures. Experiments conducted on varied architectures (Intel Xeon Gold, AMD Zen 2, and Arm A64FX), show that the proposed vector extension optimized reduction operations significantly reduce completion time for collective communication reductions. With these optimizations, we achieve higher memory bandwidth and an increased efficiency for local computations, which directly benefit the overall cost of collective reductions and applications based on them.},
	issn = {01678191},
	doi = {10.1016/j.parco.2021.102871},
	url = {https://www.sciencedirect.com/science/article/pii/S0167819121001137},
	author = {Zhong, Dong and Cao, Qinglei and George Bosilca and Dongarra, Jack}
}
@article {,
	title = {Accelerating FFT towards Exascale Computing},
	year = {2021},
	publisher = {NVIDIA GPU Technology Conference (GTC2021)},
	author = {Alan Ayala and Stanimire Tomov and Haidar, Azzam and Stoyanov, M. and Cayrols, Sebastien and Li, Jiali and George Bosilca and Jack Dongarra}
}
@article {,
	title = {Callback-based completion notification using MPI Continuations},
	journal = {Parallel Computing},
	volume = {21238566},
	year = {2021},
	month = {Jan-05-2021},
	pages = {102793},
	abstract = {Asynchronous programming models (APM) are gaining more and more traction, allowing applications to expose the available concurrency to a runtime system tasked with coordinating the execution. While MPI has long provided support for multi-threaded communication and nonblocking operations, it falls short of adequately supporting APMs as correctly and efficiently handling MPI communication in different models is still a challenge. We have previously proposed an extension to the MPI standard providing operation completion notifications using callbacks, so-called MPI Continuations. This interface is flexible enough to accommodate a wide range of different APMs.

In this paper, we present an extension to the previously described interface that allows for finer control of the behavior of the MPI Continuations interface. We then present some of our first experiences in using the interface in the context of different applications, including the NAS parallel benchmarks, the PaRSEC task-based runtime system, and a load-balancing scheme within an adaptive mesh refinement solver called ExaHyPE. We show that the interface, implemented inside Open MPI, enables low-latency, high-throughput completion notifications that outperform solutions implemented in the application space.},
	keywords = {MPI, MPI Continuations, OmpSs, OpenMP, parsec, TAMPI, Task-based programming models},
	issn = {01678191},
	doi = {10.1016/j.parco.2021.102793},
	url = {https://www.sciencedirect.com/science/article/abs/pii/S0167819121000466?via\%3Dihub},
	author = {Schuchart, Joseph and Samfass, Philipp and Niethammer, Christoph and Gracia, Jos{\'e} and George Bosilca}
}
@conference {,
	title = {Distributed-Memory Multi-GPU Block-Sparse Tensor Contraction for Electronic Structure},
	booktitle = {35th IEEE International Parallel \&  Distributed Processing Symposium (IPDPS 2021)},
	year = {2021},
	month = {2021-05},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Portland, OR},
	abstract = {Many domains of scientific simulation (chemistry, condensed matter physics, data science) increasingly eschew dense tensors for block-sparse tensors, sometimes with additional structure (recursive hierarchy, rank sparsity, etc.). Distributed-memory parallel computation with block-sparse tensorial data is paramount to minimize the time-tosolution (e.g., to study dynamical problems or for real-time analysis) and to accommodate problems of realistic size that are too large to fit into the host/device memory of a single node equipped with accelerators. Unfortunately, computation with such irregular data structures is a poor match to the dominant imperative, bulk-synchronous parallel programming model. In this paper, we focus on the critical element of block-sparse tensor algebra, namely binary tensor contraction, and report on an efficient and scalable implementation using the task-focused PaRSEC runtime. High performance of the block-sparse tensor contraction on the Summit supercomputer is demonstrated for synthetic data as well as for real data involved in electronic structure simulations of unprecedented size.},
	keywords = {block-sparse matrix multiplication, distributed-memory, Electronic structure, multi-GPU node, parsec, tensor contraction},
	url = {https://hal.inria.fr/hal-02970659/document},
	author = {Thomas Herault and Yves Robert and George Bosilca and Robert Harrison and Cannada Lewis and Edward Valeev and Jack Dongarra}
}
@article {,
	title = {DTE: PaRSEC Enabled Libraries and Applications},
	year = {2021},
	month = {2021-04},
	publisher = {2021 Exascale Computing Project Annual Meeting},
	author = {George Bosilca and Thomas Herault and Jack Dongarra}
}
@article {,
	title = {An international survey on MPI users},
	journal = {Parallel Computing},
	volume = {108},
	year = {2021},
	month = {2021-12},
	abstract = {The Message Passing Interface (MPI) plays a crucial part in the parallel computing ecosystem, a driving force behind many of thehigh-performance computing (HPC) successes. To maintain its relevance to the user community{\textemdash}and in particular to the growingHPC community at large{\textemdash}the MPI standard needs to identify and understand the MPI users{\textquoteright} concerns and expectations, and adaptaccordingly to continue to efficiently bridge the gap between users and hardware.  This questionnaire survey was conducted usingtwo online questionnaire frameworks and has gathered more than 850 answers from 42 countries since February 2019.  Some ofpreceding surveys of MPI uses are questionnaire surveys like ours, while others are conducted either by analyzing MPI programsto reveal static behavior or by using profiling tools to analyze the dynamic runtime behavior of MPI jobs.  Our survey is differentfrom other questionnaire surveys in terms of its larger number of participants and wide geographic spread. As a result, it is possibleto illustrate the current status of MPI users more accurately and with a wider geographical distribution. In this report, we will showsome interesting findings, compare the results with preceding studies when possible, and provide some recommendations for MPIForum based on the findings.},
	keywords = {message passing interface, MPI, survey},
	doi = {10.1016/j.parco.2021.102853},
	url = {https://www.sciencedirect.com/science/article/abs/pii/S0167819121000983},
	author = {Atsushi Hori and Emmanuel Jeannot and George Bosilca and Takahiro Ogura and Balazs Gerofi and Jie Yin and Yutaka Ishikawa}
}
@inbook {,
	title = {An Introduction to High Performance Computing and Its Intersection with Advances in Modeling Rare Earth Elements and Actinides},
	booktitle = {Rare Earth Elements and Actinides: Progress in Computational Science Applications},
	volume = {1388},
	year = {2021},
	month = {2021-10},
	pages = {3-53},
	publisher = {American Chemical Society},
	organization = {American Chemical Society},
	chapter = {1},
	address = {Washington, DC},
	abstract = {Computationally driven solutions in nuclear and radiochemistry heavily depend on efficient modeling of Rare Earth Elements (REEs) and actinides. Accurate modeling of REEs and actinides faces challenges stemming from limitations from an imbalanced hardware-software ecosystem and its implications on inefficient use of High Performance Computing (HPC). This chapter provides a historical perspective on the evolution of HPC hardware, its intersectionality with domain sciences, the importance of benchmarks for performance, and an overview of challenges and advances in modeling REEs and actinides. This chapter intends to provide an introduction for researchers at the intersection of scientific computing, software development for HPC, and applied computational modeling of REEs and actinides. The chapter is structured in five sections. First, the Introduction includes subsections focusing on the Importance of REEs and Actinides (1.1), Hardware, Software, and the HPC Ecosystem (1.2), and Electronic Structure Modeling of REEs and Actinides (1.3). Second, a section in High Performance Computing focuses on the TOP500 (2.1), HPC Performance (2.2), HPC Benchmarks: Processing, Bandwidth, and Latency (2.3), and HPC Benchmarks and their Relationship to Chemical Modeling (2.4). Third, the Software Challenges and Advances focus on NWChem/NWChemEx (3.1), MADNESS (3.2), and MPQC (3.3). The fourth section provides a short overview of Artificial Intelligence in HPC applications relevant to nuclear and radiochemistry. The fifth section illustrates A Protocol to Evaluate Complexation Preferences in Separations of REEs and Actinides through Computational Modeling.},
	keywords = {actinide, Computational modeling, HPC, REE},
	isbn = {ISBN13: 9780841298255 eISBN: 9780841298248},
	doi = {10.1021/bk-2021-1388.ch001},
	url = {https://pubs.acs.org/doi/10.1021/bk-2021-1388.ch001},
	author = {Deborah A. Penchoff and Edward Valeev and Heike Jagode and Piotr Luszczek and Anthony Danalis and George Bosilca and Robert J. Harrison and Jack Dongarra and Theresa L. Windus}
}
@conference {,
	title = {Leveraging PaRSEC Runtime Support to Tackle Challenging 3D Data-Sparse Matrix Problems},
	booktitle = {35th IEEE International Parallel \& Distributed Processing Symposium (IPDPS 2021)},
	year = {2021},
	month = {2021-05},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Portland, OR},
	abstract = {The task-based programming model associated with dynamic runtime systems has gained popularity for challenging problems because of workload imbalance, heterogeneous resources, or extreme concurrency. During the last decade, lowrank matrix approximations, where the main idea consists of exploiting data sparsity typically by compressing off-diagonal tiles up to an application-specific accuracy threshold, have been adopted to address the curse of dimensionality at extreme scale. In this paper, we create a bridge between the runtime and the linear algebra by communicating knowledge of the data sparsity to the runtime. We design and implement this synergistic approach with high user productivity in mind, in the context of the PaRSEC runtime system and the HiCMA numerical library. This requires to extend PaRSEC with new features to integrate rank information into the dataflow so that proper decisions can be taken at runtime. We focus on the tile low-rank (TLR) Cholesky factorization for solving 3D data-sparse covariance matrix problems arising in environmental applications. In particular, we employ the 3D exponential model of Matern matrix kernel, which exhibits challenging nonuniform {\textasciiacute}high ranks in off-diagonal tiles. We first provide a dynamic data structure management driven by a performance model to reduce extra floating-point operations. Next, we optimize the memory footprint of the application by relying on a dynamic memory allocator, and supported by a rank-aware data distribution to cope with the workload imbalance. Finally, we expose further parallelism using kernel recursive formulations to shorten the critical path. Our resulting high-performance implementation outperforms existing data-sparse TLR Cholesky factorization by up to 7-fold on a large-scale distributed-memory system, while minimizing the memory footprint up to a 44-fold factor. This multidisciplinary work highlights the need to empower runtime systems beyond their original duty of task scheduling for servicing next-generation low-rank matrix algebra libraries.},
	keywords = {asynchronous executions and load balancing, dynamic runtime system, environmental applications, High-performance computing, low-rank matrix computations, task-based programming model, user productivity},
	author = {Qinglei Cao and Yu Pei and Kadir Akbudak and George Bosilca and Hatem Ltaief and David Keyes and Jack Dongarra}
}
@conference {,
	title = {Quo Vadis MPI RMA? Towards a More Efficient Use of MPI One-Sided Communication},
	booktitle = {EuroMPI{\textquoteright}21},
	year = {2021},
	address = {Garching, Munich Germany},
	abstract = { The MPI standard has long included one-sided communication abstractions through the MPI Remote Memory Access (RMA) interface. Unfortunately, the MPI RMA chapter in the 4.0 version of the MPI standard still contains both well-known and lesser known short-comings for both implementations and users, which lead to potentially non-optimal usage patterns. In this paper, we identify a set of issues and propose ways for applications to better express anticipated usage of RMA routines, allowing the MPI implementation to better adapt to the application{\textquoteright}s needs. In order to increase the flexibility of the RMA interface, we add the capability to duplicate windows, allowing access to the same resources encapsulated by a window using different configurations. In the same vein, we introduce the concept of MPI memory handles, meant to provide life-time guarantees on memory attached to dynamic windows, removing the overhead currently present in using dynamically exposed memory. We will show that our extensions provide improved accumulate latencies, reduced overheads for multi-threaded flushes, and allow for zero overhead dynamic memory window usage. },
	keywords = {Memory Handles, MPI, MPI-RMA, RDMA},
	url = {https://arxiv.org/abs/2111.08142},
	author = {Schuchart, Joseph and Niethammer, Christoph and Gracia, Jos{\'e} and George Bosilca}
}
@inproceedings {,
	title = {Revisiting Credit Distribution Algorithms for Distributed Termination Detection},
	journal = {2021 IEEE International Parallel and Distributed Processing Symposium Workshops (IPDPSW)},
	year = {2021},
	pages = {611{\textendash}620},
	publisher = {IEEE},
	abstract = {This paper revisits distributed termination detection
algorithms in the context of High-Performance Computing (HPC)
applications. We introduce an efficient variant of the Credit
Distribution Algorithm (CDA) and compare it to the original
algorithm (HCDA) as well as to its two primary competitors: the
Four Counters algorithm (4C) and the Efficient Delay-Optimal
Distributed algorithm (EDOD). We analyze the behavior of each
algorithm for some simplified task-based kernels and show the
superiority of CDA in terms of the number of control messages.},
	keywords = {control messages, credit distribution algorithms, task-based HPC application, Termination detection},
	doi = {10.1109/IPDPSW52791.2021.00095},
	author = {George Bosilca and Aurelien Bouteiller and Thomas Herault and Le F{\`e}vre, Valentin and Robert, Yves and Jack Dongarra}
}
@conference {1478,
	title = {Communication Avoiding 2D Stencil Implementations over PaRSEC Task-Based Runtime},
	booktitle = {2020 IEEE International Parallel and Distributed Processing Symposium Workshops (IPDPSW)},
	year = {2020},
	month = {2020-05},
	publisher = {IEEE},
	organization = {IEEE},
	address = {New Orleans, LA},
	abstract = {Stencil computation or general sparse matrix-vector product (SpMV) are key components in many algorithms like geometric multigrid or Krylov solvers. But their low arithmetic intensity means that memory bandwidth and network latency will be the performance limiting factors. The current architectural trend favors computations over bandwidth, worsening the already unfavorable imbalance. Previous work approached stencil kernel optimization either by improving memory bandwidth usage or by providing a Communication Avoiding (CA) scheme to minimize network latency in repeated sparse vector multiplication by replicating remote work in order to delay communications on the critical path. Focusing on minimizing communication bottleneck in distributed stencil computation, in this study we combine a CA scheme with the computation and communication overlapping that is inherent in a dataflow task-based runtime system such as PaRSEC to demonstrate their combined benefits. We implemented the 2D five point stencil (Jacobi iteration) in PETSc, and over PaRSEC in two flavors, full communications (base-PaRSEC) and CA-PaRSEC which operate directly on a 2D compute grid. Our results running on two clusters, NaCL and Stampede2 indicate that we can achieve 2{\texttimes} speedup over the standard SpMV solution implemented in PETSc, and in certain cases when kernel execution is not dominating the execution time, the CA-PaRSEC version achieved up to 57\% and 33\% speedup over base-PaRSEC implementation on NaCL and Stampede2 respectively.},
	doi = {https://doi.org/10.1109/IPDPSW50202.2020.00127},
	author = {Yu Pei and Qinglei Cao and George Bosilca and Piotr Luszczek and Victor Eijkhout and Jack Dongarra}
}
@conference {,
	title = {DeepFreeze: Towards Scalable Asynchronous Checkpointing of Deep Learning Models},
	booktitle = {20th IEEE/ACM International Symposium on Cluster, Cloud and Internet Computing (CCGRID)},
	year = {2020},
	month = {2020-05},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Melbourne, VIC, Australia},
	abstract = {In the age of big data, deep learning has emerged as a powerful tool to extract insight and exploit its value, both in industry and scientific applications. One common pattern emerging in such applications is frequent checkpointing of the state of the learning model during training, needed in a variety of scenarios: analysis of intermediate states to explain features and correlations with training data, exploration strategies involving alternative models that share a common ancestor, knowledge transfer, resilience, etc. However, with increasing size of the learning models and popularity of distributed data-parallel training approaches, simple checkpointing techniques used so far face several limitations: low serialization performance, blocking I/O, stragglers due to the fact that only a single process is involved in checkpointing. This paper proposes a checkpointing technique specifically designed to address the aforementioned limitations, introducing efficient asynchronous techniques to hide the overhead of serialization and I/O, and distribute the load over all participating processes. Experiments with two deep learning applications (CANDLE and ResNet) on a pre-Exascale HPC platform (Theta) shows significant improvement over state-of-art, both in terms of checkpointing duration and runtime overhead.},
	doi = {https://doi.org/10.1109/CCGrid49817.2020.00-76},
	author = {Bogdan Nicolae and Jiali Li and Justin M. Wozniak and George Bosilca and Matthieu Dorier and Franck Cappello}
}
@article {,
	title = {DTE: PaRSEC Enabled Libraries and Applications (Poster)},
	year = {2020},
	month = {2020-02},
	publisher = {2020 Exascale Computing Project Annual Meeting},
	address = {Houston, TX},
	author = {George Bosilca and Thomas Herault and Jack Dongarra}
}
@article {,
	title = {DTE: PaRSEC Systems and Interfaces (Poster)},
	year = {2020},
	month = {2020-02},
	publisher = {2020 Exascale Computing Project Annual Meeting},
	address = {Houston, TX},
	author = {George Bosilca and Thomas Herault and Jack Dongarra}
}
@conference {,
	title = {Extreme-Scale Task-Based Cholesky Factorization Toward Climate and Weather Prediction Applications},
	booktitle = {Platform for Advanced Scientific Computing Conference (PASC20)},
	year = {2020},
	month = {2020-06},
	publisher = {ACM},
	organization = {ACM},
	address = {Geneva, Switzerland},
	abstract = {Climate and weather can be predicted statistically via geospatial Maximum Likelihood Estimates (MLE), as an alternative to running large ensembles of forward models. The MLE-based iterative optimization procedure requires the solving of large-scale linear systems that performs a Cholesky factorization on a symmetric positive-definite covariance matrix---a demanding dense factorization in terms of memory footprint and computation. We propose a novel solution to this problem: at the mathematical level, we reduce the computational requirement by exploiting the data sparsity structure of the matrix off-diagonal tiles by means of low-rank approximations; and, at the programming-paradigm level, we integrate PaRSEC, a dynamic, task-based runtime to reach unparalleled levels of efficiency for solving extreme-scale linear algebra matrix operations. The resulting solution leverages fine-grained computations to facilitate asynchronous execution while providing a flexible data distribution to mitigate load imbalance. Performance results are reported using 3D synthetic datasets up to 42M geospatial locations on 130, 000 cores, which represent a cornerstone toward fast and accurate predictions of environmental applications.},
	doi = {https://doi.org/10.1145/3394277.3401846},
	author = {Qinglei Cao and Yu Pei and Kadir Akbudak and Aleksandr Mikhalev and George Bosilca and Hatem Ltaief and David Keyes and Jack Dongarra}
}
@article {1459,
	title = {Fault Tolerance of MPI Applications in Exascale Systems: The ULFM Solution},
	journal = {Future Generation Computer Systems},
	volume = {106},
	year = {2020},
	month = {2020-05},
	pages = { 467-481},
	abstract = {The growth in the number of computational resources used by high-performance computing (HPC) systems leads to an increase in failure rates. Fault-tolerant techniques will become essential for long-running applications executing in future exascale systems, not only to ensure the completion of their execution in these systems but also to improve their energy consumption. Although the Message Passing Interface (MPI) is the most popular programming model for distributed-memory HPC systems, as of now, it does not provide any fault-tolerant construct for users to handle failures. Thus, the recovery procedure is postponed until the application is aborted and re-spawned. The proposal of the User Level Failure Mitigation (ULFM) interface in the MPI forum provides new opportunities in this field, enabling the implementation of resilient MPI applications, system runtimes, and programming language constructs able to detect and react to failures without aborting their execution. This paper presents a global overview of the resilience interfaces provided by the ULFM specification, covers archetypal usage patterns and building blocks, and surveys the wide variety of application-driven solutions that have exploited them in recent years. The large and varied number of approaches in the literature proves that ULFM provides the necessary flexibility to implement efficient fault-tolerant MPI applications. All the proposed solutions are based on application-driven recovery mechanisms, which allows reducing the overhead and obtaining the required level of efficiency needed in the future exascale platforms.},
	keywords = {Application-level checkpointing, MPI, resilience, ULFM},
	issn = {0167-739X},
	doi = {https://doi.org/10.1016/j.future.2020.01.026},
	url = {https://www.sciencedirect.com/science/article/pii/S0167739X1930860X},
	author = {Nuria Losada and Patricia Gonz{\'a}lez and Mar{\'\i}a J. Mart{\'\i}n and George Bosilca and Aurelien Bouteiller and Keita Teranishi}
}
@conference {,
	title = {FFT-Based Gradient Sparsification for the Distributed Training of Deep Neural Networks},
	booktitle = {9th International Symposium on High-Performance Parallel and Distributed Computing (HPDC 20)},
	year = {2020},
	month = {2020-06},
	publisher = {ACM},
	organization = {ACM},
	address = {Stockholm, Sweden},
	abstract = {The performance and efficiency of distributed training of Deep Neural Networks (DNN) highly depend on the performance of gradient averaging among participating processes, a step bound by communication costs. There are two major approaches to reduce communication overhead: overlap communications with computations (lossless), or reduce communications (lossy). The lossless solution works well for linear neural architectures, e.g. VGG, AlexNet, but more recent networks such as ResNet and Inception limit the opportunity for such overlapping. Therefore, approaches that reduce the amount of data (lossy) become more suitable. In this paper, we present a novel, explainable lossy method that sparsifies gradients in the frequency domain, in addition to a new range-based float point representation to quantize and further compress gradients. These dynamic techniques strike a balance between compression ratio, accuracy, and computational overhead, and are optimized to maximize performance in heterogeneous environments.

Unlike existing works that strive for a higher compression ratio, we stress the robustness of our methods, and provide guidance to recover accuracy from failures. To achieve this, we prove how the FFT sparsification affects the convergence and accuracy, and show that our method is guaranteed to converge using a diminishing θ in training. Reducing θ can also be used to recover accuracy from the failure. Compared to STOA lossy methods, e.g., QSGD, TernGrad, and Top-k sparsification, our approach incurs less approximation error, thereby better in both the wall-time and accuracy. On an 8 GPUs, InfiniBand interconnected cluster, our techniques effectively accelerate AlexNet training up to 2.26x to the baseline of no compression, and 1.31x to QSGD, 1.25x to Terngrad and 1.47x to Top-K sparsification.},
	keywords = {FFT, Gradient Compression, Loosy Gradients, Machine Learning, Neural Networks},
	doi = {https://doi.org/10.1145/3369583.3392681},
	author = {Linnan Wang and Wei Wu and Junyu Zhang and Hang Liu and George Bosilca and Maurice Herlihy and Rodrigo Fonseca}
}
@conference {,
	title = {Flexible Data Redistribution in a Task-Based Runtime System},
	booktitle = {IEEE International Conference on Cluster Computing (Cluster 2020)},
	year = {2020},
	month = {2020-09},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Kobe, Japan},
	abstract = {Data redistribution aims to reshuffle data to optimize some objective for an algorithm. The objective can be multi-dimensional, such as improving computational load balance or decreasing communication volume or cost, with the ultimate goal to increase the efficiency and therefore decrease the time-to-solution for the algorithm. The classical redistribution problem focuses on optimally scheduling communications when reshuffling data between two regular, usually block-cyclic, data distributions. Recently, task-based runtime systems have gained popularity as a potential candidate to address the programming complexity on the way to exascale. In addition to an increase in portability against complex hardware and software systems, task-based runtime systems have the potential to be able to more easily cope with less-regular data distribution, providing a more balanced computational load during the lifetime of the execution. In this scenario, it becomes paramount to develop a general redistribution algorithm for task-based runtime systems, which could support all types of regular and irregular data distributions. In this paper, we detail a flexible redistribution algorithm, capable of dealing with redistribution problems without constraints of data distribution and data size and implement it in a task-based runtime system, PaRSEC. Performance results show great capability compared to ScaLAPACK, and applications highlight an increased efficiency with little overhead in terms of data distribution and data size.},
	doi = {https://doi.org/10.1109/CLUSTER49012.2020.00032},
	author = {Qinglei Cao and George Bosilca and Wei Wu and Dong Zhong and Aurelien Bouteiller and Jack Dongarra}
}
@conference {,
	title = {HAN: A Hierarchical AutotuNed Collective Communication Framework},
	booktitle = {IEEE Cluster Conference},
	year = {2020},
	month = {2020-09},
	publisher = {Best Paper Award, IEEE Computer Society Press},
	organization = {Best Paper Award, IEEE Computer Society Press},
	address = {Kobe, Japan},
	abstract = {High-performance computing (HPC) systems keep growing in scale and heterogeneity to satisfy the increasing computational need, and this brings new challenges to the design of MPI libraries, especially with regard to collective operations. To address these challenges, we present "HAN," a new hierarchical autotuned collective communication framework in Open MPI, which selects suitable homogeneous collective communication modules as submodules for each hardware level, uses collective operations from the submodules as tasks, and organizes these tasks to perform efficient hierarchical collective operations. With a task-based design, HAN can easily swap out submodules, while keeping tasks intact, to adapt to new hardware. This makes HAN suitable for the current platform and provides a strong and flexible support for future HPC systems. To provide a fast and accurate autotuning mechanism, we present a novel cost model based on benchmarking the tasks instead of a whole collective operation. This method drastically reduces tuning time, as the cost of tasks can be reused across different message sizes, and is more accurate than existing cost models. Our cost analysis suggests the autotuning component can find the optimal configuration in most cases. The evaluation of the HAN framework suggests our design significantly improves the default Open MPI and achieves decent speedups against state-of-the-art MPI implementations on tested applications.},
	author = {Xi Luo and Wei Wu and George Bosilca and Yu Pei and Qinglei Cao and Thananon Patinyasakdikul and Dong Zhong and Jack Dongarra}
}
@article {1460,
	title = {Overhead of Using Spare Nodes},
	journal = {The International Journal of High Performance Computing Applications},
	year = {2020},
	month = {2020-02},
	abstract = {With the increasing fault rate on high-end supercomputers, the topic of fault tolerance has been gathering attention. To cope with this situation, various fault-tolerance techniques are under investigation; these include user-level, algorithm-based fault-tolerance techniques and parallel execution environments that enable jobs to continue following node failure. Even with these techniques, some programs with static load balancing, such as stencil computation, may underperform after a failure recovery. Even when spare nodes are present, they are not always substituted for failed nodes in an effective way. This article considers the questions of how spare nodes should be allocated, how to substitute them for faulty nodes, and how much the communication performance is affected by such a substitution. The third question stems from the modification of the rank mapping by node substitutions, which can incur additional message collisions. In a stencil computation, rank mapping is done in a straightforward way on a Cartesian network without incurring any message collisions. However, once a substitution has occurred, the optimal node-rank mapping may be destroyed. Therefore, these questions must be answered in a way that minimizes the degradation of communication performance. In this article, several spare node allocation and failed node substitution methods will be proposed, analyzed, and compared in terms of communication performance following the substitution. The proposed substitution methods are named sliding methods. The sliding methods are analyzed by using our developed simulation program and evaluated by using the K computer, Blue Gene/Q (BG/Q), and TSUBAME 2.5. It will be shown that when failures occur, the stencil communication performance on the K and BG/Q can be slowed around 10 times depending on the number of node failures. The barrier performance on the K can be cut in half. On BG/Q, barrier performance can be slowed by a factor of 10. Further, it will also be shown that almost no such communication performance degradation can be seen on TSUBAME 2.5. This is because TSUBAME 2.5 has an Infiniband network connected with a FatTree topology, while the K computer and BG/Q have dedicated Cartesian networks. Thus, the communication performance degradation depends on network characteristics.},
	keywords = {communication performance, fault mitigation, Fault tolerance, sliding method, spare node},
	issn = {1094-3420},
	doi = {https://doi.org/10.1177\%2F1094342020901885},
	url = {https://journals.sagepub.com/doi/10.1177/1094342020901885},
	author = {Atsushi Hori and Kazumi Yoshinaga and Thomas Herault and Aurelien Bouteiller and George Bosilca and Yutaka Ishikawa}
}
@conference {,
	title = {Predicting MPI Collective Communication Performance Using Machine Learning},
	booktitle = {2020 IEEE International Conference on Cluster Computing (CLUSTER)},
	year = {2020},
	month = {2020-09},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Kobe, Japan},
	abstract = {The Message Passing Interface (MPI) defines the semantics of data communication operations, while the implementing libraries provide several parameterized algorithms for each operation. Each algorithm of an MPI collective operation may work best on a particular system and may be dependent on the specific communication problem. Internally, MPI libraries employ heuristics to select the best algorithm for a given communication problem when being called by an MPI application. The majority of MPI libraries allow users to override the default algorithm selection, enabling the tuning of this selection process. The problem then becomes how to select the best possible algorithm for a specific case automatically. In this paper, we address the algorithm selection problem for MPI collective communication operations. To solve this problem, we propose an auto-tuning framework for collective MPI operations based on machine-learning techniques. First, we execute a set of benchmarks of an MPI library and its entire set of collective algorithms. Second, for each algorithm, we fit a performance model by applying regression learners. Last, we use the regression models to predict the best possible (fastest) algorithm for an unseen communication problem. We evaluate our approach for different MPI libraries and several parallel machines. The experimental results show that our approach outperforms the standard algorithm selection heuristics, which are hard-coded into the MPI libraries, by a significant margin.},
	keywords = {Auto-tuning, GAM, KNN, Machine Learning, message passing interface, Performance Prediction, XGBoost},
	doi = {https://doi.org/10.1109/CLUSTER49012.2020.00036},
	author = {Sascha Hunold and Abhinav Bhatele and George Bosilca and Peter Knees}
}
@article {,
	title = {A Report of the MPI International Survey (Poster)},
	year = {2020},
	month = {2020-09},
	publisher = {EuroMPI/USA {\textquoteright}20: 27th European MPI Users{\textquoteright} Group Meeting},
	address = {Austin, TX},
	author = {Atsushi Hori and Takahiro Ogura and Balazs Gerofi and Jie Yin and Yutaka Ishikawa and Emmanuel Jeannot and George Bosilca}
}
@conference {,
	title = {Task Bench: A Parameterized Benchmark for Evaluating Parallel Runtime Performance},
	booktitle = {International Conference for High Performance Computing Networking, Storage, and Analysis (SC20)},
	year = {2020},
	month = {2020-11},
	publisher = {ACM},
	organization = {ACM},
	abstract = {We present Task Bench, a parameterized benchmark designed to explore the performance of distributed programming systems under a variety of application scenarios. Task Bench dramatically lowers the barrier to benchmarking and comparing multiple programming systems by making the implementation for a given system orthogonal to the benchmarks themselves: every benchmark constructed with Task Bench runs on every Task Bench implementation. Furthermore, Task Bench{\textquoteright}s parameterization enables a wide variety of benchmark scenarios that distill the key characteristics of larger applications.

To assess the effectiveness and overheads of the tested systems, we introduce a novel metric, minimum effective task granularity (METG). We conduct a comprehensive study with 15 programming systems on up to 256 Haswell nodes of the Cori supercomputer. Running at scale, 100μs-long tasks are the finest granularity that any system runs efficiently with current technologies. We also study each system{\textquoteright}s scalability, ability to hide communication and mitigate load imbalance.},
	url = {https://dl.acm.org/doi/10.5555/3433701.3433783},
	author = {Elliott Slaughter and Wei Wu and Yuankun Fu and Legend Brandenburg and Nicolai Garcia and Wilhem Kautz and Emily Marx and Kaleb S. Morris and Qinglei Cao and George Bosilca and Seema Mirchandaney and Wonchan Lee and Sean Treichler and Patrick McCormick and Alex Aiken}
}
@conference {,
	title = {The Template Task Graph (TTG) - An Emerging Practical Dataflow Programming Paradigm for Scientific Simulation at Extreme Scale},
	booktitle = { 2020 IEEE/ACM 5th International Workshop on Extreme Scale Programming Models and Middleware (ESPM2)},
	year = {2020},
	month = {2020-11},
	publisher = {IEEE},
	organization = {IEEE},
	abstract = {We describe TESSE, an emerging general-purpose, open-source software ecosystem that attacks the twin challenges of programmer productivity and portable performance for advanced scientific applications on modern high-performance computers. TESSE builds upon and extends the ParsecDAG/-dataflow runtime with a new Domain Specific Languages (DSL) and new integration capabilities. Motivating this work is our belief that such a dataflow model, perhaps with applications composed in domain specific languages, can overcome many of the challenges faced by a wide variety of irregular applications that are poorly served by current programming and execution models. Two such applications from many-body physics and applied mathematics are briefly explored. This paper focuses upon the Template Task Graph (TTG), which is TESSE{\textquoteright}s main C++ Api that provides a powerful work/data-flow programming model. Algorithms on spatial trees, block-sparse tensors, and wave fronts are used to illustrate the API and associated concepts, as well as to compare with related approaches.},
	keywords = {dag, dataflow, exascale, graph, High-performance computing, workflow},
	doi = {https://doi.org/10.1109/ESPM251964.2020.00011},
	author = {George Bosilca and Robert Harrison and Thomas Herault and Mohammad Mahdi Javanmard and Poornima Nookala and Edward Valeev}
}
@conference {,
	title = {Using Advanced Vector Extensions AVX-512 for MPI Reduction},
	booktitle = {EuroMPI/USA {\textquoteright}20: 27th European MPI Users{\textquoteright} Group Meeting},
	year = {2020},
	month = {2020-09},
	address = {Austin, TX},
	abstract = {As the scale of high-performance computing (HPC) systems continues to grow, researchers are devoted themselves to explore increasing levels of parallelism to achieve optimal performance. The modern CPU{\textquoteright}s design, including its features of hierarchical memory and SIMD/vectorization capability, governs algorithms{\textquoteright} efficiency. The recent introduction of wide vector instruction set extensions (AVX and SVE) motivated vectorization to become of critical importance to increase efficiency and close the gap to peak performance. In this paper, we propose an implementation of predefined MPI reduction operations utilizing AVX, AVX2 and AVX-512 intrinsics to provide vector-based reduction operation and to improve the timeto- solution of these predefined MPI reduction operations. With these optimizations, we achieve higher efficiency for local computations, which directly benefit the overall cost of collective reductions. The evaluation of the resulting software stack under different scenarios demonstrates that the solution is at the same time generic and efficient. Experiments are conducted on an Intel Xeon Gold cluster, which shows our AVX-512 optimized reduction operations achieve 10X performance benefits than Open MPI default for MPI local reduction.},
	keywords = {Instruction level parallelism, Intel AVX2/AVX-512, Long vector extension, MPI reduction operation, Single instruction multiple data, Vector operation},
	doi = {https://doi.org/10.1145/3416315.3416316},
	author = {Dong Zhong and Qinglei Cao and George Bosilca and Jack Dongarra}
}
@article {,
	title = {Using Advanced Vector Extensions AVX-512  for MPI Reduction (Poster)},
	year = {2020},
	month = {2020-09},
	publisher = {EuroMPI/USA {\textquoteright}20: 27th European MPI Users{\textquoteright} Group Meeting},
	address = {Austin, TX},
	author = {Dong Zhong and George Bosilca and Qinglei Cao and Jack Dongarra}
}
@conference {1484,
	title = {Using Arm Scalable Vector Extension to Optimize Open MPI},
	booktitle = {20th IEEE/ACM International Symposium on Cluster, Cloud and Internet Computing (CCGRID 2020)},
	year = {2020},
	month = {2020-05},
	publisher = {IEEE/ACM},
	organization = {IEEE/ACM},
	address = {Melbourne, Australia},
	abstract = {As the scale of high-performance computing (HPC) systems continues to grow, increasing levels of parallelism must be implored to achieve optimal performance. Recently, the processors support wide vector extensions, vectorization becomes much more important to exploit the potential peak performance of target architecture. Novel processor architectures, such as the Armv8-A architecture, introduce Scalable Vector Extension (SVE) - an optional separate architectural extension with a new set of A64 instruction encodings, which enables even greater parallelisms. In this paper, we analyze the usage and performance of the SVE instructions in Arm SVE vector Instruction Set Architecture (ISA); and utilize those instructions to improve the memcpy and various local reduction operations. Furthermore, we propose new strategies to improve the performance of MPI operations including datatype packing/unpacking and MPI reduction. With these optimizations, we not only provide a higher-parallelism for a single node, but also achieve a more efficient communication scheme of message exchanging. The resulting efforts have been implemented in the context of OPEN MPI, providing efficient and scalable capabilities of SVE usage and extending the possible implementations of SVE to a more extensive range of programming and execution paradigms. The evaluation of the resulting software stack under different scenarios with both simulator and Fujitsu{\textquoteright}s A64FX processor demonstrates that the solution is at the same time generic and efficient.},
	keywords = {ARMIE, datatype pack and unpack, local reduction, non-contiguous accesses, SVE, Vector Length Agnostic},
	doi = {https://doi.org/10.1109/CCGrid49817.2020.00-71},
	author = {Dong Zhong and Pavel Shamis and Qinglei Cao and George Bosilca and Jack Dongarra}
}
@conference {1453,
	title = {Asynchronous Receiver-Driven Replay for Local Rollback of MPI Applications},
	booktitle = {Fault Tolerance for HPC at eXtreme Scale (FTXS) Workshop at The International Conference for High Performance Computing, Networking, Storage, and Analysis (SC{\textquoteright}19)},
	year = {2019},
	month = {2019-11},
	abstract = {With the increase in scale and architectural complexity of supercomputers, the management of failures has become integral to successfully executing a long-running high performance computing application. In many instances, failures have a localized scope, usually impacting a subset of the resources being used, yet widely used failure recovery strategies (like checkpoint/restart) fail to take advantage and rely on global, synchronous recovery actions. Even with local rollback recovery,
in which only the fault impacted processes are restarted from a checkpoint, the consistency of further progress in the execution is achieved through the replay of communication from a message log. This theoretically sound approach encounters some practical limitations: the presence of collective operations
forces a synchronous recovery that prevents survivor processes from continuing their execution, removing any possibility for overlapping further computation with the recovery; and the amount of resources required at recovering peers can be untenable. In this work, we solved both problems by implementing an asynchronous, receiver-driven replay of point-to-point and collective communications, and by exploiting remote-memory access capabilities to access the message logs. This new protocol is evaluated in an implementation of local rollback over the User Level Failure Mitigation fault tolerant Message Passing Interface (MPI). It reduces the recovery times of the failed processes by an average of 59\%, while the time spent in the recovery by the survivor processes is reduced by 95\% when compared to an
equivalent global rollback protocol, thus living to the promise of a truly localized impact of recovery actions.},
	keywords = {checkpoint/restart, Fault tolerance, Message logging, MPI, ULFM, User Level Fault Mitigation},
	url = {https://sc19.supercomputing.org/proceedings/workshops/workshop_files/ws_ftxs103s2-file1.pdf},
	author = {Nuria Losada and Aurelien Bouteiller and George Bosilca}
}
@article {1305,
	title = {Checkpointing Strategies for Shared High-Performance Computing Platforms},
	journal = {International Journal of Networking and Computing},
	volume = {9},
	number = {1},
	year = {2019},
	pages = {28{\textendash}52},
	abstract = {Input/output (I/O) from various sources often contend for scarcely available bandwidth. For example, checkpoint/restart (CR) protocols can help to ensure application progress in failure-prone environments. However, CR I/O alongside an application{\textquoteright}s normal, requisite I/O can increase I/O contention and might negatively impact performance. In this work, we consider different aspects (system-level scheduling policies and hardware) that optimize the overall performance of concurrently executing CR-based applications that share I/O resources. We provide a theoretical model and derive a set of necessary constraints to minimize the global waste on a given platform. Our results demonstrate that Young/Daly{\textquoteright}s optimal checkpoint interval, despite providing a sensible metric for a single, undisturbed application, is not sufficient to optimally address resource contention at scale. We show that by combining optimal checkpointing periods with contention-aware system-level I/O scheduling strategies, we can significantly improve overall application performance and maximize the platform throughput. Finally, we evaluate how specialized hardware, namely burst buffers, may help to mitigate the I/O contention problem. Overall, these results provide critical analysis and direct guidance on how to design efficient, CR ready, large -scale platforms without a large investment in the I/O subsystem.},
	issn = {2185-2847},
	url = {http://www.ijnc.org/index.php/ijnc/article/view/195},
	author = {Thomas Herault and Yves Robert and Aurelien Bouteiller and Dorian Arnold and Kurt Ferreira and George Bosilca and Jack Dongarra}
}
@article {1301,
	title = {Comparing the Performance of Rigid, Moldable, and Grid-Shaped Applications on Failure-Prone HPC Platforms},
	journal = {Parallel Computing},
	volume = {85},
	year = {2019},
	month = {2019-07},
	pages = {1{\textendash}12},
	doi = {https://doi.org/10.1016/j.parco.2019.02.002},
	author = {Valentin Le F{\`e}vre and Thomas Herault and Yves Robert and Aurelien Bouteiller and Atsushi Hori and George Bosilca and Jack Dongarra}
}
@conference {1451,
	title = {Evaluation of Programming Models to Address Load Imbalance on Distributed Multi-Core CPUs: A Case Study with Block Low-Rank Factorization},
	booktitle = {PAW-ATM Workshop at SC19},
	year = {2019},
	month = {2019-11},
	publisher = {ACM},
	organization = {ACM},
	address = {Denver, CO},
	author = {Yu Pei and George Bosilca and Ichitaro Yamazaki and Akihiro Ida and Jack Dongarra}
}
@conference {1410,
	title = {Generic Matrix Multiplication for Multi-GPU Accelerated Distributed-Memory Platforms over PaRSEC},
	booktitle = {ScalA{\textquoteright}19: 10th Workshop on Latest Advances in Scalable Algorithms for Large-Scale Systems},
	year = {2019},
	month = {2019-11},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Denver, CO},
	author = {Thomas Herault and Yves Robert and George Bosilca and Jack Dongarra}
}
@conference {1395,
	title = {Give MPI Threading a Fair Chance: A Study of Multithreaded MPI Designs},
	booktitle = {IEEE Cluster},
	year = {2019},
	month = {2019-09},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Albuquerque, NM},
	abstract = {The Message Passing Interface (MPI) has been one of the most prominent programming paradigms in highperformance computing (HPC) for the past decade. Lately, with changes in modern hardware leading to a drastic increase in the number of processor cores, developers of parallel applications are moving toward more integrated parallel programming paradigms, where MPI is used along with other, possibly node-level, programming paradigms, or MPI+X. MPI+threads emerged as one of the favorite choices in HPC community, according to a survey of the HPC community. However, threading support in MPI comes with many compromises to the overall performance delivered, and, therefore, its adoption is compromised. This paper studies in depth the MPI multi-threaded implementation design in one of the leading MPI implementations, Open MPI, and expose some of the shortcomings of the current design. We propose, implement, and evaluate a new design of the internal handling of communication progress which allows for a significant boost in multi-threading performance, increasing the viability of MPI in the MPI+X programming paradigm.},
	keywords = {communication contention, MPI, thread},
	author = {Thananon Patinyasakdikul and David Eberius and George Bosilca and Nathan Hjelm}
}
@conference {1403,
	title = {Impacts of Multi-GPU MPI Collective Communications on Large FFT Computation},
	booktitle = {Workshop on Exascale MPI (ExaMPI) at SC19},
	year = {2019},
	month = {2019-11},
	address = {Denver, CO},
	keywords = {Collective MPI, Exascale applications, FFT, Heterogeneous systems, scalable},
	author = {Alan Ayala and Stanimire Tomov and Xi Luo and Hejer Shaiek and Azzam Haidar and George Bosilca and Jack Dongarra}
}
@article {1230,
	title = {Local Rollback for Resilient MPI Applications with Application-Level Checkpointing and Message Logging},
	journal = {Future Generation Computer Systems},
	volume = {91},
	year = {2019},
	month = {2019-02},
	pages = {450-464},
	abstract = {The resilience approach generally used in high-performance computing (HPC) relies on coordinated checkpoint/restart, a global rollback of all the processes that are running the application. However, in many instances, the failure has a more localized scope and its impact is usually restricted to a subset of the resources being used. Thus, a global rollback would result in unnecessary overhead and energy consumption, since all processes, including those unaffected by the failure, discard their state and roll back to the last checkpoint to repeat computations that were already done. The User Level Failure Mitigation (ULFM) interface {\textendash} the last proposal for the inclusion of resilience features in the Message Passing Interface (MPI) standard {\textendash} enables the deployment of more flexible recovery strategies, including localized recovery. This work proposes a local rollback approach that can be generally applied to Single Program, Multiple Data (SPMD) applications by combining ULFM, the ComPiler for Portable Checkpointing (CPPC) tool, and the Open MPI VProtocol system-level message logging component. Only failed processes are recovered from the last checkpoint, while consistency before further progress in the execution is achieved through a two-level message logging process. To further optimize this approach point-to-point communications are logged by the Open MPI VProtocol component, while collective communications are optimally logged at the application level{\textemdash}thereby decoupling the logging protocol from the particular collective implementation. This spatially coordinated protocol applied by CPPC reduces the log size, the log memory requirements and overall the resilience impact on the applications.},
	keywords = {Application-level checkpointing, Local rollback, Message logging, MPI, resilience},
	doi = {https://doi.org/10.1016/j.future.2018.09.041},
	author = {Nuria Losada and George Bosilca and Aurelien Bouteiller and Patricia Gonz{\'a}lez and Mar{\'\i}a J. Mart{\'\i}n}
}
@conference {1452,
	title = {Performance Analysis of Tile Low-Rank Cholesky Factorization Using PaRSEC Instrumentation Tools},
	booktitle = {Workshop on Programming and Performance Visualization Tools (ProTools 19) at SC19},
	year = {2019},
	month = {2019-11},
	publisher = {ACM},
	organization = {ACM},
	address = {Denver, CO},
	author = {Qinglei Cao and Yu Pei and Thomas Herault and Kadir Akbudak and Aleksandr Mikhalev and George Bosilca and Hatem Ltaief and David Keyes and Jack Dongarra}
}
@conference {1402,
	title = {Runtime Level Failure Detection and Propagation in HPC Systems},
	booktitle = {European MPI Users{\textquoteright} Group Meeting (EuroMPI {\textquoteright}19)},
	year = {2019},
	month = {2019-09},
	publisher = {ACM},
	organization = {ACM},
	address = {Z{\"u}rich, Switzerland},
	abstract = {As the scale of high-performance computing (HPC) systems continues to grow, mean-time-to-failure (MTTF) of these HPC systems is negatively impacted and tends to decrease. In order to efficiently run long computing jobs on these systems, handling system failures becomes a prime challenge. We present here the design and implementation of an efficient runtime-level failure detection and propagation strategy targeting large-scale, dynamic systems that is able to detect both node and process failures. Multiple overlapping topologies are used to optimize the detection and propagation, minimizing the incurred overheads and guaranteeing the scalability of the entire framework. The resulting framework has been implemented in the context of a system-level runtime for parallel environments, PMIx Reference RunTime Environment (PRRTE), providing efficient and scalable capabilities of fault management to a large range of programming and execution paradigms. The experimental evaluation of the resulting software stack on different machines demonstrate that the solution is at the same time generic and efficient.},
	isbn = {978-1-4503-7175-9},
	doi = {https://doi.org/10.1145/3343211.3343225},
	author = {Dong Zhong and Aurelien Bouteiller and Xi Luo and George Bosilca}
}
@inbook {1302,
	title = {System Software for Many-Core and Multi-Core Architectures},
	booktitle = {Advanced Software Technologies for Post-Peta Scale Computing: The Japanese Post-Peta CREST Research Project},
	year = {2019},
	pages = {59{\textendash}75},
	publisher = {Springer Singapore},
	organization = {Springer Singapore},
	address = {Singapore},
	abstract = {In this project, the software technologies for the post-peta scale computing were explored. More specifically, OS technologies for heterogeneous architectures, lightweight thread, scalable I/O, and fault mitigation were investigated. As for the OS technologies, a new parallel execution model, Partitioned Virtual Address Space (PVAS), for the many-core CPU was proposed. For the heterogeneous architectures, where multi-core CPU and many-core CPU are connected with an I/O bus, an extension of PVAS, Multiple-PVAS, to have a unified virtual address space of multi-core and many-core CPUs was proposed. The proposed PVAS was also enhanced to have multiple processes where process context switch can take place at the user level (named User-Level Process: ULP). As for the scalable I/O, EARTH, optimization techniques for MPI collective I/O, was proposed. Lastly, for the fault mitigation, User Level Fault Mitigation, ULFM was improved to have faster agreement process, and sliding methods to substitute failed nodes with spare nodes was proposed. The funding of this project was ended in 2016; however, many proposed technologies are still being propelled.},
	isbn = {978-981-13-1924-2},
	doi = {10.1007/978-981-13-1924-2_4},
	url = {https://doi.org/10.1007/978-981-13-1924-2_4},
	author = {Atsushi Hori and Tsujita, Yuichi and Shimada, Akio and Yoshinaga, Kazumi and Mitaro, Namiki and Fukazawa, Go and Sato, Mikiko and George Bosilca and Aurelien Bouteiller and Thomas Herault},
	editor = {Sato, Mitsuhisa}
}
@conference {,
	title = {Towards Portable Online Prediction of Network Utilization Using MPI-Level Monitoring},
	booktitle = {2019 European Conference on Parallel Processing (Euro-Par 2019)},
	year = {2019},
	month = {2019-08},
	publisher = {Springer},
	organization = {Springer},
	address = {G{\"o}ttingen, Germany},
	abstract = {Stealing network bandwidth helps a variety of HPC runtimes and services to run additional operations in the background without negatively affecting the applications. A key ingredient to make this possible is an accurate prediction of the future network utilization, enabling the runtime to plan the background operations in advance, such as to avoid competing with the application for network bandwidth. In this paper, we propose a portable deep learning predictor that only uses the information available through MPI introspection to construct a recurrent sequence-to-sequence neural network capable of forecasting network utilization. We leverage the fact that most HPC applications exhibit periodic behaviors to enable predictions far into the future (at least the length of a period). Our online approach does not have an initial training phase, it continuously improves itself during application execution without incurring significant computational overhead. Experimental results show better accuracy and lower computational overhead compared with the state-of-the-art on two representative applications.},
	doi = {https://doi.org/10.1007/978-3-030-29400-7_4},
	author = {Shu-Mei Tseng and Bogdan Nicolae and George Bosilca and Emmanuel Jeannot and Aparna Chandramowlishwaran and Franck Cappello}
}
@conference {,
	title = {Understanding Scalability and Fine-Grain Parallelism of Synchronous Data Parallel Training},
	booktitle = {2019 IEEE/ACM Workshop on Machine Learning in High Performance Computing Environments (MLHPC)},
	year = {2019},
	month = {2019-11},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Denver, CO},
	abstract = {In the age of big data, deep learning has emerged as a powerful tool to extract insight and exploit its value, both in industry and scientific applications. With increasing complexity of learning models and amounts of training data, data-parallel approaches based on frequent all-reduce synchronization steps are increasingly popular. Despite the fact that high-performance computing (HPC) technologies have been designed to address such patterns efficiently, the behavior of data-parallel approaches on HPC platforms is not well understood. To address this issue, in this paper we study the behavior of Horovod, a popular data-parallel approach that relies on MPI, on Theta, a pre-Exascale machine at Argonne National Laboratory. Using two representative applications, we explore two aspects: (1) how performance and scalability is affected by important parameters such as number of nodes, number of workers, threads per node, batch size; (2) how computational phases are interleaved withall-reduce communication phases at fine granularity and what consequences this interleaving has in terms of potential bottlenecks. Our findings show that pipelining of back-propagation, gradient reduction and weight updates mitigate the effects of stragglers during all-reduce only partially. Furthermore, there can be significant delays between weights update, which can be leveraged to mask the overhead of additional background operations that are coupled with the training.},
	doi = {https://doi.org/10.1109/MLHPC49564.2019.00006},
	author = {Jiali Li and Bogdan Nicolae and Justin M. Wozniak and George Bosilca}
}
@conference {1205,
	title = {ADAPT: An Event-Based Adaptive Collective Communication Framework},
	booktitle = {The 27th International Symposium on High-Performance Parallel and Distributed Computing (HPDC {\textquoteright}18)},
	year = {2018},
	month = {2018-06},
	publisher = {ACM Press},
	organization = {ACM Press},
	address = {Tempe, Arizona},
	abstract = {The increase in scale and heterogeneity of high-performance computing (HPC) systems predispose the performance of Message Passing Interface (MPI) collective communications to be susceptible to noise, and to adapt to a complex mix of hardware capabilities. The designs of state of the art MPI collectives heavily rely on synchronizations; these designs magnify noise across the participating processes, resulting in significant performance slowdown. Therefore, such design philosophy must be reconsidered to efficiently and robustly run on the large-scale heterogeneous platforms. In this paper, we present ADAPT, a new collective communication framework in Open MPI, using event-driven techniques to morph collective algorithms to heterogeneous environments. The core concept of ADAPT is to relax synchronizations, while mamtaining the minimal data dependencies of MPI collectives. To fully exploit the different bandwidths of data movement lanes in heterogeneous systems, we extend the ADAPT collective framework with a topology-aware communication tree. This removes the boundaries of different hardware topologies while maximizing the speed of data movements. We evaluate our framework with two popular collective operations: broadcast and reduce on both CPU and GPU clusters. Our results demonstrate drastic performance improvements and a strong resistance against noise compared to other state of the art MPI libraries. In particular, we demonstrate at least 1.3X and 1.5X speedup for CPU data and 2X and 10X speedup for GPU data using ADAPT event-based broadcast and reduce operations.},
	isbn = {9781450357852},
	doi = {10.1145/3208040.3208054},
	author = {Xi Luo and Wei Wu and George Bosilca and Thananon Patinyasakdikul and Linnan Wang and Jack Dongarra}
}
@techreport {1200,
	title = {Data Movement Interfaces to Support Dataflow Runtimes},
	journal = {Innovative Computing Laboratory Technical Report},
	number = {ICL-UT-18-03},
	year = {2018},
	month = {2018-05},
	publisher = {University of Tennessee},
	abstract = {This document presents the design study and reports on the implementation of a portable hosted accelerator device support in the PaRSEC Dataflow Tasking at Exascale runtime, undertaken as part of the ECP contract 17-SC-20-SC. The document discusses different technological approaches to transfer data to/from hosted accelerators, issues recommendations for technology providers, and presents the design of an OpenMP-based accelerator support in PaRSEC.},
	author = {Aurelien Bouteiller and George Bosilca and Thomas Herault and Jack Dongarra}
}
@techreport {1320,
	title = {Distributed Termination Detection for HPC Task-Based Environments},
	journal = {Innovative Computing Laboratory Technical Report},
	number = {ICL-UT-18-14},
	year = {2018},
	month = {2018-06},
	publisher = {University of Tennessee},
	abstract = {This paper revisits distributed termination detection algorithms in the context of high-performance computing applications in task systems. We first outline the need to efficiently detect termination in workflows for which the total number of tasks is data dependent and therefore not known statically but only revealed dynamically during execution. We introduce an efficient variant of the Credit Distribution Algorithm (CDA) and compare it to the original algorithm (HCDA) as well as to its two primary competitors: the Four Counters algorithm (4C) and the Efficient Delay-Optimal Distributed algorithm (EDOD). On the theoretical side, we analyze the behavior of each algorithm for some simplified task-based kernels and show the superiority of CDA in terms of the number of control messages. On the practical side, we provide a highly tuned implementation of each termination detection algorithm within PaRSEC and compare their performance for a variety of benchmarks, extracted from scientific applications that exhibit dynamic behaviors.},
	author = {George Bosilca and Aurelien Bouteiller and Thomas Herault and Valentin Le F{\`e}vre and Yves Robert and Jack Dongarra}
}
@conference {1214,
	title = {Do moldable applications perform better on failure-prone HPC platforms?},
	booktitle = {11th Workshop on Resiliency in High Performance Computing in Clusters, Clouds, and Grids},
	series = {LNCS},
	year = {2018},
	month = {2018-08},
	publisher = {Springer Verlag},
	organization = {Springer Verlag},
	address = {Turin, Italy},
	abstract = {This paper compares the performance of different approaches to tolerate failures using checkpoint/restart when executed on large-scale failure-prone platforms. We study (i) Rigid applications, which use a constant number of processors throughout execution; (ii) Moldable applications, which can use a different number of processors after each restart following a fail-stop error; and (iii) GridShaped applications, which are moldable applications restricted to use rectangular processor grids (such as many dense linear algebra kernels). For each application type, we compute the optimal number of failures to tolerate before relinquishing the current allocation and waiting until a new resource can be allocated, and we determine the optimal yield that can be achieved. We instantiate our performance model with a realistic applicative scenario and make it publicly available for further usage.},
	author = {Valentin Le F{\`e}vre and George Bosilca and Aurelien Bouteiller and Thomas Herault and Atsushi Hori and Yves Robert and Jack Dongarra}
}
@article {1089,
	title = {A Failure Detector for HPC Platforms},
	journal = {The International Journal of High Performance Computing Applications},
	volume = {32},
	year = {2018},
	month = {2018-01},
	pages = {139{\textendash}158},
	abstract = {Building an infrastructure for exascale applications requires, in addition to many other key components, a stable and efficient failure detector. This article describes the design and evaluation of a robust failure detector that can maintain and distribute the correct list of alive resources within proven and scalable bounds. The detection and distribution of the fault information follow different overlay topologies that together guarantee minimal disturbance to the applications. A virtual observation ring minimizes the overhead by allowing each node to be observed by another single node, providing an unobtrusive behavior. The propagation stage uses a nonuniform variant of a reliable broadcast over a circulant graph overlay network and guarantees a logarithmic fault propagation. Extensive simulations, together with experiments on the Titan Oak Ridge National Laboratory supercomputer, show that the algorithm performs extremely well and exhibits all the desired properties of an exascale-ready algorithm.},
	keywords = {failure detection, Fault tolerance, MPI},
	doi = {https://doi.org/10.1177/1094342017711505},
	author = {George Bosilca and Aurelien Bouteiller and Amina Guermouche and Thomas Herault and Yves Robert and Pierre Sens and Jack Dongarra}
}
@conference {1196,
	title = {Optimal Cooperative Checkpointing for Shared High-Performance Computing Platforms},
	booktitle = {2018 IEEE International Parallel and Distributed Processing Symposium Workshops (IPDPSW), Best Paper Award},
	year = {2018},
	month = {2018-05},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Vancouver, BC, Canada},
	abstract = {In high-performance computing environments, input/output (I/O) from various sources often contend for scarce available bandwidth. Adding to the I/O operations inherent to the failure-free execution of an application, I/O from checkpoint/restart (CR) operations (used to ensure progress in the presence of failures) place an additional burden as it increase I/O contention, leading to degraded performance. In this work, we consider a cooperative scheduling policy that optimizes the overall performance of concurrently executing CR-based applications which share valuable I/O resources. First, we provide a theoretical model and then derive a set of necessary constraints needed to minimize the global waste on the platform. Our results demonstrate that the optimal checkpoint interval, as defined by Young/Daly, despite providing a sensible metric for a single application, is not sufficient to optimally address resource contention at the platform scale. We therefore show that combining optimal checkpointing periods with I/O scheduling strategies can provide a significant improvement on the overall application performance, thereby maximizing platform throughput. Overall, these results provide critical analysis and direct guidance on checkpointing large-scale workloads in the presence of competing I/O while minimizing the impact on application performance.},
	doi = {10.1109/IPDPSW.2018.00127},
	author = {Thomas Herault and Yves Robert and Aurelien Bouteiller and Dorian Arnold and Kurt Ferreira and George Bosilca and Jack Dongarra}
}
@article {1231,
	title = {A Survey of MPI Usage in the US Exascale Computing Project},
	journal = {Concurrency Computation: Practice and Experience},
	year = {2018},
	month = {2018-09},
	type = {Special Issue},
	abstract = {The Exascale Computing Project (ECP) is currently the primary effort in theUnited States focused on developing {\textquotedblleft}exascale{\textquotedblright} levels of computing capabilities, including hardware, software, and applications. In order to obtain amore thorough understanding of how the software projects under the ECPare using, and planning to use theMessagePassing Interface (MPI), and help guide the work of our own project within the ECP, we created a survey.Of the 97 ECP projects active at the time the survey was distributed, we received 77 responses, 56 of which reported that their projects were usingMPI. This paper reports the results of that survey for the benefit of the broader community of MPI developers.},
	keywords = {exascale, MPI},
	doi = {https://doi.org/10.1002/cpe.4851},
	author = {David E. Bernholdt and Swen Boehm and George Bosilca and Manjunath Gorentla Venkata and Ryan E. Grant and Thomas Naughton and Howard P. Pritchard and Martin Schulz and Geoffroy R. Vallee}
}
@techreport {1280,
	title = {Tensor Contraction on Distributed Hybrid Architectures using a Task-Based Runtime System},
	journal = {Innovative Computing Laboratory Technical Report},
	number = {ICL-UT-18-13},
	year = {2018},
	month = {2018-12},
	publisher = {University of Tennessee},
	abstract = {The needs for predictive simulation of electronic structure in chemistry and materials science calls for fast/reduced-scaling formulations of quantum n-body methods that replace the traditional dense tensors with element-, block-, rank-, and block-rank-sparse (data-sparse) tensors. The resulting, highly irregular data structures are a poor match to imperative, bulk-synchronous parallel programming style due to the dynamic nature of the problem and to the lack of clear domain decomposition to guarantee a fair load-balance. TESSE runtime and the associated programming model aim to support performance-portable composition of applications involving irregular and dynamically changing data. In this paper we report an implementation of irregular dense tensor contraction in a paradigmatic electronic structure application based on the TESSE extension of PaRSEC, a distributed hybrid task runtime system, and analyze the resulting performance on a distributed memory cluster of multi-GPU nodes. Unprecedented strong scaling and promising efficiency indicate a viable future for task-based programming of complete production-quality reduced scaling models of electronic structure.},
	author = {George Bosilca and Damien Genet and Robert Harrison and Thomas Herault and Mohammad Mahdi Javanmard and Chong Peng and Edward Valeev}
}
@article {1176,
	title = {Argobots: A Lightweight Low-Level Threading and Tasking Framework},
	journal = {IEEE Transactions on Parallel and Distributed Systems},
	year = {2017},
	month = {2017-10},
	abstract = {In the past few decades, a number of user-level threading and tasking models have been proposed in the literature to address the shortcomings of OS-level threads, primarily with respect to cost and flexibility. Current state-of-the-art user-level threading and tasking models, however, are either too specific to applications or architectures or are not as powerful or flexible. In this paper, we present Argobots, a lightweight, low-level threading and tasking framework that is designed as a portable and performant substrate for high-level programming models or runtime systems. Argobots offers a carefully designed execution model that balances generality of functionality with providing a rich set of controls to allow specialization by the user or high-level programming model. We describe the design, implementation, and optimization of Argobots and present integrations with three example high-level models: OpenMP, MPI, and co-located I/O service. Evaluations show that (1) Argobots outperforms existing generic threading runtimes; (2) our OpenMP runtime offers more efficient interoperability capabilities than production OpenMP runtimes do; (3) when MPI interoperates with Argobots instead of Pthreads, it enjoys reduced synchronization costs and better latency hiding capabilities; and (4) I/O service with Argobots reduces interference with co-located applications, achieving performance competitive with that of the Pthreads version.},
	keywords = {Argobots, context switch, I/O, interoperability, lightweight, MPI, OpenMP, stackable scheduler, tasklet, user-level thread},
	doi = {10.1109/TPDS.2017.2766062},
	url = {http://ieeexplore.ieee.org/document/8082139/},
	author = {Sangmin Seo and Abdelhalim Amer and Pavan Balaji and Cyril Bordage and George Bosilca and Alex Brooks and Philip Carns and Adrian Castello and Damien Genet and Thomas Herault and Shintaro Iwasaki and Prateek Jindal and Sanjay Kale and Sriram Krishnamoorthy and Jonathan Lifflander and Huiwei Lu and Esteban Meneses and Mar Snir and Yanhua Sun and Kenjiro Taura and Pete Beckman}
}
@inproceedings {1183,
	title = {Dynamic Task Discovery in PaRSEC- A data-flow task-based Runtime},
	journal = {ScalA17},
	year = {2017},
	month = {2017-09},
	publisher = {ACM},
	address = {Denver},
	abstract = {Successfully exploiting distributed collections of heterogeneous many-cores architectures with complex memory hierarchy through a portable programming model is a challenge for application developers. The literature is not short of proposals addressing this problem, including many evolutionary solutions that seek to extend the capabilities of current message passing paradigms with intranode features (MPI+X). A different, more revolutionary, solution explores data-flow task-based runtime systems as a substitute to both local and distributed data dependencies management. The solution explored in this paper, PaRSEC, is based on such a programming paradigm, supported by a highly efficient task-based runtime. This paper compares two programming paradigms present in PaRSEC, Parameterized Task Graph (PTG) and Dynamic Task Discovery (DTD) in terms of capabilities, overhead and potential benefits.},
	keywords = {data-flow, dynamic task-graph, parsec, task-based runtime},
	isbn = {978-1-4503-5125-6},
	doi = {10.1145/3148226.3148233},
	url = {https://dl.acm.org/citation.cfm?doid=3148226.3148233},
	author = {Reazul Hoque and Thomas Herault and George Bosilca and Jack Dongarra}
}
@conference {1100,
	title = {Efficient Communications in Training Large Scale Neural Networks},
	booktitle = {ACM MultiMedia Workshop 2017},
	year = {2017},
	month = {2017-10},
	publisher = {ACM},
	organization = {ACM},
	address = {Mountain View, CA},
	abstract = {We consider the problem of how to reduce the cost of communication that is required for the parallel training of a neural network. The state-of-the-art method, Bulk Synchronous Parallel Stochastic Gradient Descent (BSP-SGD), requires many collective communication operations, like broadcasts of parameters or reductions for sub-gradient aggregations, which for large messages quickly dominates overall execution time and limits parallel scalability. To address this problem, we develop a new technique for collective operations, referred to as Linear Pipelining (LP). It is tuned to the message sizes that arise in BSP-SGD, and works effectively on multi-GPU systems. Theoretically, the cost of LP is invariant to P, where P is the number of GPUs, while the cost of more conventional Minimum Spanning Tree (MST) scales like O(logP). LP also demonstrate up to 2x faster bandwidth than Bidirectional Exchange (BE) techniques that are widely adopted by current MPI implementations. We apply these collectives to BSP-SGD, showing that the proposed implementations reduce communication bottlenecks in practice while preserving the attractive convergence properties of BSP-SGD.},
	author = {Yiyang Zhao and Linnan Wan and Wei Wu and George Bosilca and Richard Vuduc and Jinmian Ye and Wenqi Tang and Zenglin Xu}
}
@conference {1127,
	title = {Using Software-Based Performance Counters to Expose Low-Level Open MPI Performance Information},
	booktitle = {EuroMPI},
	year = {2017},
	month = {2017-09},
	publisher = {ACM},
	organization = {ACM},
	address = {Chicago, IL},
	abstract = {This paper details the implementation and usage of software-based performance counters to understand the performance of a particular implementation of the MPI standard, Open MPI.  Such counters can expose intrinsic features of the software stack that are not available otherwise in a generic and portable way. The PMPI-interface is useful for instrumenting MPI applications at a user level, however it is insufficient for providing meaningful internal MPI performance details.  While the Peruse interface provides more detailed information on state changes within Open MPI, it has not seen widespread adoption.  We introduce a simple low-level approach that instruments the Open MPI code at key locations to provide fine-grained MPI performance metrics.  We evaluate the overhead associated with adding these counters to Open MPI as well as their use in determining bottlenecks and areas for improvement both in user code and the MPI implementation itself.},
	keywords = {MPI, Performance Counters, Profiling, Tools},
	isbn = {978-1-4503-4849-2/17/09},
	doi = {https://doi.org/10.1145/3127024.3127039},
	url = {https://dl.acm.org/citation.cfm?id=3127024},
	author = {David Eberius and Thananon Patinyasakdikul and George Bosilca}
}
@article {924,
	title = {Assessing the Cost of Redistribution followed by a Computational Kernel: Complexity and Performance Results},
	journal = {Parallel Computing},
	volume = {52},
	year = {2016},
	month = {2016-02},
	pages = {22-41},
	abstract = {The classical redistribution problem aims at optimally scheduling communications when reshuffling from an initial data distribution to a target data distribution. This target data distribution is usually chosen to optimize some objective for the algorithmic kernel under study (good computational balance or low communication volume or cost), and therefore to provide high efficiency for that kernel. However, the choice of a distribution minimizing the target objective is not unique. This leads to generalizing the redistribution problem as follows: find a re-mapping of data items onto processors such that the data redistribution cost is minimal, and the operation remains as efficient. This paper studies the complexity of this generalized problem. We compute optimal solutions and evaluate, through simulations, their gain over classical redistribution. We also show the NP-hardness of the problem to find the optimal data partition and processor permutation (defined by new subsets) that minimize the cost of redistribution followed by a simple computational kernel. Finally, experimental validation of the new redistribution algorithms are conducted on a multicore cluster, for both a 1D-stencil kernel and a more compute-intensive dense linear algebra routine.},
	keywords = {Data partition, linear algebra, parsec, QR factorization, Redistribution, Stencil},
	doi = {doi:10.1016/j.parco.2015.09.005},
	author = {Julien Herrmann and George Bosilca and Thomas Herault and Loris Marchal and Yves Robert and Jack Dongarra}
}
@techreport {978,
	title = {Context Identifier Allocation in Open MPI},
	journal = {University of Tennessee Computer Science Technical Report},
	number = { ICL-UT-16-01},
	year = {2016},
	month = {2016-01},
	publisher = {Innovative Computing Laboratory, University of Tennessee},
	abstract = {The concept of communicators is a central notion in Message Passing Interface, allowing on one side the MPI implemen- tation to specialize it{\textquoteright}s matching and deliver messages in the right context, and on the other side the library developers to contextualize their message exchanges, and scope different algorithms to well-defined groups of processes. More pre- cisely, all communication objects in MPI are derived from a communicator at some point. All MPI functions allowing the creation of new communicators have a collective mean- ing, either over the group of processes from the parent com- municator or those from the target communicator. Thus, the perfromance of the communicator creation is tied to the col- lective communication performance, as well as the amount of data needed to be exchanged in order to consistently create this new communicator. We introduce several communica- tor creation algorithms, and present their implementation in the context of Open MPI. We explore the performance of these new algorithms and compare them with state-of-the- art algorithms available in other MPI implementations.},
	author = {George Bosilca and Thomas Herault and Jack Dongarra}
}
@inproceedings {979,
	title = {Failure Detection and Propagation in HPC Systems},
	journal = { Proceedings of the The International Conference for High Performance Computing, Networking, Storage and Analysis (SC{\textquoteright}16)},
	year = {2016},
	month = {2016-11},
	pages = {27:1-27:11},
	publisher = {IEEE Press},
	address = {Salt Lake City, Utah},
	keywords = {failure detection, fault-tolerance, MPI},
	isbn = {978-1-4673-8815-3},
	url = {http://dl.acm.org/citation.cfm?id=3014904.3014941},
	author = {George Bosilca and Aurelien Bouteiller and Amina Guermouche and Thomas Herault and Yves Robert and Pierre Sens and Jack Dongarra}
}
@conference {941,
	title = {GPU-Aware Non-contiguous Data Movement In Open MPI},
	booktitle = {25th International Symposium on High-Performance Parallel and Distributed Computing (HPDC{\textquoteright}16)},
	year = {2016},
	month = {2016-06},
	publisher = {ACM},
	organization = {ACM},
	address = {Kyoto, Japan},
	abstract = {<p>Due to better parallel density and power efficiency, GPUs have become more popular for use in scientific applications. Many of these applications are based on the ubiquitous Message Passing Interface (MPI) programming paradigm, and take advantage of non-contiguous memory layouts to exchange data between processes. However, support for efficient non-contiguous data movements for GPU-resident data is still in its infancy, imposing a negative impact on the overall application performance.</p>

<p>To address this shortcoming, we present a solution where we take advantage of the inherent parallelism in the datatype packing and unpacking operations. We developed a close integration between Open MPI{\textquoteright}s stack-based datatype engine, NVIDIA{\textquoteright}s Unied Memory Architecture and GPUDirect capabilities. In this design the datatype packing and unpacking operations are offloaded onto the GPU and handled by specialized GPU kernels, while the CPU remains the driver for data movements between nodes. By incorporating our design into the Open MPI library we have shown significantly better performance for non-contiguous GPU-resident data transfers on both shared and distributed memory machines.</p>},
	keywords = {datatype, gpu, hybrid architecture, MPI, non-contiguous data},
	doi = {http://dx.doi.org/10.1145/2907294.2907317},
	author = {Wei Wu and George Bosilca and Rolf vandeVaart and Sylvain Jeaugey and Jack Dongarra}
}
@inproceedings {1308,
	title = {Surviving Errors with OpenSHMEM},
	journal = {OpenSHMEM and Related Technologies. Enhancing OpenSHMEM for Hybrid Environments},
	year = {2016},
	pages = {66{\textendash}81},
	publisher = {Springer International Publishing},
	address = {Baltimore, MD, USA},
	abstract = {Unexpected error conditions stem from a variety of underlying causes, including resource exhaustion, network failures, hardware failures, or program errors. As the scale of HPC systems continues to grow, so does the probability of encountering a condition that causes a failure; meanwhile, error recovery and run-through failure management are becoming mature, and interoperable HPC programming paradigms are beginning to feature advanced error management. As a result from these developments, it becomes increasingly desirable to gracefully handle error conditions in OpenSHMEM. In this paper, we present the design and rationale behind an extension of the OpenSHMEM API that can (1) notify user code of unexpected erroneous conditions, (2) permit customized user response to errors without incurring overhead on an error-free execution path, (3) propagate the occurence of an error condition to all Processing Elements, and (4) consistently close the erroneous epoch in order to resume the application.},
	isbn = {978-3-319-50995-2},
	author = {Aurelien Bouteiller and George Bosilca and Manjunath Gorentla Venkata},
	editor = {Manjunath Gorentla Venkata and Imam, Neena and Pophale, Swaroop and Mintz, Tiffany M.}
}
@conference {921,
	title = {Accelerating NWChem Coupled Cluster through dataflow-based Execution},
	booktitle = {11th International Conference on Parallel Processing and Applied Mathematics (PPAM 2015)},
	year = {2015},
	month = {2015-09},
	publisher = {Springer International Publishing},
	organization = {Springer International Publishing},
	address = {Krakow, Poland},
	abstract = {Numerical techniques used for describing many-body systems, such as the Coupled Cluster methods (CC) of the quantum chemistry package NWChem, are of extreme interest to the computational chemistry community in fields such as catalytic reactions, solar energy,
and bio-mass conversion. In spite of their importance, many of these computationally intensive algorithms have traditionally been thought of in a fairly linear fashion, or are parallelised in coarse chunks. In this paper, we present our effort of converting the NWChem{\textquoteright}s
CC code into a dataflow-based form that is capable of utilizing the task scheduling system PaRSEC (Parallel Runtime Scheduling and Execution Controller) {\textendash} a software package designed to enable high performance computing at scale. We discuss the modularity of our approach and explain how the PaRSEC-enabled dataflow version of the subroutines seamlessly integrate into the NWChem codebase. Furthermore, we argue how the CC algorithms can be easily decomposed into finer grained tasks (compared to the original version of NWChem); and how data distribution and load balancing are decoupled and can be tuned independently. We demonstrate performance acceleration by more than a factor of two in the execution of the entire CC component of NWChem, concluding that the utilization of dataflow-based execution for CC methods enables more efficient and scalable computation.},
	keywords = {CCSD, dag, dataflow, NWChem, parsec, ptg, tasks},
	author = {Heike Jagode and Anthony Danalis and George Bosilca and Jack Dongarra}
}
@article {820,
	title = {Algorithm-based Fault Tolerance for Dense Matrix Factorizations, Multiple Failures, and Accuracy},
	journal = {ACM Transactions on Parallel Computing},
	volume = {1},
	number = {10},
	year = {2015},
	month = {2015-01},
	pages = {10:1-10:28},
	abstract = {Dense matrix factorizations, such as LU, Cholesky and QR, are widely used for scientific applications that
require solving systems of linear equations, eigenvalues and linear least squares problems. Such computations
are normally carried out on supercomputers, whose ever-growing scale induces a fast decline of the
Mean Time To Failure (MTTF). This paper proposes a new hybrid approach, based on Algorithm-Based Fault
Tolerance (ABFT), to help matrix factorizations algorithms survive fail-stop failures. We consider extreme
conditions, such as the absence of any reliable node and the possibility of losing both data and checksum from a single failure. We will present a generic solution for protecting the right factor, where the updates are applied, of all above mentioned factorizations. For the left factor, where the panel has been applied, we propose a scalable checkpointing algorithm. This algorithm features high degree of checkpointing parallelism and cooperatively utilizes the checksum storage leftover from the right factor protection. The fault-tolerant algorithms derived from this hybrid solution is applicable to a wide range of dense matrix factorizations, with minor modifications. Theoretical analysis shows that the fault tolerance overhead decreases inversely to the scaling in the number of computing units and the problem size. Experimental results of LU and QR factorization on the Kraken (Cray XT5) supercomputer validate the theoretical evaluation and confirm negligible overhead, with- and without-errors. Applicability to tolerate multiple failures and accuracy after multiple recovery is also considered.},
	keywords = {ABFT, algorithms, fault-tolerance, High Performance Computing, linear algebra},
	doi = {10.1145/2686892},
	author = {Aurelien Bouteiller and Thomas Herault and George Bosilca and Peng Du and Jack Dongarra},
	editor = {Phillip B. Gibbons}
}
@article {852,
	title = {Composing Resilience Techniques: ABFT, Periodic, and Incremental Checkpointing},
	journal = {International Journal of Networking and Computing},
	volume = {5},
	number = {1},
	year = {2015},
	month = {2015-01},
	pages = {2-15},
	abstract = {Algorithm Based Fault Tolerant (ABFT) approaches promise unparalleled scalability and performance in failure-prone environments. Thanks to recent advances in the understanding of the involved mechanisms, a growing number of important algorithms (including all widely used factorizations) have been proven ABFT-capable. In the context of larger applications, these algorithms provide a temporal section of the execution, where the data is protected by its own intrinsic properties, and can therefore be algorithmically recomputed without the need of checkpoints. However, while typical scientific applications spend a significant fraction of their execution time in library calls that can be ABFT-protected, they interleave sections that are difficult or even impossible to protect with ABFT. As a consequence, the only practical fault-tolerance approach for these applications is checkpoint/restart. In this paper we propose a model to investigate the efficiency of a composite protocol, that alternates between ABFT and checkpoint/restart for the effective protection of an iterative application composed of ABFT- aware and ABFT-unaware sections. We also consider an incremental checkpointing composite approach in which the algorithmic knowledge is leveraged by a novel optimal dynamic program- ming to compute checkpoint dates. We validate these models using a simulator. The model and simulator show that the composite approach drastically increases the performance delivered by an execution platform, especially at scale, by providing the means to increase the interval between checkpoints while simultaneously decreasing the volume of each checkpoint. },
	keywords = {ABFT, checkpoint, fault-tolerance, High-performance computing, model, performance evaluation, resilience},
	issn = {ISSN 2185-2839},
	author = {George Bosilca and Aurelien Bouteiller and Thomas Herault and Yves Robert and Jack Dongarra}
}
@conference {840,
	title = {Design for a Soft Error Resilient Dynamic Task-based Runtime},
	booktitle = {29th IEEE International Parallel \& Distributed Processing Symposium (IPDPS)},
	year = {2015},
	month = {2015-05},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Hyderabad, India},
	abstract = {As the scale of modern computing systems grows, failures will happen more frequently. On the way to Exascale a generic, low-overhead, resilient extension becomes a desired aptitude of any programming paradigm. In this paper we explore three additions to a dynamic task-based runtime to build a generic framework providing soft error resilience to task-based programming paradigms. The first recovers the application by re-executing the minimum required sub-DAG, the second takes critical checkpoints of the data flowing between tasks to minimize the necessary re-execution, while the last one takes advantage of algorithmic properties to recover the data without re-execution. These mechanisms have been implemented in the PaRSEC task-based runtime framework. Experimental results validate our approach and quantify the overhead introduced by such mechanisms.},
	author = {Chongxiao Cao and George Bosilca and Thomas Herault and Jack Dongarra}
}
@inproceedings {1310,
	title = {From MPI to OpenSHMEM: Porting LAMMPS},
	journal = {OpenSHMEM and Related Technologies. Experiences, Implementations, and Technologies},
	year = {2015},
	pages = {121{\textendash}137},
	publisher = {Springer International Publishing},
	address = {Annapolis, MD, USA},
	abstract = {This work details the opportunities and challenges of porting a Petascale, MPI-based application {\textendash}-LAMMPS{\textendash}- to OpenSHMEM. We investigate the major programming challenges stemming from the differences in communication semantics, address space organization, and synchronization operations between the two programming models. This work provides several approaches to solve those challenges for representative communication patterns in LAMMPS, e.g., by considering group synchronization, peer{\textquoteright}s buffer status tracking, and unpacked direct transfer of scattered data. The performance of LAMMPS is evaluated on the Titan HPC system at ORNL. The OpenSHMEM implementations are compared with MPI versions in terms of both strong and weak scaling. The results outline that OpenSHMEM provides a rich semantic to implement scalable scientific applications. In addition, the experiments demonstrate that OpenSHMEM can compete with, and often improve on, the optimized MPI implementation.},
	isbn = {978-3-319-26428-8},
	doi = {10.1007/978-3-319-26428-8_8},
	author = {Tang, Chunyan and Aurelien Bouteiller and Thomas Herault and Manjunath Gorentla Venkata and George Bosilca},
	editor = {Manjunath Gorentla Venkata and Shamis, Pavel and Imam, Neena and M. Graham Lopez}
}
@conference {841,
	title = {Hierarchical DAG scheduling for Hybrid Distributed Systems},
	booktitle = {29th IEEE International Parallel \& Distributed Processing Symposium (IPDPS)},
	year = {2015},
	month = {2015-05},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Hyderabad, India},
	abstract = {Accelerator-enhanced computing platforms have drawn a lot of attention due to their massive peak com-putational capacity. Despite significant advances in the pro-gramming interfaces to such hybrid architectures, traditional programming paradigms struggle mapping the resulting multi-dimensional heterogeneity and the expression of algorithm parallelism, resulting in sub-optimal effective performance. Task-based programming paradigms have the capability to alleviate some of the programming challenges on distributed hybrid many-core architectures. In this paper we take this concept a step further by showing that the potential of task-based programming paradigms can be greatly increased with minimal modification of the underlying runtime combined with the right algorithmic changes. We propose two novel recursive algorithmic variants for one-sided factorizations and describe the changes to the PaRSEC task-scheduling runtime to build a framework where the task granularity is dynamically adjusted to adapt the degree of available parallelism and kernel effi-ciency according to runtime conditions. Based on an extensive set of results we show that, with one-sided factorizations, i.e. Cholesky and QR, a carefully written algorithm, supported by an adaptive tasks-based runtime, is capable of reaching a degree of performance and scalability never achieved before in distributed hybrid environments. },
	keywords = {dense linear algebra, gpu, heterogeneous architecture, PaRSEC runtime},
	author = {Wei Wu and Aurelien Bouteiller and George Bosilca and Mathieu Faverge and Jack Dongarra}
}
@conference {915,
	title = {PaRSEC in Practice: Optimizing a Legacy Chemistry Application through Distributed Task-Based Execution},
	booktitle = {2015 IEEE International Conference on Cluster Computing},
	year = {2015},
	month = {2015-09},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Chicago, IL},
	abstract = {Task-based execution has been growing in popularity as a means to deliver a good balance between performance and portability in the post-petascale era. The Parallel Runtime Scheduling and Execution Control (PARSEC) framework is a task-based runtime system that we designed to achieve high performance computing at scale. PARSEC offers a programming paradigm that is different than what has been traditionally used to develop large scale parallel scientific applications. In this paper, we discuss the use of PARSEC to convert a part of the Coupled Cluster (CC) component of the Quantum Chemistry package NWCHEM into a task-based form. We explain how we organized the computation of the CC methods in individual tasks with explicitly defined data dependencies between them and re-integrated the modified code into NWCHEM. We present a thorough performance evaluation and demonstrate that the modified code outperforms the original by more than a factor of two. We also compare the performance of different variants of the modified code and explain the different behaviors that lead to the differences in performance.},
	keywords = {dag, parsec, ptg, tasks},
	author = {Anthony Danalis and Heike Jagode and George Bosilca and Jack Dongarra}
}
@conference {890,
	title = {Plan B: Interruption of Ongoing MPI Operations to Support Failure Recovery},
	booktitle = {22nd European MPI Users{\textquoteright} Group Meeting},
	year = {2015},
	month = {2015-09},
	publisher = {ACM},
	organization = {ACM},
	address = {Bordeaux, France},
	abstract = {Advanced failure recovery strategies in HPC system benefit tremendously from in-place failure recovery, in which the MPI infrastructure can survive process crashes and resume communication services. In this paper we present the rationale behind the specification, and an effective implementation of the Revoke MPI operation. The purpose of the Revoke operation is the propagation of failure knowledge, and
the interruption of ongoing, pending communication, under the control of the user. We explain that the Revoke operation can be implemented with a reliable broadcast over the scalable and failure resilient Binomial Graph (BMG) overlay network. Evaluation at scale, on a Cray XC30 supercomputer, demonstrates that the Revoke operation has a small latency, and does not introduce system noise outside of failure recovery periods.},
	doi = {10.1145/2802658.2802668},
	author = {Aurelien Bouteiller and George Bosilca and Jack Dongarra}
}
@techreport {872,
	title = {Practical Scalable Consensus for Pseudo-Synchronous Distributed Systems: Formal Proof},
	journal = {Innovative Computing Laboratory Technical Report},
	number = {ICL-UT-15-01},
	year = {2015},
	month = {2015-04},
	author = {Thomas Herault and Aurelien Bouteiller and George Bosilca and Marc Gamell and Keita Teranishi and Manish Parashar and Jack Dongarra}
}
@conference {889,
	title = {Practical Scalable Consensus for Pseudo-Synchronous Distributed Systems},
	booktitle = {The International Conference for High Performance Computing, Networking, Storage and Analysis (SC15)},
	year = {2015},
	month = {2015-11},
	publisher = {ACM},
	organization = {ACM},
	address = {Austin, TX},
	abstract = {The ability to consistently handle faults in a distributed environment requires, among a small set of basic routines, an agreement algorithm allowing surviving entities to reach a consensual decision between a bounded set of volatile resources. This paper presents an algorithm that implements an Early Returning Agreement (ERA) in pseudo-synchronous systems, which optimistically allows a process to resume its activity while guaranteeing strong progress. We prove the correctness of our ERA algorithm, and expose its logarithmic behavior, which is an extremely desirable property for any algorithm which targets future exascale platforms. We detail a practical implementation of this consensus algorithm in the context of an MPI library, and evaluate both its efficiency and scalability through a set of benchmarks and two fault tolerant scientific applications.},
	author = {Thomas Herault and Aurelien Bouteiller and George Bosilca and Marc Gamell and Keita Teranishi and Manish Parashar and Jack Dongarra}
}
@inproceedings {1309,
	title = {UCX: An Open Source Framework for HPC Network APIs and Beyond},
	journal = {2015 IEEE 23rd Annual Symposium on High-Performance Interconnects},
	year = {2015},
	month = {Aug},
	pages = {40-43},
	publisher = {IEEE},
	address = {Santa Clara, CA, USA},
	abstract = {This paper presents Unified Communication X (UCX), a set of network APIs and their implementations for high throughput computing. UCX comes from the combined effort of national laboratories, industry, and academia to design and implement a high-performing and highly-scalable network stack for next generation applications and systems. UCX design provides the ability to tailor its APIs and network functionality to suit a wide variety of application domains and hardware. We envision these APIs to satisfy the networking needs of many programming models such as Message Passing Interface (MPI), OpenSHMEM, Partitioned Global Address Space (PGAS) languages, task-based paradigms and I/O bound applications. To evaluate the design we implement the APIs and protocols, and measure the performance of overhead-critical network primitives fundamental for implementing many parallel programming models and system libraries. Our results show that the latency, bandwidth, and message rate achieved by the portable UCX prototype is very close to that of the underlying driver. With UCX, we achieved a message exchange latency of 0.89 us, a bandwidth of 6138.5 MB/s, and a message rate of 14 million messages per second. As far as we know, this is the highest bandwidth and message rate achieved by any network stack (publicly known) on this hardware.},
	keywords = {application program interfaces, Bandwidth, Electronics packaging, Hardware, high throughput computing, highly-scalable network stack, HPC, HPC network APIs, I/O bound applications, Infiniband, input-output programs, Libraries, Memory management, message passing, message passing interface, Middleware, MPI, open source framework, OpenSHMEM, parallel programming, parallel programming models, partitioned global address space languages, PGAS, PGAS languages, Programming, protocols, public domain software, RDMA, system libraries, task-based paradigms, UCX, Unified Communication X},
	isbn = {978-1-4673-9160-3},
	doi = {10.1109/HOTI.2015.13},
	author = {P. Shamis and Manjunath Gorentla Venkata and M. Graham Lopez and M. B. Baker and O. Hernandez and Y. Itigin and M. Dubman and G. Shainer and R. L. Graham and L. Liss and Y. Shahar and S. Potluri and D. Rossetti and D. Becker and D. Poole and C. Lamb and S. Kumar and C. Stunkel and George Bosilca and Aurelien Bouteiller}
}
@conference {853,
	title = {Assembly Operations for Multicore Architectures using Task-Based Runtime Systems},
	booktitle = {Euro-Par 2014},
	year = {2014},
	month = {2014-08},
	publisher = {Springer International Publishing},
	organization = {Springer International Publishing},
	address = {Porto, Portugal},
	abstract = {Traditionally, numerical simulations based on finite element methods consider the algorithm as being divided in three major steps: the generation of a set of blocks and vectors, the assembly of these blocks in a matrix and a big vector, and the inversion of the matrix. In this paper we tackle the second step, the block assembly, where no parallel algorithm is widely available. Several strategies are proposed to decompose the assembly problem while relying on a scheduling middle-ware to maximize the overlap between stages and increase the parallelism and thus the performance. These strategies are quantified using examples covering two extremes in the field, large number of non-overlapping small blocks for CFD-like problems, and a smaller number of larger blocks with significant overlap which can be met in sparse linear algebra solvers.},
	author = {Damien Genet and Abdou Guermouche and George Bosilca}
}
@conference {708,
	title = {Assessing the Impact of ABFT and Checkpoint Composite Strategies},
	booktitle = {16th Workshop on Advances in Parallel and Distributed Computational Models, IPDPS 2014},
	year = {2014},
	month = {2014-05},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Phoenix, AZ},
	abstract = {Algorithm-specific fault tolerant approaches promise unparalleled scalability and performance in failure-prone environments. With the advances in the theoretical and practical understanding of algorithmic traits enabling such approaches, a growing number of frequently used algorithms (including all widely used factorization kernels) have been proven capable of such properties. These algorithms provide a temporal section of the execution when the data is protected by it{\textquoteright}s own intrinsic properties, and can be algorithmically recomputed without the need of checkpoints. However, while typical scientific applications spend a significant fraction of their execution time in library calls that can be ABFT-protected, they interleave sections that are difficult or even impossible to protect with ABFT. As a consequence, the only fault-tolerance approach that is currently used for these applications is checkpoint/restart. In this paper we propose a model and a simulator to investigate the behavior of a composite protocol, that alternates between ABFT and checkpoint/restart protection for effective protection of each phase of an iterative application composed of ABFT-aware and ABFTunaware sections. We highlight this approach drastically increases the performance delivered by the system, especially at scale, by providing means to rarefy the checkpoints while simultaneously decreasing the volume of data needed to be checkpointed.},
	keywords = {ABFT, checkpoint, fault-tolerance, High-performance computing, resilience},
	author = {George Bosilca and Aurelien Bouteiller and Thomas Herault and Yves Robert and Jack Dongarra}
}
@techreport {822,
	title = {Design for a Soft Error Resilient Dynamic Task-based Runtime},
	journal = {ICL Technical Report},
	number = {ICL-UT-14-04},
	year = {2014},
	month = {2014-11},
	publisher = {University of Tennessee},
	abstract = {Abstract{\textemdash}As the scale of modern computing systems grows, failures will happen more frequently. On the way to Exascale a generic, low-overhead, resilient extension becomes a desired aptitude of any programming paradigm. In this paper we explore three additions to a dynamic task-based runtime to build a generic framework providing soft error resilience to task-based programming paradigms. The first recovers the application by re-executing the minimum required sub-DAG, the second takes critical checkpoints of the data flowing between tasks to minimize the necessary re-execution, while the last one takes advantage of algorithmic properties to recover the data without re-execution. These mechanisms have been implemented in the PaRSEC task-based runtime framework. Experimental results validate our approach and quantify the overhead introduced by such mechanisms.},
	author = {Chongxiao Cao and Thomas Herault and George Bosilca and Jack Dongarra}
}
@article {821,
	title = {An Efficient Distributed Randomized Algorithm for Solving Large Dense Symmetric Indefinite Linear Systems},
	journal = {Parallel Computing},
	volume = {40},
	year = {2014},
	month = {2014-07},
	pages = {213-223},
	abstract = {Randomized algorithms are gaining ground in high-performance computing applications as they have the potential to outperform deterministic methods, while still providing accurate results. We propose a randomized solver for distributed multicore architectures to efficiently solve large dense symmetric indefinite linear systems that are encountered, for instance, in parameter estimation problems or electromagnetism simulations. The contribution of this paper is to propose efficient kernels for applying random butterfly transformations and a new distributed implementation combined with a runtime (PaRSEC) that automatically adjusts data structures, data mappings, and the scheduling as systems scale up. Both the parallel distributed solver and the supporting runtime environment are innovative. To our knowledge, the randomization approach associated with this solver has never been used in public domain software for symmetric indefinite systems. The underlying runtime framework allows seamless data mapping and task scheduling, mapping its capabilities to the underlying hardware features of heterogeneous distributed architectures. The performance of our software is similar to that obtained for symmetric positive definite systems, but requires only half the execution time and half the amount of data storage of a general dense solver.},
	keywords = {Distributed linear algebra solvers, LDLT factorization, PaRSEC runtime, plasma, Randomized algorithms, Symmetric indefinite systems},
	doi = {10.1016/j.parco.2013.12.003},
	author = {Marc Baboulin and Du Becker and George Bosilca and Anthony Danalis and Jack Dongarra}
}
@conference {867,
	title = {A Multithreaded Communication Substrate for OpenSHMEM},
	booktitle = {8th International Conference on Partitioned Global Address Space Programming Models (PGAS)},
	year = {2014},
	month = {2014-10},
	address = {Eugene, OR},
	abstract = {OpenSHMEM scalability is strongly dependent on the capa- bility of its communication layer to efficiently handle multi- ple threads. In this paper, we present an early evaluation of the thread safety specification in the Unified Common Com- munication Substrate (UCCS) employed in OpenSHMEM. Results demonstrate that thread safety can be provided at an acceptable cost and can improve efficiency for some op- erations, compared to serializing communication.},
	author = {Aurelien Bouteiller and Thomas Herault and George Bosilca}
}
@conference {871,
	title = {PTG: An Abstraction for Unhindered Parallelism},
	booktitle = {International Workshop on Domain-Specific Languages and High-Level Frameworks for High Performance Computing (WOLFHPC)},
	year = {2014},
	month = {2014-11},
	publisher = {IEEE Press},
	organization = {IEEE Press},
	address = {New Orleans, LA},
	abstract = {<p>
Increased parallelism and use of heterogeneous computing resources is now an established trend in High Performance Computing (HPC), a trend that, looking forward to Exascale, seems bound to intensify. Despite the evolution of hardware over the past decade, the programming paradigm of choice was invariably derived from Coarse Grain Parallelism with explicit data movements. We argue that message passing has remained the de facto standard in HPC because, until now, the ever increasing challenges that application developers had to address to create efficient portable applications remained manageable for expert programmers.
</p>
<p>
Data-flow based programming is an alternative approach with significant potential. In this paper, we discuss the Parameterized Task Graph (PTG) abstraction and present the specialized input language that we use to specify PTGs in our data-flow task-based runtime system, PaRSEC. This language and the corresponding execution model are in contrast with the execution model of explicit message passing as well as the model of alternative task based runtime systems. The Parameterized Task Graph language decouples the expression of the parallelism in the algorithm from the control-flow ordering, load balance, and data distribution. Thus, programs are more adaptable and map more efficiently on challenging hardware, as well as maintain portability across diverse architectures. To support these claims, we discuss the different challenges of HPC programming and how PaRSEC can address them, and we demonstrate that in today{\textquoteright}s large scale supercomputers, PaRSEC can significantly outperform state-of-the-art MPI applications and libraries, a trend that will increase with future architectural evolution.
</p>},
	keywords = {dte, parsec, plasma},
	author = {Anthony Danalis and George Bosilca and Aurelien Bouteiller and Thomas Herault and Jack Dongarra}
}
@conference {709,
	title = {Taking Advantage of Hybrid Systems for Sparse Direct Solvers via Task-Based Runtimes},
	booktitle = {23rd International Heterogeneity in Computing Workshop, IPDPS 2014},
	year = {2014},
	month = {2014-05},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Phoenix, AZ},
	abstract = {The ongoing hardware evolution exhibits an escalation in the number, as well as in the heterogeneity, of the computing resources. The pressure to maintain reasonable levels of performance and portability, forces the application developers to leave the traditional programming paradigms and explore alternative solutions. PaStiX is a parallel sparse direct solver, based on a dynamic scheduler for modern hierarchical architectures. In this paper, we study the replacement of the highly specialized internal scheduler in PaStiX by two generic runtime frameworks: PaRSEC and StarPU. The tasks graph of the factorization step is made available to the two runtimes, providing them with the opportunity to optimize it in order to maximize the algorithm eefficiency for a predefined execution environment. A comparative study of the performance of the PaStiX solver with the three schedulers { native PaStiX, StarPU and PaRSEC schedulers { on different execution contexts is performed. The analysis highlights the similarities from a performance point of view between the different execution supports. These results demonstrate that these generic DAG-based runtimes provide a uniform and portable programming interface across heterogeneous environments, and are, therefore, a sustainable solution for hybrid environments.},
	keywords = {DAG based runtime, gpu, Multicore, Sparse linear solver},
	author = {Xavier Lacoste and Mathieu Faverge and Pierre Ramet and Samuel Thibault and George Bosilca}
}
@conference {863,
	title = {Task-Based Programming for Seismic Imaging: Preliminary Results},
	booktitle = {2014 IEEE International Conference on High Performance Computing and Communications (HPCC)},
	year = {2014},
	month = {2014-08},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Paris, France},
	abstract = {The level of hardware complexity of current supercomputers is forcing the High Performance Computing (HPC) community to reconsider parallel programming paradigms and standards. The high-level of hardware abstraction provided by task-based paradigms make them excellent candidates for writing portable codes that can consistently deliver high performance across a wide range of platforms. While this paradigm has proved efficient for achieving such goals for dense and sparse linear solvers, it is yet to be demonstrated that industrial parallel codes{\textemdash}relying on the classical Message Passing Interface (MPI) standard and that accumulate dozens of years of expertise (and countless lines of code){\textemdash}may be revisited to turn them into efficient task-based programs. In this paper, we study the applicability of task-based programming in the case of a Reverse Time Migration (RTM) application for Seismic Imaging. The initial MPI-based application is turned into a task-based code executed on top of the PaRSEC runtime system. Preliminary results show that the approach is competitive with (and even potentially superior to) the original MPI code on a homogeneous multicore node, and can more efficiently exploit complex hardware such as a cache coherent Non Uniform Memory Access (ccNUMA) node or an Intel Xeon Phi accelerator.},
	keywords = {plasma},
	author = {Lionel Boillot and George Bosilca and Emmanuel Agullo and Henri Calandra}
}
@conference {768,
	title = {Utilizing Dataflow-based Execution for Coupled Cluster Methods},
	booktitle = {2014 IEEE International Conference on Cluster Computing},
	number = {ICL-UT-14-02},
	year = {2014},
	month = {2014-09},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Madrid, Spain},
	abstract = {Computational chemistry comprises one of the driving forces of High Performance Computing. In particular, many-body methods, such as Coupled Cluster (CC) methods of the quantum chemistry package NWCHEM, are of particular interest for the applied chemistry community.

Harnessing large fractions of the processing power of modern large scale computing platforms has become increasingly difficult. With the increase in scale, complexity, and heterogeneity of modern platforms, traditional programming models fail to deliver the expected performance scalability. On our way to Exascale and with these extremely hybrid platforms, dataflow-based programming models may be the only viable way for achieving and maintaining computation at scale.

In this paper, we discuss a dataflow-based programming model and its applicability to NWCHEM{\textquoteright}s CC methods. Our dataflow version of the CC kernels breaks down the algorithm into fine-grained tasks with explicitly defined data dependencies. As a result, many of the traditional synchronization points can be eliminated, allowing for a dynamic reshaping of the execution based on the ongoing availability of computational resources. We build this experiment using PARSEC {\textendash} a task-based dataflow-driven execution engine {\textendash} that enables efficient task scheduling on distributed systems, providing a desirable portability layer for application developers.},
	author = {Heike McCraw and Anthony Danalis and George Bosilca and Jack Dongarra and Karol Kowalski and Theresa Windus}
}
@techreport {694,
	title = {Assessing the impact of ABFT and Checkpoint composite strategies},
	journal = {University of Tennessee Computer Science Technical Report},
	number = {ICL-UT-13-03},
	year = {2013},
	abstract = {Algorithm-specific fault tolerant approaches promise unparalleled scalability and performance in failure-prone environments. With the advances in the theoretical and practical understanding of algorithmic traits enabling such approaches, a growing number of frequently used algorithms (including all widely used factorization kernels) have been proven capable of such properties. These algorithms provide a temporal section of the execution when the data is protected by it{\textquoteright}s own intrinsic properties, and can be algorithmically recomputed without the need of checkpoints. However, while typical scientific applications spend a significant fraction of their execution time in library calls that can be ABFT-protected, they interleave sections that are difficult or even impossible to protect with ABFT. As a consequence, the only fault-tolerance approach that is currently used for these applications is checkpoint/restart. In this paper we propose a model and a simulator to investigate the behavior of a composite protocol, that alternates between ABFT and checkpoint/restart protection for effective protection of each phase of an iterative application composed of ABFT-aware and ABFT-unaware sections. We highlight this approach drastically increases the performance delivered by the system, especially at scale, by providing means to rarefy the checkpoints while simultaneously decreasing the volume of data needed to be checkpointed.},
	keywords = {ABFT, checkpoint, fault-tolerance, High-performance computing, resilience},
	author = {George Bosilca and Aurelien Bouteiller and Thomas Herault and Yves Robert and Jack Dongarra}
}
@article {750,
	title = {Correlated Set Coordination in Fault Tolerant Message Logging Protocols},
	journal = {Concurrency and Computation: Practice and Experience},
	volume = {25},
	year = {2013},
	month = {2013-03},
	pages = {572-585},
	abstract = {With our current expectation for the exascale systems, composed of hundred of thousands of many-core nodes, the mean time between failures will become small, even under the most optimistic assumptions. One of the most scalable checkpoint restart techniques, the message logging approach, is the most challenged when the number of cores per node increases because of the high overhead of saving the message payload. Fortunately, for two processes on the same node, the failure probability is correlated, meaning that coordinated recovery is free. In this paper, we propose an intermediate approach that uses coordination between correlated processes but retains the scalability advantage of message logging between independent ones. The algorithm still belongs to the family of event logging protocols but eliminates the need for costly payload logging between coordinated processes.},
	doi = {10.1002/cpe.2859},
	author = {Aurelien Bouteiller and Thomas Herault and George Bosilca and Jack Dongarra}
}
@inproceedings {716,
	title = {CPU-GPU Hybrid Bidiagonal Reduction With Soft Error Resilience},
	journal = {ScalA {\textquoteright}13 Proceedings of the Workshop on Latest Advances in Scalable Algorithms for Large-Scale Systems},
	year = {2013},
	month = {2013-11},
	address = {Montpellier, France},
	abstract = {Soft errors pose a real challenge to applications running on modern hardware as the feature size becomes smaller and the integration density increases for both the modern processors and the memory chips. Soft errors manifest themselves as bit-flips that alter the user value, and numerical software is a category of software that is sensitive to such data changes. In this paper, we present a design of a bidiagonal reduction algorithm that is resilient to soft errors, and we also describe its implementation on hybrid CPU-GPU architectures. Our fault-tolerant algorithm employs Algorithm Based Fault Tolerance, combined with reverse computation, to detect, locate, and correct soft errors. The tests were performed on a Sandy Bridge CPU coupled with an NVIDIA Kepler GPU. The included experiments show that our resilient bidiagonal reduction algorithm adds very little overhead compared to the error-prone code. At matrix size 10110 x 10110, our algorithm only has a performance overhead of 1.085\% when one error occurs, and 0.354\% when no errors occur.},
	author = {Yulu Jia and Piotr Luszczek and George Bosilca and Jack Dongarra}
}
@article {icl:698,
	title = {Dense Linear Algebra on Distributed Heterogeneous Hardware with a Symbolic DAG Approach},
	journal = {Scalable Computing and Communications: Theory and Practice},
	year = {2013},
	month = {2013-03},
	pages = {699-735},
	publisher = {John Wiley \& Sons},
	author = {George Bosilca and Aurelien Bouteiller and Anthony Danalis and Thomas Herault and Piotr Luszczek and Jack Dongarra},
	editor = {Samee Khan and Lin-Wang Wang and Albert Zomaya}
}
@conference {700,
	title = {Efficient Parallelization of Batch Pattern Training Algorithm on Many-core and Cluster Architectures},
	booktitle = {7th IEEE International Conference on Intelligent Data Acquisition and Advanced Computing Systems},
	year = {2013},
	month = {2013-09},
	address = {Berlin, Germany},
	abstract = {Abstract{\textemdash}The experimental research of the parallel batch pattern back propagation training algorithm on the example of recirculation neural network on many-core high performance computing systems is presented in this paper. The choice of recirculation neural network among the multilayer perceptron, recurrent and radial basis neural networks is proved. The model of a recirculation neural network and usual sequential batch pattern algorithm of its training are theoretically described. An algorithmic description of the parallel version of the batch pattern training method is presented. The experimental research is fulfilled using the Open MPI, Mvapich and Intel MPI message passing libraries. The results obtained on many-core AMD system and Intel MIC are compared with the results obtained on a cluster system. Our results show that the parallelization efficiency is about 95\% on 12 cores located inside one physical AMD processor for the considered minimum and maximum scenarios. The parallelization efficiency is about 70-75\% on 48 AMD cores for the minimum and maximum scenarios. These results are higher by 15-36\% (depending on the version of MPI library) in comparison with the results obtained on 48 cores of a cluster system. The parallelization efficiency obtained on Intel MIC architecture is surprisingly low, asking for deeper analysis.},
	keywords = {many-core system, parallel batch pattern training, parallelization efficiency, recirculation neural network},
	author = {Volodymyr Turchenko and George Bosilca and Aurelien Bouteiller and Jack Dongarra}
}
@article {869,
	title = {An evaluation of User-Level Failure Mitigation support in MPI},
	journal = {Computing},
	volume = {95},
	year = {2013},
	month = {2013-12},
	pages = {1171-1184},
	abstract = {As the scale of computing platforms becomes increasingly extreme, the requirements for application fault tolerance are increasing as well. Techniques to address this problem by improving the resilience of algorithms have been developed, but they currently receive no support from the programming model, and without such support, they are bound to fail. This paper discusses the failure-free overhead and recovery impact of the user-level failure mitigation proposal presented in the MPI Forum. Experiments demonstrate that fault-aware MPI has little or no impact on performance for a range of applications, and produces satisfactory recovery times when there are failures.},
	keywords = {Fault tolerance, MPI, User-level fault mitigation},
	doi = {10.1007/s00607-013-0331-3},
	author = {Wesley Bland and Aurelien Bouteiller and Thomas Herault and Joshua Hursey and George Bosilca and Jack Dongarra}
}
@article {691,
	title = {Extending the scope of the Checkpoint-on-Failure protocol for forward recovery in standard MPI},
	journal = {Concurrency and Computation: Practice and Experience},
	year = {2013},
	month = {2013-07},
	abstract = {Most predictions of exascale machines picture billion ways parallelism, encompassing not only millions of cores but also tens of thousands of nodes. Even considering extremely optimistic advances in hardware reliability, probabilistic amplification entails that failures will be unavoidable. Consequently, software fault tolerance is paramount to maintain future scientific productivity. Two major problems hinder ubiquitous adoption of fault tolerance techniques: (i) traditional checkpoint-based approaches incur a steep overhead on failure free operations and (ii) the dominant programming paradigm for parallel applications (the message passing interface (MPI) Standard) offers extremely limited support of software-level fault tolerance approaches. In this paper, we present an approach that relies exclusively on the features of a high quality implementation, as defined by the current MPI Standard, to enable advanced forward recovery techniques, without incurring the overhead of customary periodic checkpointing. With our approach, when failure strikes, applications regain control to make a checkpoint before quitting execution. This checkpoint is in reaction to the failure occurrence rather than periodic. This checkpoint is reloaded in a new MPI application, which restores a sane environment for the forward, application-based recovery technique to repair the failure-damaged dataset. The validity and performance of this approach are evaluated on large-scale systems, using the QR factorization as an example. Published 2013. This article is a US Government work and is in the public domain in the USA.},
	doi = {10.1002/cpe.3100},
	url = {http://doi.wiley.com/10.1002/cpe.3100},
	author = {Wesley Bland and Peng Du and Aurelien Bouteiller and Thomas Herault and George Bosilca and Jack Dongarra}
}
@article {icl:734,
	title = {Kernel-assisted and topology-aware MPI collective communications on multi-core/many-core platforms},
	journal = {Journal of Parallel and Distributed Computing},
	volume = {73},
	year = {2013},
	month = {2013-07},
	pages = {1000-1010},
	abstract = {Multicore Clusters, which have become the most prominent form of High Performance Computing (HPC) systems, challenge the performance of MPI applications with non-uniform memory accesses and shared cache hierarchies. Recent advances in MPI collective communications have alleviated the performance issue exposed by deep memory hierarchies by carefully considering the mapping between the collective topology and the hardware topologies, as well as the use of single-copy kernel assisted mechanisms. However, on distributed environments, a single level approach cannot encompass the extreme variations not only in bandwidth and latency capabilities, but also in the capability to support duplex communications or operate multiple concurrent copies. This calls for a collaborative approach between multiple layers of collective algorithms, dedicated to extracting the maximum degree of parallelism from the collective algorithm by consolidating the intra- and inter-node communications.

In this work, we present HierKNEM, a kernel-assisted topology-aware collective framework, and the mechanisms deployed by this framework to orchestrate the collaboration between multiple layers of collective algorithms. The resulting scheme maximizes the overlap of intra- and inter-node communications. We demonstrate experimentally, by considering three of the most used collective operations (Broadcast, Allgather and Reduction), that (1) this approach is immune to modifications of the underlying process-core binding; (2) it outperforms state-of-art MPI libraries (Open MPI, MPICH2 and MVAPICH2) demonstrating up to a 30x speedup for synthetic benchmarks, and up to a 3x acceleration for a parallel graph application (ASP); (3) it furthermore demonstrates a linear speedup with the increase of the number of cores per compute node, a paramount requirement for scalability on future many-core hardware.
},
	keywords = {Cluster, Collective communication, Hierarchical, HPC, MPI, Multicore},
	doi = {10.1016/j.jpdc.2013.01.015},
	url = {http://www.sciencedirect.com/science/article/pii/S0743731513000166},
	author = {Teng Ma and George Bosilca and Aurelien Bouteiller and Jack Dongarra}
}
@conference {705,
	title = {Parallel Reduction to Hessenberg Form with Algorithm-Based Fault Tolerance},
	booktitle = {International Conference for High Performance Computing, Networking, Storage and Analysis, IEEE-SC 2013},
	year = {2013},
	month = {2013-11},
	address = {Denver, CO},
	abstract = {This paper studies the resilience of a two-sided factorization and presents a generic algorithm-based approach capable of making two-sided factorizations resilient. We establish the theoretical proof of the correctness and the numerical stability of the approach in the context of a Hessenberg Reduction (HR) and present the scalability and performance results of a practical implementation. Our method is a hybrid algorithm combining an Algorithm Based Fault Tolerance (ABFT) technique with diskless checkpointing to fully protect the data. We protect the trailing and the initial part of the matrix with checksums, and protect finished panels in the panel scope with diskless checkpoints. Compared with the original HR (the ScaLAPACK PDGEHRD routine) our fault-tolerant algorithm introduces very little overhead, and maintains the same level of scalability. We prove that the overhead shows a decreasing trend as the size of the matrix or the size of the process grid increases.},
	author = {Yulu Jia and George Bosilca and Piotr Luszczek and Jack Dongarra}
}
@article {749,
	title = {PaRSEC: Exploiting Heterogeneity to Enhance Scalability},
	journal = {IEEE Computing in Science and Engineering},
	volume = {15},
	year = {2013},
	month = {2013-11},
	pages = {36-45},
	abstract = {New high-performance computing system designs with steeply escalating processor and core counts, burgeoning heterogeneity and accelerators, and increasingly unpredictable memory access times call for dramatically new programming paradigms. These new approaches must react and adapt quickly to unexpected contentions and delays, and they must provide the execution environment with sufficient intelligence and flexibility to rearrange the execution to improve resource utilization.},
	doi = {10.1109/MCSE.2013.98},
	author = {George Bosilca and Aurelien Bouteiller and Anthony Danalis and Mathieu Faverge and Thomas Herault and Jack Dongarra}
}
@article {693,
	title = {Post-failure recovery of MPI communication capability: Design and rationale},
	journal = {International Journal of High Performance Computing Applications},
	volume = {27},
	year = {2013},
	month = {2013-01},
	pages = {244 - 254},
	abstract = {As supercomputers are entering an era of massive parallelism where the frequency of faults is increasing, the MPI Standard remains distressingly vague on the consequence of failures on MPI communications. Advanced fault-tolerance techniques have the potential to prevent full-scale application restart and therefore lower the cost incurred for each failure, but they demand from MPI the capability to detect failures and resume communications afterward. In this paper, we present a set of extensions to MPI that allow communication capabilities to be restored, while maintaining the extreme level of performance to which MPI users have become accustomed. The motivation behind the design choices are weighted against alternatives, a task that requires simultaneously considering MPI from the viewpoint of both the user and the implementor. The usability of the interfaces for expressing advanced recovery techniques is then discussed, including the difficult issue of enabling separate software layers to coordinate their recovery. },
	issn = {1094-3420},
	doi = {10.1177/1094342013488238},
	url = {http://hpc.sagepub.com/cgi/doi/10.1177/1094342013488238},
	author = {Wesley Bland and Aurelien Bouteiller and Thomas Herault and George Bosilca and Jack Dongarra}
}
@inbook {695,
	title = {Scalable Dense Linear Algebra on Heterogeneous Hardware},
	booktitle = {HPC: Transition Towards Exascale Processing, in the series Advances in Parallel Computing},
	year = {2013},
	abstract = {Abstract. Design of systems exceeding 1 Pflop/s and the push toward 1 Eflop/s, forced a dramatic shift in hardware design. Various physical and engineering constraints resulted in introduction of massive parallelism and functional hybridization with the use of accelerator units. This paradigm change brings about a serious challenge for application developers, as the management of multicore proliferation and heterogeneity rests on software. And it is reasonable to expect, that this situation will not change in the foreseeable future. This chapter presents a methodology of dealing with this issue in three common scenarios. In the context of shared-memory multicore installations, we show how high performance and scalability go hand in hand, when the well-known linear algebra algorithms are recast in terms of Direct Acyclic Graphs (DAGs), which are then transparently scheduled at runtime inside the Parallel Linear Algebra Software for Multicore Architectures (PLASMA) project. Similarly, Matrix Algebra on GPU and Multicore Architectures (MAGMA) schedules DAG-driven computations on multicore processors and accelerators. Finally, Distributed PLASMA (DPLASMA), takes the approach to distributed-memory machines with the use of automatic dependence analysis and the Direct Acyclic Graph Engine (DAGuE) to deliver high performance at the scale of many thousands of cores.},
	author = {George Bosilca and Aurelien Bouteiller and Anthony Danalis and Thomas Herault and Jakub Kurzak and Piotr Luszczek and Stanimire Tomov and Jack Dongarra}
}
@article {748,
	title = {Unified Model for Assessing Checkpointing Protocols at Extreme-Scale},
	journal = {Concurrency and Computation: Practice and Experience},
	year = {2013},
	month = {2013-11},
	abstract = {In this paper, we present a unified model for several well-known checkpoint/restart protocols. The proposed model is generic enough to encompass both extremes of the checkpoint/restart space, from coordinated approaches to a variety of uncoordinated checkpoint strategies (with message logging). We identify a set of crucial parameters, instantiate them, and compare the expected efficiency of the fault tolerant protocols, for a given application/platform pair. We then propose a detailed analysis of several scenarios, including some of the most powerful currently available high performance computing platforms, as well as anticipated Exascale designs. The results of this analytical comparison are corroborated by a comprehensive set of simulations. Altogether, they outline comparative behaviors of checkpoint strategies at very large scale, thereby providing insight that is hardly accessible to direct experimentation.},
	doi = {10.1002/cpe.3173},
	author = {George Bosilca and Aurelien Bouteiller and Elisabeth Brunet and Franck Cappello and Jack Dongarra and Amina Guermouche and Thomas Herault and Yves Robert and Frederic Vivien and Dounia Zaidouni}
}
@inproceedings {icl:672,
	title = {Algorithm-Based Fault Tolerance for Dense Matrix Factorization},
	journal = {Proceedings of the 17th ACM SIGPLAN Symposium on Principles and Practice of Parallel Programming, PPOPP 2012},
	year = {2012},
	month = {2012-02},
	pages = {225-234},
	publisher = {ACM},
	address = {New Orleans, LA, USA},
	abstract = {Dense matrix factorizations, such as LU, Cholesky and QR, are widely used for scientific applications that require solving systems of linear equations, eigenvalues and linear least squares problems. Such computations are normally carried out on supercomputers, whose ever-growing scale induces a fast decline of the Mean Time To Failure (MTTF). This paper proposes a new hybrid approach, based on Algorithm-Based Fault Tolerance (ABFT), to help matrix factorizations algorithms survive fail-stop failures. We consider extreme conditions, such as the absence of any reliable component and the possibility of loosing both data and checksum from a single failure. We will present a generic solution for protecting the right factor, where the updates are applied, of all above mentioned factorizations. For the left factor, where the panel has been applied, we propose a scalable checkpointing algorithm. This algorithm features high degree of checkpointing parallelism and cooperatively utilizes the checksum storage leftover from the right factor protection. The fault-tolerant algorithms derived from this hybrid solution is applicable to a wide range of dense matrix factorizations, with minor modifications. Theoretical analysis shows that the fault tolerance overhead sharply decreases with the scaling in the number of computing units and the problem size. Experimental results of LU and QR factorization on the Kraken (Cray XT5) supercomputer validate the theoretical evaluation and confirm negligible overhead, with- and without-errors.},
	keywords = {ft-la, ftmpi},
	doi = {10.1145/2145816.2145845},
	author = {Peng Du and Aurelien Bouteiller and George Bosilca and Thomas Herault and Jack Dongarra},
	editor = {J. Ramanujam and P. Sadayappan}
}
@inproceedings {icl:679,
	title = {A Checkpoint-on-Failure Protocol for Algorithm-Based Recovery in Standard MPI},
	journal = {18th International European Conference on Parallel and Distributed Computing (Euro-Par 2012) (Best Paper Award)},
	year = {2012},
	month = {2012-08},
	publisher = {Springer-Verlag},
	address = {Rhodes, Greece},
	author = {Wesley Bland and Peng Du and Aurelien Bouteiller and Thomas Herault and George Bosilca and Jack Dongarra},
	editor = {Christos Kaklamanis and Theodore Papatheodorou and Paul Spirakis}
}
@article {icl:670,
	title = {DAGuE: A generic distributed DAG Engine for High Performance Computing.},
	journal = {Parallel Computing},
	volume = {38},
	number = {1-2},
	year = {2012},
	month = {2012-00},
	pages = {27-51},
	publisher = {Elsevier},
	keywords = {dague, parsec},
	author = {George Bosilca and Aurelien Bouteiller and Anthony Danalis and Thomas Herault and Pierre Lemariner and Jack Dongarra}
}
@techreport {icl:683,
	title = {An efficient distributed randomized solver with application to large dense linear systems},
	journal = {ICL Technical Report},
	number = {ICL-UT-12-02},
	year = {2012},
	month = {2012-07},
	keywords = {dague, dplasma, parsec, plasma},
	author = {Marc Baboulin and Dulceneia Becker and George Bosilca and Anthony Danalis and Jack Dongarra}
}
@inproceedings {icl:680,
	title = {An Evaluation of User-Level Failure Mitigation Support in MPI},
	journal = {Proceedings of Recent Advances in Message Passing Interface - 19th European MPI Users{\textquoteright} Group Meeting, EuroMPI 2012},
	year = {2012},
	month = {2012-09},
	publisher = {Springer},
	address = {Vienna, Austria},
	author = {Wesley Bland and Aurelien Bouteiller and Thomas Herault and Joshua Hursey and George Bosilca and Jack Dongarra}
}
@techreport {icl:724,
	title = {Extending the Scope of the Checkpoint-on-Failure Protocol for Forward Recovery in Standard MPI},
	journal = {University of Tennessee Computer Science Technical Report},
	number = {ut-cs-12-702},
	year = {2012},
	month = {2012-00},
	keywords = {ftmpi},
	author = {Wesley Bland and Peng Du and Aurelien Bouteiller and Thomas Herault and George Bosilca and Jack Dongarra}
}
@conference {icl:699,
	title = {From Serial Loops to Parallel Execution on Distributed Systems},
	booktitle = {International European Conference on Parallel and Distributed Computing (Euro-Par {\textquoteright}12)},
	year = {2012},
	month = {2012-08},
	address = {Rhodes, Greece},
	author = {George Bosilca and Aurelien Bouteiller and Anthony Danalis and Thomas Herault and Jack Dongarra}
}
@article {icl:700,
	title = {HierKNEM: An Adaptive Framework for Kernel-Assisted and Topology-Aware Collective Communications on Many-core Clusters},
	journal = {IPDPS 2012 (Best Paper)},
	year = {2012},
	month = {2012-05},
	address = {Shanghai, China},
	author = {Teng Ma and George Bosilca and Aurelien Bouteiller and Jack Dongarra}
}
@article {icl:730,
	title = {Matrices Over Runtime Systems at Exascale},
	journal = {Supercomputing {\textquoteright}12 (poster)},
	year = {2012},
	month = {2012-11},
	address = {Salt Lake City, Utah},
	author = {Emmanuel Agullo and George Bosilca and Cedric Castagn{\`e}de and Jack Dongarra and Hatem Ltaeif and Stanimire Tomov}
}
@inproceedings {icl:710,
	title = {Power Profiling of Cholesky and QR Factorizations on Distributed Memory Systems},
	journal = {Third International Conference on Energy-Aware High Performance Computing},
	year = {2012},
	month = {2012-09},
	address = {Hamburg, Germany},
	author = {George Bosilca and Jack Dongarra and Hatem Ltaeif}
}
@techreport {icl:667,
	title = {A Proposal for User-Level Failure Mitigation in the MPI-3 Standard},
	journal = {University of Tennessee Electrical Engineering and Computer Science Technical Report},
	number = {ut-cs-12-693},
	year = {2012},
	month = {2012-02},
	publisher = {University of Tennessee},
	keywords = {ftmpi},
	author = {Wesley Bland and George Bosilca and Aurelien Bouteiller and Thomas Herault and Jack Dongarra}
}
@techreport {icl:716,
	title = {Unified Model for Assessing Checkpointing Protocols at Extreme-Scale},
	journal = {University of Tennessee Computer Science Technical Report (also LAWN 269)},
	number = {UT-CS-12-697},
	year = {2012},
	month = {2012-06},
	author = {George Bosilca and Aurelien Bouteiller and Elisabeth Brunet and Franck Cappello and Jack Dongarra and Amina Guermouche and Thomas Herault and Yves Robert and Frederic Vivien and Dounia Zaidouni}
}
@techreport {icl:626,
	title = {Algorithm-based Fault Tolerance for Dense Matrix Factorizations},
	journal = {University of Tennessee Computer Science Technical Report},
	number = {UT-CS-11-676},
	year = {2011},
	month = {2011-08},
	address = {Knoxville, TN},
	keywords = {ft-la},
	author = {Peng Du and Aurelien Bouteiller and George Bosilca and Thomas Herault and Jack Dongarra}
}
@inproceedings {icl:673,
	title = {Correlated Set Coordination in Fault Tolerant Message Logging Protocols},
	journal = {Proceedings of 17th International Conference, Euro-Par 2011, Part II},
	volume = {6853},
	year = {2011},
	month = {2011-08},
	pages = {51-64},
	publisher = {Springer},
	address = {Bordeaux, France},
	keywords = {ftmpi},
	author = {Aurelien Bouteiller and Thomas Herault and George Bosilca and Jack Dongarra},
	editor = {Emmanuel Jeannot and Raymond Namyst and Jean Roman}
}
@inproceedings {icl:675,
	title = {DAGuE: A Generic Distributed DAG Engine for High Performance Computing},
	journal = {Proceedings of the Workshops of the 25th IEEE International Symposium on Parallel and Distributed Processing (IPDPS 2011 Workshops)},
	year = {2011},
	month = {2011-00},
	pages = {1151-1158},
	publisher = {IEEE},
	address = {Anchorage, Alaska, USA},
	keywords = {dague, parsec},
	author = {George Bosilca and Aurelien Bouteiller and Anthony Danalis and Thomas Herault and Pierre Lemariner and Jack Dongarra}
}
@inproceedings {icl:676,
	title = {Flexible Development of Dense Linear Algebra Algorithms on Massively Parallel Architectures with DPLASMA},
	journal = {Proceedings of the Workshops of the 25th IEEE International Symposium on Parallel and Distributed Processing (IPDPS 2011 Workshops)},
	year = {2011},
	month = {2011-05},
	pages = {1432-1441},
	publisher = {IEEE},
	address = {Anchorage, Alaska, USA},
	keywords = {dague, dplasma, parsec},
	author = {George Bosilca and Aurelien Bouteiller and Anthony Danalis and Mathieu Faverge and Azzam Haidar and Thomas Herault and Jakub Kurzak and Julien Langou and Pierre Lemariner and Hatem Ltaeif and Piotr Luszczek and Asim YarKhan and Jack Dongarra}
}
@article {icl:646,
	title = {Impact of Kernel-Assisted MPI Communication over Scientific Applications: CPMD and FFTW},
	journal = {18th EuroMPI},
	year = {2011},
	month = {2011-09},
	pages = {247-254},
	publisher = {Springer},
	address = {Santorini, Greece},
	keywords = {dague},
	author = {Teng Ma and Aurelien Bouteiller and George Bosilca and Jack Dongarra},
	editor = {Yiannis Cotronis and Anthony Danalis and Dimitrios S. Nikolopoulos and Jack Dongarra}
}
@inproceedings {icl:649,
	title = {Kernel Assisted Collective Intra-node MPI Communication Among Multi-core and Many-core CPUs},
	journal = {Int{\textquoteright}l Conference on Parallel Processing (ICPP {\textquoteright}11)},
	year = {2011},
	month = {2011-09},
	address = {Taipei, Taiwan},
	author = {Teng Ma and George Bosilca and Aurelien Bouteiller and Brice Goglin and J. Squyres and Jack Dongarra}
}
@article {icl:647,
	title = {OMPIO: A Modular Software Architecture for MPI I/O},
	journal = {18th EuroMPI},
	year = {2011},
	month = {2011-09},
	pages = {81-89},
	publisher = {Springer},
	address = {Santorini, Greece},
	author = {Mohamad Chaarawi and Edgar Gabriel and Rainer Keller and Richard L. Graham and George Bosilca and Jack Dongarra},
	editor = {Yiannis Cotronis and Anthony Danalis and Dimitrios S. Nikolopoulos and Jack Dongarra}
}
@article {icl:636,
	title = {Performance Portability of a GPU Enabled Factorization with the DAGuE Framework},
	journal = {IEEE Cluster: workshop on Parallel Programming on Accelerator Clusters (PPAC)},
	year = {2011},
	month = {2011-06},
	keywords = {dague, magma, parsec},
	author = {George Bosilca and Aurelien Bouteiller and Thomas Herault and Pierre Lemariner and Narapat Ohm Saengpatsa and Stanimire Tomov and Jack Dongarra}
}
@inproceedings {icl:648,
	title = {Process Distance-aware Adaptive MPI Collective Communications},
	journal = {IEEE Int{\textquoteright}l Conference on Cluster Computing (Cluster 2011)},
	year = {2011},
	month = {2011-00},
	address = {Austin, Texas},
	author = {Teng Ma and Thomas Herault and George Bosilca and Jack Dongarra}
}
@techreport {icl:612,
	title = {On Scalability for MPI Runtime Systems},
	journal = {University of Tennessee Computer Science Technical Report},
	number = {ICL-UT-11-05},
	year = {2011},
	month = {2011-05},
	address = {Knoxville, TN},
	author = {George Bosilca and Thomas Herault and A. Rezmerita and Jack Dongarra}
}
@inproceedings {icl:671,
	title = {On Scalability for MPI Runtime Systems},
	journal = {International Conference on Cluster Computing (CLUSTER)},
	year = {2011},
	month = {2011-09},
	pages = {187-195},
	publisher = {IEEEE},
	address = {Austin, TX, USA},
	keywords = {harness},
	author = {George Bosilca and Thomas Herault and A. Rezmerita and Jack Dongarra}
}
@inproceedings {icl:674,
	title = {Scalable Runtime for MPI: Efficiently Building the Communication Infrastructure},
	journal = {Proceedings of Recent Advances in the Message Passing Interface - 18th European MPI Users{\textquoteright} Group Meeting, EuroMPI 2011},
	volume = {6960},
	year = {2011},
	month = {2011-09},
	pages = {342-344},
	publisher = {Springer},
	address = {Santorini, Greece},
	keywords = {ftmpi},
	author = {George Bosilca and Thomas Herault and Pierre Lemariner and Jack Dongarra and A. Rezmerita},
	editor = {Yiannis Cotronis and Anthony Danalis and Dimitrios S. Nikolopoulos and Jack Dongarra}
}
@inproceedings {icl:593,
	title = {A Unified HPC Environment for Hybrid Manycore/GPU Distributed Systems},
	journal = {IEEE International Parallel and Distributed Processing Symposium (submitted)},
	year = {2011},
	month = {2011-05},
	address = {Anchorage, AK},
	keywords = {dague},
	author = {George Bosilca and Aurelien Bouteiller and Thomas Herault and Pierre Lemariner and Narapat Ohm Saengpatsa and Stanimire Tomov and Jack Dongarra}
}
@article {1363,
	title = {Accelerating Linear Algebra on Heterogeneous Architectures of Multicore and GPUs using MAGMA and DPLASMA and StarPU Schedulers},
	year = {2010},
	month = {2010-07},
	publisher = {2010 Symposium on Application Accelerators in. High-Performance Computing (SAAHPC{\textquoteright}10), Tutorial},
	author = {Stanimire Tomov and George Bosilca and Cedric Augonnet}
}
@article {icl:555,
	title = {Constructing Resiliant Communication Infrastructure for Runtime Environments in Advances in Parallel Computing},
	journal = {Advances in Parallel Computing - Parallel Computing: From Multicores and GPU{\textquoteright}s to Petascale},
	volume = {19},
	year = {2010},
	pages = {441-451},
	doi = {10.3233/978-1-60750-530-3-441},
	author = {George Bosilca and Camille Coti and Thomas Herault and Pierre Lemariner and Jack Dongarra},
	editor = {Barbara Chapman and Frederic Desprez and Gerhard R. Joubert and Alain Lichnewsky and Frans Peters and T. Priol}
}
@techreport {icl:528,
	title = {DAGuE: A generic distributed DAG engine for high performance computing},
	journal = {Innovative Computing Laboratory Technical Report},
	number = {ICL-UT-10-01},
	year = {2010},
	month = {2010-04},
	keywords = {dague},
	author = {George Bosilca and Aurelien Bouteiller and Anthony Danalis and Thomas Herault and Pierre Lemariner and Jack Dongarra}
}
@techreport {icl:563,
	title = {Distributed Dense Numerical Linear Algebra Algorithms on Massively Parallel Architectures: DPLASMA},
	journal = {University of Tennessee Computer Science Technical Report, UT-CS-10-660},
	year = {2010},
	month = {2010-09},
	keywords = {dague, dplasma, parsec, plasma},
	author = {George Bosilca and Aurelien Bouteiller and Anthony Danalis and Mathieu Faverge and Azzam Haidar and Thomas Herault and Jakub Kurzak and Julien Langou and Pierre Lemariner and Hatem Ltaeif and Piotr Luszczek and Asim YarKhan and Jack Dongarra}
}
@techreport {icl:529,
	title = {Distributed-Memory Task Execution and Dependence Tracking within DAGuE and the DPLASMA Project},
	journal = {Innovative Computing Laboratory Technical Report},
	number = {ICL-UT-10-02},
	year = {2010},
	month = {2010-00},
	keywords = {dague, plasma},
	author = {George Bosilca and Aurelien Bouteiller and Anthony Danalis and Mathieu Faverge and Azzam Haidar and Thomas Herault and Jakub Kurzak and Julien Langou and Pierre Lemariner and Hatem Ltaeif and Piotr Luszczek and Asim YarKhan and Jack Dongarra}
}
@inproceedings {icl:534,
	title = {Dodging the Cost of Unavoidable Memory Copies in Message Logging Protocols},
	journal = {Proceedings of EuroMPI 2010},
	year = {2010},
	month = {2010-09},
	publisher = {Springer},
	address = {Stuttgart, Germany},
	keywords = {ftmpi},
	author = {George Bosilca and Aurelien Bouteiller and Thomas Herault and Pierre Lemariner and Jack Dongarra},
	editor = {Jack Dongarra and Michael Resch and Rainer Keller and Edgar Gabriel}
}
@inproceedings {icl:527,
	title = {Improvement of parallelization efficiency of batch pattern BP training algorithm using Open MPI},
	journal = {Proceedings of International Conference on Computational Science, ICCS 2010 (to appear)},
	year = {2010},
	month = {2010-06},
	publisher = {Elsevier},
	address = {Amsterdam The Netherlands},
	keywords = {hpcchallenge},
	author = {Volodymyr Turchenko and Lucio Grandinetti and George Bosilca and Jack Dongarra}
}
@techreport {icl:597,
	title = {Kernel Assisted Collective Intra-node Communication Among Multicore and Manycore CPUs},
	journal = {University of Tennessee Computer Science Technical Report, UT-CS-10-663},
	year = {2010},
	month = {2010-11},
	author = {Teng Ma and George Bosilca and Aurelien Bouteiller and Brice Goglin and J. Squyres and Jack Dongarra}
}
@inproceedings {icl:535,
	title = {Locality and Topology aware Intra-node Communication Among Multicore CPUs},
	journal = {Proceedings of the 17th EuroMPI conference},
	year = {2010},
	month = {2010-09},
	publisher = {LNCS},
	address = {Stuttgart, Germany},
	author = {Teng Ma and Aurelien Bouteiller and George Bosilca and Jack Dongarra}
}
@article {icl:565,
	title = {Redesigning the Message Logging Model for High Performance},
	journal = {Concurrency and Computation: Practice and Experience (online version)},
	year = {2010},
	month = {2010-06},
	author = {Aurelien Bouteiller and George Bosilca and Jack Dongarra}
}
@article {icl:567,
	title = {Self-Healing Network for Scalable Fault-Tolerant Runtime Environments},
	journal = {Future Generation Computer Systems},
	volume = {26},
	number = {3},
	year = {2010},
	month = {2010-03},
	pages = {479-485},
	author = {Thara Angskun and Graham Fagg and George Bosilca and Jelena Pjesivac{\textendash}Grbovic and Jack Dongarra}
}
@article {icl:490,
	title = {Algorithmic Based Fault Tolerance Applied to High Performance Computing},
	journal = {Journal of Parallel and Distributed Computing},
	volume = {69},
	year = {2009},
	month = {2009-00},
	pages = {410-416},
	author = {Jack Dongarra and George Bosilca and Remi Delmas and Julien Langou}
}
@techreport {icl:484,
	title = {Constructing resiliant communication infrastructure for runtime environments},
	journal = {Innovative Computing Laboratory Technical Report},
	number = {ICL-UT-09-02},
	year = {2009},
	month = {2009-07},
	author = {George Bosilca and Camille Coti and Thomas Herault and Pierre Lemariner and Jack Dongarra}
}
@article {icl:517,
	title = {Constructing Resilient Communication Infrastructure for Runtime Environments},
	journal = {ParCo 2009},
	year = {2009},
	month = {2009-09},
	address = {Lyon France},
	author = {Pierre Lemariner and George Bosilca and Camille Coti and Thomas Herault and Jack Dongarra}
}
@conference {870,
	title = {Reasons for a Pessimistic or Optimistic Message Logging Protocol in MPI Uncoordinated Failure Recovery},
	booktitle = {CLUSTER {\textquoteright}09},
	year = {2009},
	month = {2009-08},
	publisher = {IEEE},
	organization = {IEEE},
	address = {New Orleans},
	abstract = {With the growing scale of high performance computing platforms, fault tolerance has become a major issue. Among the various approaches for providing fault tolerance to MPI applications, message logging has been proved to tolerate higher failure rate. However, this advantage comes at the expense of a higher overhead on communications, due to latency intrusive logging of events to a stable storage. Previous work proposed and evaluated several protocols relaxing the synchronicity of event logging to moderate this overhead. Recently, the model of message logging has been refined to better match the reality of high performance network cards, where message receptions are decomposed in multiple interdependent events. According to this new model, deterministic and non-deterministic events are clearly discriminated, reducing the overhead induced by message logging. In this paper we compare, experimentally, a pessimistic and an optimistic message logging protocol, using this new model and implemented in the Open MPI library. Although pessimistic and optimistic message logging are, respectively, the most and less synchronous message logging paradigms, experiments show that most of the time their performance is comparable.},
	keywords = {fault tolerant computing, libraries message passing, parallel machines, protocols},
	doi = {10.1109/CLUSTR.2009.5289157},
	author = {Aurelien Bouteiller and Thomas Ropars and George Bosilca and Christine Morin and Jack Dongarra}
}
@techreport {icl:426,
	title = {Algorithmic Based Fault Tolerance Applied to High Performance Computing},
	journal = {University of Tennessee Computer Science Technical Report, UT-CS-08-620 (also LAPACK Working Note 205)},
	year = {2008},
	month = {2008-01},
	author = {George Bosilca and Remi Delmas and Jack Dongarra and Julien Langou}
}
@inproceedings {icl:456,
	title = {Redesigning the Message Logging Model for High Performance},
	journal = {International Supercomputer Conference (ISC 2008)},
	year = {2008},
	month = {2008-01},
	address = {Dresden, Germany},
	author = {Aurelien Bouteiller and George Bosilca and Jack Dongarra}
}
@inproceedings {icl:355,
	title = {Binomial Graph: A Scalable and Fault- Tolerant Logical Network Topology},
	journal = {Proceedings of The Fifth International Symposium on Parallel and Distributed Processing and Applications (ISPA07)},
	year = {2007},
	month = {2007-08},
	publisher = {Springer},
	address = {Niagara Falls, Canada},
	keywords = {ftmpi},
	author = {Thara Angskun and George Bosilca and Jack Dongarra}
}
@article {icl:360,
	title = {A Comparison of Application Performance Using Open MPI and Cray MPI},
	journal = {Cray User Group, CUG 2007},
	year = {2007},
	month = {2007-05},
	author = {Richard L. Graham and George Bosilca and Jelena Pjesivac{\textendash}Grbovic}
}
@article {icl:357,
	title = {Decision Trees and MPI Collective Algorithm Selection Problem},
	journal = {Euro-Par 2007},
	year = {2007},
	month = {2007-08},
	pages = {105{\textendash}115},
	publisher = {Springer},
	address = {Rennes, France},
	keywords = {ftmpi},
	author = {Jelena Pjesivac{\textendash}Grbovic and George Bosilca and Graham Fagg and Thara Angskun and Jack Dongarra}
}
@article {icl:366,
	title = {Disaster Survival Guide in Petascale Computing: An Algorithmic Approach},
	journal = {in Petascale Computing: Algorithms and Applications (to appear)},
	year = {2007},
	month = {2007-00},
	publisher = {Chapman \& Hall - CRC Press},
	author = {Jack Dongarra and Zizhong Chen and George Bosilca and Julien Langou}
}
@article {icl:359,
	title = {An Evaluation of Open MPI{\textquoteright}s Matching Transport Layer on the Cray XT},
	journal = {EuroPVM/MPI 2007},
	year = {2007},
	month = {2007-09},
	author = {Richard L. Graham and Ron Brightwell and Brian Barrett and George Bosilca and Jelena Pjesivac{\textendash}Grbovic}
}
@article {icl:356,
	title = {MPI Collective Algorithm Selection and Quadtree Encoding},
	journal = {Parallel Computing (Special Edition: EuroPVM/MPI 2006)},
	year = {2007},
	month = {2007-00},
	publisher = {Elsevier},
	keywords = {ftmpi},
	author = {Jelena Pjesivac{\textendash}Grbovic and George Bosilca and Graham Fagg and Thara Angskun and Jack Dongarra}
}
@inbook {875,
	title = {A New Approach to MPI Collective Communication Implementations},
	booktitle = {Distributed and Parallel Systems},
	year = {2007},
	pages = {45-54},
	publisher = {Springer US},
	organization = {Springer US},
	abstract = {Recent research into the optimization of collective MPI operations has resulted in a wide variety of algorithms and corresponding implementations, each typically only applicable in a relatively narrow scope: on a specific architecture, on a specific network, with a specific number of processes, with a specific data size and/or data-type {\textendash} or any combination of these (or other) factors. This situation presents an enormous challenge to portable MPI implementations which are expected to provide optimized collective operation performance on all platforms. Many portable implementations have attempted to provide a token number of algorithms that are intended to realize good performance on most systems. However, many platform configurations are still left without well-tuned collective operations. This paper presents a proposal for a framework that will allow a wide variety of collective algorithm implementations and a flexible, multi-tiered selection process for choosing which implementation to use when an application invokes an MPI collective function.},
	keywords = {Automatic Selection, Collective Operation, Framework, Message Passing (MPI), Open MPI},
	isbn = {978-0-387-69857-1},
	doi = {10.1007/978-0-387-69858-8_5},
	author = {Torsten Hoefler and Jeffrey M. Squyres and Graham Fagg and George Bosilca and Wolfgang Rehm and Andrew Lumsdaine}
}
@inproceedings {icl:374,
	title = {Optimal Routing in Binomial Graph Networks},
	journal = {The International Conference on Parallel and Distributed Computing, applications and Technologies (PDCAT)},
	year = {2007},
	month = {2007-12},
	publisher = {IEEE Computer Society},
	address = {Adelaide, Australia},
	keywords = {ftmpi},
	author = {Thara Angskun and George Bosilca and Brad Vander Zanden and Jack Dongarra}
}
@article {icl:358,
	title = {Performance Analysis of MPI Collective Operations},
	journal = {Cluster computing},
	volume = {10},
	number = {2},
	year = {2007},
	month = {2007-06},
	pages = {127-143},
	publisher = {Springer Netherlands},
	keywords = {ftmpi},
	author = {Jelena Pjesivac{\textendash}Grbovic and Thara Angskun and George Bosilca and Graham Fagg and Edgar Gabriel and Jack Dongarra}
}
@article {icl:397,
	title = {Recovery Patterns for Iterative Methods in a Parallel Unstable Environment},
	journal = {SIAM SISC (to appear)},
	year = {2007},
	month = {2007-05},
	author = {Julien Langou and Zizhong Chen and George Bosilca and Jack Dongarra}
}
@inproceedings {icl:354,
	title = {Reliability Analysis of Self-Healing Network using Discrete-Event Simulation},
	journal = {Proceedings of Seventh IEEE International Symposium on Cluster Computing and the Grid (CCGrid {\textquoteright}07)},
	year = {2007},
	month = {2007-05},
	pages = {437-444},
	publisher = {IEEE Computer Society},
	keywords = {ftmpi},
	author = {Thara Angskun and George Bosilca and Graham Fagg and Jelena Pjesivac{\textendash}Grbovic and Jack Dongarra}
}
@article {icl:353,
	title = {Retrospect: Deterministic Relay of MPI Applications for Interactive Distributed Debugging},
	journal = {Accepted for Euro PVM/MPI 2007},
	year = {2007},
	month = {2007-09},
	publisher = {Springer},
	keywords = {ftmpi},
	author = {Aurelien Bouteiller and George Bosilca and Jack Dongarra}
}
@techreport {icl:364,
	title = {SCOP3: A Rough Guide to Scientific Computing On the PlayStation 3},
	journal = {University of Tennessee Computer Science Dept. Technical Report, UT-CS-07-595},
	year = {2007},
	month = {2007-00},
	keywords = {multi-core},
	author = {Alfredo Buttari and Piotr Luszczek and Jakub Kurzak and Jack Dongarra and George Bosilca}
}
@inproceedings {icl:380,
	title = {Self-Healing in Binomial Graph Networks},
	journal = {2nd International Workshop On Reliability in Decentralized Distributed Systems (RDDS 2007)},
	year = {2007},
	month = {2007-11},
	address = {Vilamoura, Algarve, Portugal},
	author = {Thara Angskun and George Bosilca and Jack Dongarra}
}
@article {icl:315,
	title = {Flexible collective communication tuning architecture applied to Open MPI},
	journal = {2006 Euro PVM/MPI (submitted)},
	year = {2006},
	month = {2006-01},
	address = {Bonn, Germany},
	keywords = {ftmpi},
	author = {Graham Fagg and Jelena Pjesivac{\textendash}Grbovic and George Bosilca and Thara Angskun and Jack Dongarra}
}
@article {icl:651,
	title = {High Performance RDMA Protocols in HPC},
	journal = {Euro PVM/MPI 2006},
	year = {2006},
	month = {2006-09},
	address = {Bonn, Germany},
	author = {Galen M. Shipman and George Bosilca and Maccabe, Arthur B.}
}
@article {icl:652,
	title = {A High-Performance, Heterogeneous MPI},
	journal = {HeteroPar 2006},
	year = {2006},
	month = {2006-09},
	address = {Barcelona, Spain},
	author = {Richard L. Graham and Galen M. Shipman and Brian Barrett and Ralph Castain and George Bosilca and Andrew Lumsdaine}
}
@article {icl:650,
	title = {Implementation and Usage of the PERUSE-Interface in Open MPI},
	journal = {Euro PVM/MPI 2006},
	year = {2006},
	month = {2006-09},
	address = {Bonn, Germany},
	author = {Rainer Keller and George Bosilca and Graham Fagg and Michael Resch and Jack Dongarra}
}
@techreport {icl:314,
	title = {MPI Collective Algorithm Selection and Quadtree Encoding},
	journal = {ICL Technical Report},
	number = {ICL-UT-06-11},
	year = {2006},
	month = {2006-00},
	keywords = {ftmpi},
	author = {Jelena Pjesivac{\textendash}Grbovic and Graham Fagg and Thara Angskun and George Bosilca and Jack Dongarra}
}
@article {icl:323,
	title = {MPI Collective Algorithm Selection and Quadtree Encoding},
	journal = {Lecture Notes in Computer Science},
	volume = {4192},
	number = {ICL-UT-06-13},
	year = {2006},
	month = {2006-09},
	pages = {40-48},
	publisher = {Springer Berlin / Heidelberg},
	keywords = {ftmpi},
	author = {Jelena Pjesivac{\textendash}Grbovic and Graham Fagg and Thara Angskun and George Bosilca and Jack Dongarra}
}
@article {icl:316,
	title = {Scalable Fault Tolerant Protocol for Parallel Runtime Environments},
	journal = {2006 Euro PVM/MPI},
	number = {ICL-UT-06-12},
	year = {2006},
	month = {2006-00},
	address = {Bonn, Germany},
	keywords = {ftmpi},
	author = {Thara Angskun and Graham Fagg and George Bosilca and Jelena Pjesivac{\textendash}Grbovic and Jack Dongarra}
}
@article {icl:332,
	title = {Self Adapting Numerical Software SANS Effort},
	journal = {IBM Journal of Research and Development},
	volume = {50},
	number = {2/3},
	year = {2006},
	month = {2006-01},
	pages = {223-238},
	keywords = {gco},
	author = {George Bosilca and Zizhong Chen and Jack Dongarra and Victor Eijkhout and Graham Fagg and Erika Fuentes and Julien Langou and Piotr Luszczek and Jelena Pjesivac{\textendash}Grbovic and Keith Seymour and Haihang You and Sathish Vadhiyar}
}
@inproceedings {icl:330,
	title = {Self-Healing Network for Scalable Fault Tolerant Runtime Environments},
	journal = {DAPSYS 2006, 6th Austrian-Hungarian Workshop on Distributed and Parallel Systems},
	year = {2006},
	month = {2006-01},
	address = {Innsbruck, Austria},
	author = {Thara Angskun and Graham Fagg and George Bosilca and Jelena Pjesivac{\textendash}Grbovic and Jack Dongarra}
}
@inproceedings {icl:265,
	title = {Fault Tolerant High Performance Computing by a Coding Approach},
	journal = {Proceedings of ACM SIGPLAN Symposium on Principles and Practice of Parallel Programming (to appear)},
	year = {2005},
	month = {2005-01},
	address = {Chicago, Illinois},
	keywords = {ftmpi, grads, lacsi, sans},
	author = {Zizhong Chen and Graham Fagg and Edgar Gabriel and Julien Langou and Thara Angskun and George Bosilca and Jack Dongarra}
}
@inproceedings {icl:280,
	title = {Hash Functions for Datatype Signatures in MPI},
	journal = {Proceedings of 12th European Parallel Virtual Machine and Message Passing Interface Conference - Euro PVM/MPI},
	volume = {3666},
	year = {2005},
	month = {2005-09},
	pages = {76-83},
	publisher = {Springer-Verlag Berlin},
	address = {Sorrento (Naples), Italy},
	keywords = {ftmpi},
	author = {George Bosilca and Jack Dongarra and Graham Fagg and Julien Langou},
	editor = {Beniamino Di Martino}
}
@inproceedings {icl:249,
	title = {Performance Analysis of MPI Collective Operations},
	journal = {4th International Workshop on Performance Modeling, Evaluation, and Optmization of Parallel and Distributed Systems (PMEO-PDS {\textquoteright}05)},
	year = {2005},
	month = {2005-04},
	address = {Denver, Colorado},
	keywords = {ftmpi},
	author = {Jelena Pjesivac{\textendash}Grbovic and Thara Angskun and George Bosilca and Graham Fagg and Edgar Gabriel and Jack Dongarra}
}
@article {icl:306,
	title = {Performance Analysis of MPI Collective Operations},
	journal = {Cluster Computing Journal (to appear)},
	year = {2005},
	month = {2005-01},
	keywords = {ftmpi},
	author = {Jelena Pjesivac{\textendash}Grbovic and Thara Angskun and George Bosilca and Graham Fagg and Edgar Gabriel and Jack Dongarra}
}
@techreport {icl:301,
	title = {Recovery Patterns for Iterative Methods in a Parallel Unstable Environment},
	journal = {University of Tennessee Computer Science Department Technical Report, UT-CS-04-538},
	year = {2005},
	month = {2005-00},
	keywords = {ft-la},
	author = {George Bosilca and Zizhong Chen and Jack Dongarra and Julien Langou}
}
@inproceedings {icl:279,
	title = {Scalable Fault Tolerant MPI: Extending the Recovery Algorithm},
	journal = {Proceedings of 12th European Parallel Virtual Machine and Message Passing Interface Conference - Euro PVM/MPI},
	volume = {3666},
	year = {2005},
	month = {2005-09},
	pages = {67},
	publisher = {Springer-Verlag Berlin},
	address = {Sorrento (Naples) , Italy},
	keywords = {ftmpi},
	author = {Graham Fagg and Thara Angskun and George Bosilca and Jelena Pjesivac{\textendash}Grbovic and Jack Dongarra},
	editor = {Beniamino Di Martino}
}
@inproceedings {icl:230,
	title = {Extending the MPI Specification for Process Fault Tolerance on High Performance Computing Systems},
	journal = {Proceedings of ISC2004 (to appear)},
	year = {2004},
	month = {2004-06},
	address = {Heidelberg, Germany},
	keywords = {ftmpi, lacsi},
	author = {Graham Fagg and Edgar Gabriel and George Bosilca and Thara Angskun and Zizhong Chen and Jelena Pjesivac{\textendash}Grbovic and Kevin London and Jack Dongarra}
}
@article {icl:240,
	title = {Process Fault-Tolerance: Semantics, Design and Applications for High Performance Computing},
	journal = {International Journal for High Performance Applications and Supercomputing (to appear)},
	year = {2004},
	month = {2004-04},
	keywords = {ftmpi, lacsi},
	author = {Graham Fagg and Edgar Gabriel and Zizhong Chen and Thara Angskun and George Bosilca and Jelena Pjesivac{\textendash}Grbovic and Jack Dongarra}
}
@techreport {icl:251,
	title = {Recovery Patterns for Iterative Methods in a Parallel Unstable Environment},
	journal = {ICL Technical Report},
	number = {ICL-UT-04-04},
	year = {2004},
	month = {2004-01},
	author = {George Bosilca and Zizhong Chen and Jack Dongarra and Julien Langou}
}
@inproceedings {icl:153,
	title = {Fault Tolerant Communication Library and Applications for High Performance Computing},
	journal = {Los Alamos Computer Science Institute (LACSI) Symposium 2003 (presented)},
	year = {2003},
	month = {2003-10},
	address = {Santa Fe, NM},
	keywords = {ftmpi, lacsi},
	author = {Graham Fagg and Edgar Gabriel and Zizhong Chen and Thara Angskun and George Bosilca and Antonin Bukovsky and Jack Dongarra}
}
