@article {,
	title = {Then and Now: Improving Software Portability, Productivity, and 100{\texttimes} Performance},
	journal = {Computing in Science \& Engineering},
	year = {2024},
	month = {2024-04},
	pages = {1 - 10},
	abstract = {The US Exascale Computing Project (ECP) has succeeded in preparing applications to run efficiently on the first reported Exascale supercomputers in the world. To achieve this, it modernized the whole leadership software stack, from libraries to simulation codes. In this article, we contrast selected leadership software before and after ECP. We discuss how sustainable research software development for leadership computing can embrace the conversation with the hardware vendors, the leadership computing facilities, the software community, and the domain scientists who are the application developers and integrators of software products. We elaborate on how software needs to take portability as a central design principle and to benefit from interdependent teams; we also demonstrate how moving to programming languages with high momentum, like modern C++, can help improve the sustainability, interoperability, and performance of research software. Finally, we showcase how cross-institutional efforts can enable algorithm advances that are beyond incremental performance optimization.},
	issn = {1521-9615},
	doi = {10.1109/MCSE.2024.3387302},
	url = {https://ieeexplore.ieee.org/document/10495889/},
	author = {Anzt, Hartwig and Huebl, Axel and Li, Xiaoye S}
}
@conference {,
	title = {AI Benchmarking for Science: Efforts from the MLCommons Science Working Group},
	booktitle = {Lecture Notes in Computer Science},
	volume = {13387},
	year = {2023},
	month = {2023-01},
	pages = {47 - 64},
	publisher = {Springer International Publishing},
	organization = {Springer International Publishing},
	abstract = {With machine learning (ML) becoming a transformative tool for science, the scientific community needs a clear catalogue of ML techniques, and their relative benefits on various scientific problems, if they were to make significant advances in science using AI. Although this comes under the purview of benchmarking, conventional benchmarking initiatives are focused on performance, and as such, science, often becomes a secondary criteria.

In this paper, we describe a community effort from a working group, namely, MLCommons Science Working Group, in developing science-specific AI benchmarking for the international scientific community. Since the inception of the working group in 2020, the group has worked very collaboratively with a number of national laboratories, academic institutions and industries, across the world, and has developed four science-specific AI benchmarks. We will describe the overall process, the resulting benchmarks along with some initial results. We foresee that this initiative is likely to be very transformative for the AI for Science, and for performance-focused communities.},
	isbn = {978-3-031-23219-0},
	doi = {10.1007/978-3-031-23220-610.1007/978-3-031-23220-6_4},
	url = {https://link.springer.com/chapter/10.1007/978-3-031-23220-6_4},
	author = {Thiyagalingam, Jeyan and von Laszewski, Gregor and Yin, Junqi and Emani, Murali and Papay, Juri and Barrett, Gregg and Luszczek, Piotr and Tsaris, Aristeidis and Kirkpatrick, Christine and Wang, Feiyi and Gibbs, Tom and Vishwanath, Venkatram and Shankar, Mallikarjun and Fox, Geoffrey and Hey, Tony},
	editor = {Anzt, Hartwig and Bienz, Amanda and Luszczek, Piotr and Baboulin, Marc}
}
@booklet {,
	title = {Earth Virtualization Engines - A Technical Perspective},
	year = {2023},
	month = {2023-09},
	abstract = {Participants of the Berlin Summit on Earth Virtualization Engines (EVEs) discussed ideas and concepts to improve our ability to cope with climate change. EVEs aim to provide interactive and accessible climate simulations and data for a wide range of users. They combine high-resolution physics-based models with machine learning techniques to improve the fidelity, efficiency, and interpretability of climate projections. At their core, EVEs offer a federated data layer that enables simple and fast access to exabyte-sized climate data through simple interfaces. In this article, we summarize the technical challenges and opportunities for developing EVEs, and argue that they are essential for addressing the consequences of climate change.},
	url = {https://arxiv.org/abs/2309.09002},
	author = {Torsten Hoefler and Bjorn Stevens and Andreas F. Prein and Johanna Baehr and Thomas Schulthess and Thomas F. Stocker and John Taylor and Daniel Klocke and Pekka Manninen and Piers M. Forster and Tobias K{\"o}lling and Nicolas Gruber and Hartwig Anzt and Claudia Frauen and Florian Ziemen and Milan Kl{\"o}wer and Karthik Kashinath and Christoph Sch{\"a}r and Oliver Fuhrer and Bryan N. Lawrence}
}
@conference {,
	title = {GPU-based LU Factorization and Solve on Batches of Matrices with Band Structure},
	booktitle = {SC-W 2023: Workshops of The International Conference on High Performance Computing, Network, Storage, and Analysis},
	year = {2023},
	month = {2023-11},
	publisher = {ACM},
	organization = {ACM},
	address = {Denver, CO},
	abstract = {This paper presents a portable and performance-efficient approach to solve a batch of linear systems of equations using Graphics Processing Units (GPUs). Each system is represented using a special type of matrices with a band structure above and/or below the diagonal. Each matrix is factorized using an LU factorization with partial pivoting for numerical stability. Subsequently, the factors are used to find the solution for as many right hand sides as needed. The width of the band is often small enough that performing a fully dense LU factorization results in poor performance. We follow the standard LAPACK specifications for addressing this type of problems and develop a dedicated solver that runs efficiently on GPUs. No similar solver is currently available in the vendor{\textquoteright}s software stack, so performance results are shown on both NVIDIA and AMD GPUs relative to a parallel CPU solution utilizing OpenMP for thread-level parallelization.},
	isbn = {9798400707858},
	doi = {10.1145/3624062.3624247},
	url = {https://dl.acm.org/doi/abs/10.1145/3624062.3624247},
	author = {Abdelfattah, Ahmad and Tomov, Stanimire and Luszczek, Piotr and Anzt, Hartwig and Dongarra, Jack}
}
@conference {,
	title = {Mixed Precision Algebraic Multigrid on GPUs},
	booktitle = { Parallel Processing and Applied Mathematics (PPAM 2022)},
	volume = {13826},
	year = {2023},
	month = {2023-04},
	publisher = {Springer International Publishing},
	organization = {Springer International Publishing},
	address = {Cham},
	abstract = {In this paper, we present the first GPU-native platform-portable algebraic multigrid (AMG) implementation that allows the user to use different precision formats for the distinct multigrid levels. The AMG we present uses an aggregation size 2 parallel graph match as the AMG coarsening strategy. The implementation provides a high level of flexibility in terms of configuring the bottom-level solver and the precision format for the distinct levels. We present convergence and performance results on the GPUs from AMD, Intel, and NVIDIA, and compare against corresponding functionality available in other libraries.},
	keywords = {Algebraic multigrid, GPUs, mixed precision, Portability},
	isbn = {978-3-031-30441-5},
	doi = {10.1007/978-3-031-30442-2_9},
	url = {https://link.springer.com/10.1007/978-3-031-30442-2},
	author = {Tsai, Yu-Hsiang Mike and Natalie Beams and Anzt, Hartwig},
	editor = {Wyrzykowski, Roman and Dongarra, Jack and Deelman, Ewa and Karczewski, Konrad}
}
@conference {,
	title = {PAQR: Pivoting Avoiding QR factorization},
	booktitle = {2023 IEEE International Parallel and Distributed Processing Symposium (IPDPS)},
	year = {2023},
	publisher = {IEEE},
	organization = {IEEE},
	address = {St. Petersburg, FL, USA},
	doi = {10.1109/IPDPS54959.2023.00040},
	url = {https://ieeexplore.ieee.org/document/10177407/},
	author = {Sid-Lakhdar, Wissam and Cayrols, Sebastien and Bielich, Daniel and Abdelfattah, Ahmad and Luszczek, Piotr and Gates, Mark and Tomov, Stanimire and Johansen, Hans and Williams-Young, David and Davis, Timothy and Dongarra, Jack and Anzt, Hartwig}
}
@conference {,
	title = {Parallel Symbolic Cholesky Factorization},
	booktitle = {SC-W 2023: Workshops of The International Conference on High Performance Computing, Network, Storage, and Analysis},
	year = {2023},
	month = {2023-11},
	publisher = {ACM},
	organization = {ACM},
	address = {Denver, CO},
	abstract = {We present a hybrid sequential/parallel symbolic Cholesky factorization algorithm that computes the sparsity pattern of the symbolic factors in parallel. We evaluate the performance on a large subset of the SuiteSparse matrix collection and multicore CPUs as well as flagship GPUs by AMD and NVIDIA, achieving speedups of an order of magnitude compared to a state-of-the-art sequential symbolic Cholesky factorization.},
	isbn = {9798400707858},
	doi = {10.1145/3624062.3624253},
	url = {https://dl.acm.org/doi/proceedings/10.1145/3624062},
	author = {Ribizel, Tobias and Anzt, Hartwig}
}
@conference {,
	title = {Preconditioners for Batched Iterative Linear Solvers on GPUs},
	booktitle = {Smoky Mountains Computational Sciences and Engineering Conference},
	volume = {169075},
	year = {2023},
	month = {2023-01},
	pages = {38 - 53},
	publisher = {Springer Nature Switzerland},
	organization = {Springer Nature Switzerland},
	abstract = {Batched iterative solvers can be an attractive alternative to batched direct solvers if the linear systems allow for fast convergence. In non-batched settings, iterative solvers are often enhanced with sophisticated preconditioners to improve convergence. In this paper, we develop preconditioners for batched iterative solvers that improve the iterative solver convergence without incurring detrimental resource overhead and preserving much of the iterative solver flexibility. We detail the design and implementation considerations, present a user-friendly interface to the batched preconditioners, and demonstrate the convergence and runtime benefits over non-preconditioned batched iterative solvers on state-of-the-art GPUs for a variety of benchmark problems from finite difference stencil matrices, the Suitesparse matrix collection and a computational chemistry application.},
	isbn = {978-3-031-23605-1},
	doi = {10.1007/978-3-031-23606-810.1007/978-3-031-23606-8_3},
	url = {https://link.springer.com/chapter/10.1007/978-3-031-23606-8_3},
	author = {Aggarwal, Isha and Nayak, Pratik and Kashi, Aditya and Anzt, Hartwig},
	editor = {Doug, Kothe and Al, Geist and Pophale, Swaroop and Liu, Hong and Parete-Koon, Suzanne}
}
@conference {,
	title = {Reducing Data Motion and Energy Consumption of Geospatial Modeling Applications Using Automated Precision Conversion},
	booktitle = {2023 IEEE International Conference on Cluster Computing (CLUSTER)},
	year = {2023},
	month = {2023-11},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Santa Fe, NM, USA},
	abstract = {The burgeoning interest in large-scale geospatial modeling, particularly within the domains of climate and weather prediction, underscores the concomitant critical importance of accuracy, scalability, and computational speed. Harnessing these complex simulations{\textquoteright} potential, however, necessitates innovative computational strategies, especially considering the increasing volume of data involved. Recent advancements in Graphics Processing Units (GPUs) have opened up new avenues for accelerating these modeling processes. In particular, their efficient utilization necessitates new strategies, such as mixed-precision arithmetic, that can balance the trade-off between computational speed and model accuracy. This paper leverages PaRSEC runtime system and delves into the opportunities provided by mixed-precision arithmetic to expedite large-scale geospatial modeling in heterogeneous environments. By using an automated conversion strategy, our mixed-precision approach significantly improves computational performance (up to 3X) on Summit supercomputer and reduces the associated energy consumption on various Nvidia GPU generations. Importantly, this implementation ensures the requisite accuracy in environmental applications, a critical factor in their operational viability. The findings of this study bear significant implications for future research and development in high-performance computing, underscoring the transformative potential of mixed-precision arithmetic on GPUs in addressing the computational demands of large-scale geospatial modeling and making a stride toward a more sustainable, efficient, and accurate future in large-scale environmental applications.},
	doi = {10.1109/CLUSTER52292.2023.00035},
	url = {https://ieeexplore.ieee.org/document/10319946/},
	author = {Cao, Qinglei and Abdulah, Sameh and Ltaief, Hatem and Genton, Marc G. and Keyes, David and Bosilca, George}
}
@article {,
	title = {Sparse matrix-vector and matrix-multivector products for the truncated SVD on graphics processors},
	journal = {Concurrency and Computation: Practice and Experience},
	year = {2023},
	month = {2023-08},
	abstract = {Many practical algorithms for numerical rank computations implement an iterative procedure that involves repeated multiplications of a vector, or a collection of vectors, with both a sparse matrix A and its transpose. Unfortunately, the realization of these sparse products on current high performance libraries often deliver much lower arithmetic throughput when the matrix involved in the product is transposed. In this work, we propose a hybrid sparse matrix layout, named CSRC, that combines the flexibility of some well-known sparse formats to offer a number of appealing properties: (1) CSRC can be obtained at low cost from the popular CSR (compressed sparse row) format; (2) CSRC has similar storage requirements as CSR; and especially, (3) the implementation of the sparse product kernels delivers high performance for both the direct product and its transposed variant on modern graphics accelerators thanks to a significant reduction of atomic operations compared to a conventional implementation based on CSR. This solution thus renders considerably higher performance when integrated into an iterative algorithm for the truncated singular value decomposition (SVD), such as the randomized SVD or, as demonstrated in the experimental results, the block Golub{\textendash}Kahan{\textendash}Lanczos algorithm.},
	keywords = {graphics processing units, Singular value decomposition, sparse matrix-multivector product, sparse matrix-vector product},
	issn = {1532-0626},
	doi = {10.1002/cpe.7871},
	url = {https://onlinelibrary.wiley.com/doi/pdf/10.1002/cpe.7871},
	author = {Jos{\'e} I. Aliaga and Hartwig Anzt and Enrique S. Quintana-Orti and Andres E. Thomas}
}
@conference {,
	title = {Task-Based Polar Decomposition Using SLATE on Massively Parallel Systems with Hardware Accelerators},
	booktitle = {SC-W {\textquoteright}23: Proceedings of the SC {\textquoteright}23 Workshops of The International Conference on High Performance Computing, Network, Storage, and Analysis},
	year = {2023},
	month = {2023-11},
	publisher = {ACM},
	organization = {ACM},
	address = {Denver, CO},
	abstract = {We investigate a new task-based implementation of the polar decomposition on massively parallel systems augmented with multiple GPUs using SLATE. We implement the iterative QR Dynamically-Weighted Halley (QDWH) algorithm, whose building blocks mainly consist of compute-bound matrix operations, allowing for high levels of parallelism to be exploited on various hardware architectures, such as NVIDIA, AMD, and Intel GPU-based systems. To achieve both performance and portability, we implement our QDWH-based polar decomposition in the SLATE library, which uses efficient techniques in dense linear algebra, such as 2D block cyclic data distribution and communication-avoiding algorithms, as well as modern parallel programming approaches, such as dynamic scheduling and communication overlapping, and uses OpenMP tasks to track data dependencies.

We report numerical accuracy and performance results. The benchmarking campaign reveals up to an 18-fold performance speedup of the GPU accelerated implementation compared to the existing state-of-the-art implementation for the polar decomposition.},
	isbn = {9798400707858},
	doi = {10.1145/3624062.3624248},
	url = {https://dl.acm.org/doi/proceedings/10.1145/3624062},
	author = {Sukkari, Dalal and Gates, Mark and Al Farhan, Mohammed and Anzt, Hartwig and Dongarra, Jack}
}
@article {,
	title = {Three-precision algebraic multigrid on GPUs},
	journal = {Future Generation Computer Systems},
	year = {2023},
	month = {2023-07},
	abstract = {Recent research has demonstrated that using low precision inside some levels of an algebraic multigrid (AMG) solver can improve performance without negatively impacting the AMG quality. In this paper, we build upon previous research and implement an AMG that can use double, single, and half precision for the distinct multigrid levels. The implementation is platform-portable across GPU architectures from AMD, Intel, and NVIDIA. In an experimental analysis, we demonstrate that the use of half precision can be a viable option in multigrid. We evaluate the performance of different AMG configurations and demonstrate that mixed precision AMG can provide runtime savings compared to a double precision AMG.},
	keywords = {Algebraic multigrid, GPUs, mixed precision, Portability},
	issn = {0167739X},
	doi = {10.1016/j.future.2023.07.024},
	url = {https://www.sciencedirect.com/science/article/pii/S0167739X23002741},
	author = {Tsai, Yu-Hsiang Mike and Beams, Natalie and Anzt, Hartwig}
}
@article {,
	title = {Using Ginkgo{\textquoteright}s memory accessor for improving the accuracy of memory-bound low precision BLAS},
	journal = {Software: Practice and Experience},
	volume = {532},
	year = {2023},
	month = {Jan-01-2023},
	pages = {81 - 98},
	issn = {0038-0644},
	doi = {10.1002/spe.v53.110.1002/spe.3041},
	url = {https://doi.org/10.1002/spe.3041},
	author = {Gr{\"u}tzmacher, Thomas and Anzt, Hartwig and Quintana-Ort{\'\i}, Enrique S.}
}
@article {,
	title = {Accelerating Geostatistical Modeling and Prediction With Mixed-Precision Computations: A High-Productivity Approach With PaRSEC},
	journal = {IEEE Transactions on Parallel and Distributed Systems},
	volume = {33},
	year = {2022},
	month = {2022-04},
	pages = {964 - 976},
	abstract = {Geostatistical modeling, one of the prime motivating applications for exascale computing, is a technique for predicting desired quantities from geographically distributed data, based on statistical models and optimization of parameters. Spatial data are assumed to possess properties of stationarity or non-stationarity via a kernel fitted to a covariance matrix. A primary workhorse of stationary spatial statistics is Gaussian maximum log-likelihood estimation (MLE), whose central data structure is a dense, symmetric positive definite covariance matrix of the dimension of the number of correlated observations. Two essential operations in MLE are the application of the inverse and evaluation of the determinant of the covariance matrix. These can be rendered through the Cholesky decomposition and triangular solution. In this contribution, we reduce the precision of weakly correlated locations to single- or half- precision based on distance. We thus exploit mathematical structure to migrate MLE to a three-precision approximation that takes advantage of contemporary architectures offering BLAS3-like operations in a single instruction that are extremely fast for reduced precision. We illustrate application-expected accuracy worthy of double-precision from a majority half-precision computation, in a context where uniform single-precision is by itself insufficient. In tackling the complexity and imbalance caused by the mixing of three precisions, we deploy the PaRSEC runtime system. PaRSEC delivers on-demand casting of precisions while orchestrating tasks and data movement in a multi-GPU distributed-memory environment within a tile-based Cholesky factorization. Application-expected accuracy is maintained while achieving up to 1.59X by mixing FP64/FP32 operations on 1536 nodes of HAWK or 4096 nodes of Shaheen II , and up to 2.64X by mixing FP64/FP32/FP16 operations on 128 nodes of Summit , relative to FP64-only operations. This translates into up to 4.5, 4.7, ...},
	keywords = {Computational modeling, Covariance matrices, Data models, Maximum likelihood estimation, Predictive models, runtime, Task analysis},
	issn = {1045-9219},
	doi = {10.1109/TPDS.2021.3084071},
	url = {https://ieeexplore.ieee.org/document/9442267/https://ieeexplore.ieee.org/ielam/71/9575177/9442267-aam.pdfhttp://xplorestaging.ieee.org/ielx7/71/9575177/09442267.pdf?arnumber=9442267},
	author = {Abdulah, Sameh and Qinglei Cao and Pei, Yu and George Bosilca and Jack Dongarra and Genton, Marc G. and Keyes, David E. and Ltaief, Hatem and Sun, Ying}
}
@inproceedings {,
	title = {Addressing Irregular Patterns of Matrix Computations on GPUs and Their Impact on Applications Powered by Sparse Direct Solvers},
	journal = {2022 International Conference for High Performance Computing, Networking, Storage and Analysis (SC22)},
	year = {2022},
	month = {2022-11},
	pages = {354-367},
	publisher = {IEEE Computer Society},
	address = {Dallas, TX},
	abstract = {Many scientific applications rely on sparse direct solvers for their numerical robustness. However, performance optimization for these solvers remains a challenging task, especially on GPUs. This is due to workloads of small dense matrices that are different in size. Matrix decompositions on such irregular workloads are rarely addressed on GPUs. This paper addresses irregular workloads of matrix computations on GPUs, and their application to accelerate sparse direct solvers. We design an interface for the basic matrix operations supporting problems of different sizes. The interface enables us to develop irrLU-GPU, an LU decomposition on matrices of different sizes. We demonstrate the impact of irrLU-GPU on sparse direct LU solvers using NVIDIA and AMD GPUs. Experimental results are shown for a sparse direct solver based on a multifrontal sparse LU decomposition applied to linear systems arising from the simulation, using finite element discretization on unstructured meshes, of a high-frequency indefinite Maxwell problem.},
	keywords = {GPU computing, irregular computational workloads, lu factorization, multifrontal solvers, sparse direct solvers},
	url = {https://dl.acm.org/doi/abs/10.5555/3571885.3571919},
	author = {Ahmad Abdelfattah and Pieter Ghysels and Wajih Boukaram and Stanimire Tomov and Xiaoye Sherry Li and Jack Dongarra}
}
@techreport {,
	title = {Analysis of the Communication and Computation Cost of FFT Libraries towards Exascale},
	journal = {ICL Technical Report},
	number = {ICL-UT-22-07},
	year = {2022},
	month = {2022-07},
	publisher = {Innovative Computing Laboratory},
	author = {Alan Ayala and Stanimire Tomov and Piotr Luszczek and Sebastien Cayrols and Gerald Ragghianti and Jack Dongarra}
}
@inbook {,
	title = {Approximate Computing for Scientific Applications},
	booktitle = {Approximate Computing Techniques},
	year = {2022},
	month = {2022-01},
	pages = {415 - 465},
	publisher = {Springer International Publishing},
	organization = {Springer International Publishing},
	edition = {322},
	abstract = {This chapter reviews the performance benefits that result from applying (software) approximate computing to scientific applications. For this purpose, we target two particular areas, linear algebra and deep learning, with the first one selected for being ubiquitous in scientific problems and the second one for its considerable and growing number of important applications both in industry and science.

The review of linear algebra in scientific computing is focused on the iterative solution of sparse linear systems, exposing the prevalent costs of memory accesses in these methods, and demonstrating how approximate computing can help to reduce these overheads, for example, in the case of stationary solvers themselves or the application of preconditioners for the solution of sparse linear systems via Krylov subspace methods.

The discussion of deep learning is focused on the use of approximate data transfer for cutting costs of host-to-device operations, as well as the use of adaptive precision for accelerating training of classical CNN architectures. Additionally we discuss model optimization and architecture search in presence of constraints for edge devices applications.},
	isbn = {978-3-030-94704-0},
	doi = {10.1007/978-3-030-94705-7_14},
	url = {https://link.springer.com/chapter/10.1007/978-3-030-94705-7_14},
	author = {Anzt, Hartwig and Casas, Marc and Malossi, ~Cristiano I. and Quintana-Ort{\'\i}, Enrique S and Scheidegger, Florian and Zhuang, Sicong},
	editor = {Bosio, Alberto and M{\'e}nard, Daniel and Sentieys, Olivier}
}
@inbook {,
	title = {Batch QR Factorization on GPUs: Design, Optimization, and Tuning},
	booktitle = { Lecture Notes in Computer Science},
	volume = {13350},
	year = {2022},
	month = {2022-06},
	publisher = {Springer International Publishing},
	organization = {Springer International Publishing},
	address = {Cham},
	abstract = {QR factorization of dense matrices is a ubiquitous tool in high performance computing (HPC). From solving linear systems and least squares problems to eigenvalue problems, and singular value decompositions, the impact of a high performance QR factorization is fundamental to computer simulations and many applications. More importantly, the QR factorization on a batch of relatively small matrices has acquired a lot of attention in sparse direct solvers and low-rank approximations for Hierarchical matrices. To address this interest and demand, we developed and present a high performance batch QR factorization for Graphics Processing Units (GPUs). We present a multi-level blocking strategy that adjusts various algorithmic designs to the size of the input matrices. We also show that following the LAPACK QR design convention, while still useful, is significantly outperformed by unconventional code structures that increase data reuse. The performance results show multi-fold speedups against the state of the art libraries on the latest GPU architectures from both NVIDIA and AMD.},
	keywords = {Batch linear algebra, GPU computing, QR factorization},
	isbn = {978-3-031-08750-9},
	doi = {10.1007/978-3-031-08751-6_5},
	url = {https://link.springer.com/chapter/10.1007/978-3-031-08751-6_5},
	author = {Abdelfattah, Ahmad and Stanimire Tomov and Dongarra, Jack},
	editor = {Groen, Derek and de Mulatier, C{\'e}lia and Paszy{\'n}ski, Maciej and Krzhizhanovskaya, Valeria V. and Dongarra, Jack J. and Sloot, Peter M. A.}
}
@conference {,
	title = {Batched sparse iterative solvers on GPU for the collision operator for fusion plasma simulations},
	booktitle = {2022 IEEE International Parallel and Distributed Processing Symposium (IPDPS)},
	year = {2022},
	month = {2022-07},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Lyon, France},
	abstract = {Batched linear solvers, which solve many small related but independent problems, are important in several applications. This is increasingly the case for highly parallel processors such as graphics processing units (GPUs), which need a substantial amount of work to keep them operating efficiently and solving smaller problems one-by-one is not an option. Because of the small size of each problem, the task of coming up with a parallel partitioning scheme and mapping the problem to hardware is not trivial. In recent history, significant attention has been given to batched dense linear algebra. However, there is also an interest in utilizing sparse iterative solvers in a batched form, and this presents further challenges. An example use case is found in a gyrokinetic Particle-In-Cell (PIC) code used for modeling magnetically confined fusion plasma devices. The collision operator has been identified as a bottleneck, and a proxy app has been created for facilitating optimizations and porting to GPUs. The current collision kernel linear solver does not run on the GPU-a major bottleneck. As these matrices are well-conditioned, batched iterative sparse solvers are an attractive option. A batched sparse iterative solver capability has recently been developed in the Ginkgo library. In this paper, we describe how the software architecture can be used to develop an efficient solution for the XGC collision proxy app. Comparisons for the solve times on NVIDIA V100 and A100 GPUs and AMD MI100 GPUs with one dual-socket Intel Xeon Skylake CPU node with 40 OpenMP threads are presented for matrices representative of those required in the collision kernel of XGC. The results suggest that GINKGO{\textquoteright}s batched sparse iterative solvers are well suited for efficient utilization of the GPU for this problem, and the performance portability of Ginkgo in conjunction with Kokkos (used within XGC as the heterogeneous programming model) allows seamless execution for exascale oriented heterogeneous architectures at the various leadership supercomputing facilities.},
	doi = {10.1109/IPDPS53621.2022.00024},
	url = {https://ieeexplore.ieee.org/document/9820663},
	author = {Kashi, Aditya and Nayak, Pratik and Kulkarni, Dhruva and Scheinberg, Aaron and Lin, Paul and Anzt, Hartwig}
}
@techreport {,
	title = {Communication Avoiding LU with Tournament Pivoting in SLATE},
	journal = {SLATE Working Notes},
	number = {18, ICL-UT-22-01},
	year = {2022},
	month = {2022-01},
	author = {Rabab Alomairy and Mark Gates and Sebastien Cayrols and Dalal Sukkari and Kadir Akbudak and Asim YarKhan and Paul Bagwell and Jack Dongarra}
}
@article {,
	title = {Compressed basis GMRES on high-performance graphics processing units},
	journal = {The International Journal of High Performance Computing Applications},
	year = {2022},
	month = {2022-05},
	abstract = {Krylov methods provide a fast and highly parallel numerical tool for the iterative solution of many large-scale sparse linear systems. To a large extent, the performance of practical realizations of these methods is constrained by the communication bandwidth in current computer architectures, motivating the investigation of sophisticated techniques to avoid, reduce, and/or hide the message-passing costs (in distributed platforms) and the memory accesses (in all architectures). This article leverages Ginkgo{\textquoteright}s memory accessor in order to integrate a communication-reduction strategy into the (Krylov) GMRES solver that decouples the storage format (i.e., the data representation in memory) of the orthogonal basis from the arithmetic precision that is employed during the operations with that basis. Given that the execution time of the GMRES solver is largely determined by the memory accesses, the cost of the datatype transforms can be mostly hidden, resulting in the acceleration of the iterative step via a decrease in the volume of bits being retrieved from memory. Together with the special properties of the orthonormal basis (whose elements are all bounded by 1), this paves the road toward the aggressive customization of the storage format, which includes some floating-point as well as fixed-point formats with mild impact on the convergence of the iterative process. We develop a high-performance implementation of the {\textquotedblleft}compressed basis GMRES{\textquotedblright} solver in the Ginkgo sparse linear algebra library using a large set of test problems from the SuiteSparse Matrix Collection. We demonstrate robustness and performance advantages on a modern NVIDIA V100 graphics processing unit (GPU) of up to 50\% over the standard GMRES solver that stores all data in IEEE double-precision.},
	issn = {1094-3420},
	doi = {10.1177/10943420221115140},
	url = {http://journals.sagepub.com/doi/10.1177/10943420221115140},
	author = {Aliaga, Jos{\'e} I and Anzt, Hartwig and Gr{\"u}tzmacher, Thomas and Quintana-Ort{\'\i}, Enrique S and Andres E. Thomas}
}
@article {,
	title = {Compression and load balancing for efficient sparse matrix-vector product on multicore processors and graphics processing units},
	journal = {Concurrency and Computation: Practice and Experience},
	volume = {34},
	year = {2022},
	month = {2022-06},
	issn = {1532-0626},
	doi = {10.1002/cpe.6515},
	url = {https://doi.org/10.1002/cpe.6515},
	author = {Aliaga, Jos{\'e} I. and Anzt, Hartwig and Gr{\"u}tzmacher, Thomas and Quintana-Orti, Enrique S. and Andres E. Thomas}
}
@inproceedings {,
	title = {Deep Gaussian process with multitask and transfer learning for performance optimization},
	journal = {2022 IEEE High Performance Extreme Computing Conference (HPEC)},
	year = {2022},
	month = {2022-09},
	pages = {1-7},
	abstract = {We combine Deep Gaussian Processes with multitask and transfer learning for the performance modeling and optimization of HPC applications. Deep Gaussian processes merge the uncertainty quantification advantage of Gaussian Processes with the predictive power of deep learning. Multitask and transfer learning allow for improved learning efficiency when several similar tasks are to be learned simultaneously and when previous learned models are sought to help in the learning of new tasks, respectively. A comparison with state-of-the-art autotuners shows the advantage of our approach on two application problems.},
	doi = {10.1109/HPEC55821.2022.9926396},
	author = {Sid-Lakhdar, Wissam M. and Aznaveh, Mohsen and Luszczek, Piotr and Dongarra, Jack}
}
@techreport {,
	title = {FFT Benchmark Performance Experiments on Systems Targeting Exascale},
	journal = {ICL Technical Report},
	number = {ICL-UT-22-02},
	year = {2022},
	month = {2022-03},
	author = {Alan Ayala and Stanimire Tomov and Piotr Luszczek and Sebastien Cayrols and Gerald Ragghianti and Jack Dongarra}
}
@conference {,
	title = {A Framework to Exploit Data Sparsity in Tile Low-Rank Cholesky Factorization},
	booktitle = {IEEE International Parallel and Distributed Processing Symposium (IPDPS)},
	year = {2022},
	month = {2022-07},
	doi = {10.1109/IPDPS53621.2022.00047},
	url = {https://ieeexplore.ieee.org/stamp/stamp.jsp?tp=\&arnumber=9820680\&isnumber=9820610},
	author = {Qinglei Cao and Rabab Alomairy and Yu Pei and George Bosilca and Hatem Ltaief and David Keyes and Jack Dongarra}
}
@article {,
	title = {Ginkgo: A Modern Linear Operator Algebra Framework for High Performance Computing},
	journal = {ACM Transactions on Mathematical Software},
	volume = {48},
	year = {2022},
	month = {2022-03},
	pages = {1 - 33},
	abstract = {In this article, we present Ginkgo, a modern C++ math library for scientific high performance computing. While classical linear algebra libraries act on matrix and vector objects, Ginkgo{\textquoteright}s design principle abstracts all functionality as {\textquotedblleft}linear operators,{\textquotedblright} motivating the notation of a {\textquotedblleft}linear operator algebra library.{\textquotedblright} Ginkgo{\textquoteright}s current focus is oriented toward providing sparse linear algebra functionality for high performance graphics processing unit (GPU) architectures, but given the library design, this focus can be easily extended to accommodate other algorithms and hardware architectures. We introduce this sophisticated software architecture that separates core algorithms from architecture-specific backends and provide details on extensibility and sustainability measures. We also demonstrate Ginkgo{\textquoteright}s usability by providing examples on how to use its functionality inside the MFEM and deal.ii finite element ecosystems. Finally, we offer a practical demonstration of Ginkgo{\textquoteright}s high performance on state-of-the-art GPU architectures.},
	issn = {0098-3500},
	doi = {10.1145/3480935},
	url = {https://dl.acm.org/doi/10.1145/3480935},
	author = {Anzt, Hartwig and Cojean, Terry and Flegar, Goran and G{\"o}bel, Fritz and Gr{\"u}tzmacher, Thomas and Nayak, Pratik and Ribizel, Tobias and Tsai, Yuhsiang Mike and Quintana-Ort{\'\i}, Enrique S}
}
@article {,
	title = {Ginkgo{\textemdash}A math library designed for platform portability},
	journal = {Parallel Computing},
	volume = {111},
	year = {2022},
	month = {2022-02},
	pages = {102902},
	abstract = {In an era of increasing computer system diversity, the portability of software from one system to another plays a central role. Software portability is important for the software developers as many software projects have a lifetime longer than a specific system, e.g., a supercomputer, and it is important for the domain scientists that realize their scientific application in a software framework and want to be able to run on one or another system. On a high level, there exist two approaches for realizing platform portability: (1) implementing software using a portability layer leveraging any technique which always generates specific kernels from another language or through an interface for running on different architectures; and (2) providing backends for different hardware architectures, with the backends typically differing in how and in which programming language functionality is realized due to using the language of choice for each hardware (e.g., CUDA kernels for NVIDIA GPUs, SYCL (DPC++) kernels to targeting Intel GPUs and other supported hardware, {\textellipsis}). In practice, these two approaches can be combined in applications to leverage their respective strengths. In this paper, we present how we realize portability across different hardware architectures for the Ginkgo library by following the second strategy and the goal to not only port to new hardware architectures but also achieve good performance. We present the Ginkgo library design, separating algorithms from hardware-specific kernels forming the distinct hardware executors, and report our experience when adding execution backends for NVIDIA, AMD, and Intel GPUs. We also present the performance we achieve with this approach for distinct hardware backends.},
	keywords = {AMD, Intel, nVidia, performance portability, Platform Portability, Porting to GPU accelerators},
	issn = {0167-8191},
	doi = {https://doi.org/10.1016/j.parco.2022.102902},
	url = {https://www.sciencedirect.com/science/article/pii/S0167819122000096},
	author = {Terry Cojean and Yu-Hsiang Mike Tsai and Hartwig Anzt}
}
@inproceedings {,
	title = {Lossy all-to-all exchange for accelerating parallel 3-D FFTs on hybrid architectures with GPUs},
	journal = {2022 IEEE International Conference on Cluster Computing (CLUSTER)},
	year = {2022},
	month = {2022-09},
	pages = {152-160},
	abstract = {In the context of parallel applications, communication is a critical part of the infrastructure and a potential bottleneck. The traditional approach to tackle communication challenges consists of redesigning algorithms so that the complexity or the communication volume is reduced. However, there are algorithms like the Fast Fourier Transform (FFT) where reducing the volume of communication is very challenging yet can reap large benefit in terms of time-to-completion. In this paper, we revisit the implementation of the MPI all-to-all routine at the core of 3D FFTs by using advanced MPI features, such as One-Sided Communication, and integrate data compression during communication to reduce the volume of data exchanged. Since some compression techniques are {\textquoteleft}lossy{\textquoteright} in the sense that they involve a loss of accuracy, we study the impact of lossy compression in heFFTe, the state-of-the-art FFT library for large scale 3D FFTs on hybrid architectures with GPUs. Consequently, we design an approximate FFT algorithm that trades off user-controlled accuracy for speed. We show that we speedup the 3D FFTs proportionally to the compression rate. In terms of accuracy, comparing our approach with a reduced precision execution, where both the data and the computation are in reduced precision, we show that when the volume of communication is compressed to the size of the reduced precision data, the approximate FFT algorithm is as fast as the one in reduced precision while the accuracy is one order of magnitude better.},
	doi = {10.1109/CLUSTER51413.2022.00029},
	author = {Cayrols, Sebastien and Li, Jiali and George Bosilca and Stanimire Tomov and Ayala, Alan and Dongarra, Jack}
}
@techreport {,
	title = {Mixed precision and approximate 3D FFTs: Speed for accuracy trade-off with GPU-aware MPI and run-time data compression},
	journal = {ICL Technical Report},
	number = {ICL-UT-22-04},
	year = {2022},
	month = {2022-05},
	keywords = {All-to-all, Approximate FFTs, ECP, heFFTe, Lossy compression, mixed-precision algorithms, MPI},
	author = {Sebastien Cayrols and Jiali Li and George Bosilca and Stanimire Tomov and Alan Ayala and Jack Dongarra}
}
@techreport {,
	title = {PAQR: Pivoting Avoiding QR factorization},
	journal = {ICL Technical Report},
	number = {ICL-UT-22-06},
	year = {2022},
	month = {2022-06},
	abstract = {The solution of linear least-squares problems is at the heart of many scientific and engineering applications. While any method able to minimize the backward error of such problems is considered numerically stable, the theory states that the forward error depends on the condition number of the
matrix in the system of equations. On the one hand, the QR factorization is an efficient method to solve such problems, but the solutions it produces may have large forward errors when the matrix is deficient. On the other hand, QR with column pivoting (QRCP) is able to produce smaller forward errors on deficient matrices, but its cost is prohibitive compared to QR. The aim of this paper is to propose PAQR, an alternative solution method with the same cost (or smaller) as QR and as accurate as QRCP in practical cases, for the solution of rank-deficient linear least-squares problems. After presenting the algorithm and its implementations on different architectures, we compare its accuracy and performance results on a variety of application problems.
},
	author = {Wissam M. Sid-Lakhdar and Sebastien Cayrols and Daniel Bielich and Ahmad Abdelfattah and Piotr Luszczek and Mark Gates and Stanimire Tomov and Hans Johansen and David Williams-Young and Timothy A. Davis and Jack Dongarra}
}
@conference {,
	title = {Performance Analysis of Parallel FFT on Large Multi-GPU Systems},
	booktitle = {2022 IEEE International Parallel and Distributed Processing Symposium Workshops (IPDPSW)},
	year = {2022},
	month = {2022-08},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Lyon, France},
	doi = {10.1109/IPDPSW55747.2022.00072},
	url = {https://ieeexplore.ieee.org/document/9835388/},
	author = {Ayala, Alan and Stanimire Tomov and Stoyanov, Miroslav and Haidar, Azzam and Dongarra, Jack}
}
@inproceedings {,
	title = {Porting Sparse Linear Algebra to~Intel GPUs},
	journal = {Euro-Par 2021: Parallel Processing Workshops},
	volume = {13098},
	year = {2022},
	month = {2022-06},
	pages = {57 - 68},
	publisher = {Springer International Publishing},
	address = {Lisbon, Portugal},
	abstract = {With discrete Intel GPUs entering the high performance computing landscape, there is an urgent need for production-ready software stacks for these platforms. In this paper, we report how we prepare the Ginkgo math library for Intel GPUs by developing a kernel backed based on the DPC++ programming environment. We discuss conceptual differences to the CUDA and HIP programming models and describe workflows for simplified code conversion. We benchmark advanced sparse linear algebra routines utilizing the converted kernels to assess the efficiency of the DPC++ backend in the hardware-specific performance bounds. We compare the performance of basic building blocks against routines providing the same functionality that ship with Intel{\textquoteright}s oneMKL vendor library.},
	keywords = {Ginkgo, Intel GPUs, math library, oneAPI, SpMV},
	isbn = {978-3-031-06155-4},
	doi = {10.1007/978-3-031-06156-1_5},
	url = {https://link.springer.com/chapter/10.1007/978-3-031-06156-1_5},
	author = {Tsai, Yuhsiang M. and Cojean, Terry and Anzt, Hartwig},
	editor = {Chaves, Ricardo and B. Heras, Dora and Ilic, Aleksandar and Unat, Didem and Badia, Rosa M. and Bracciali, Andrea and Diehl, Patrick and Dubey, Anshu and Sangyoon, Oh and L. Scott, Stephen and Ricci, Laura}
}
@inproceedings {,
	title = {Prediction of Optimal Solvers for Sparse Linear Systems Using Deep Learning},
	journal = {2022 SIAM Conference on Parallel Processing for Scientific Computing (PP)},
	year = {2022},
	month = {2022},
	pages = {14 - 24},
	publisher = {Society for Industrial and Applied Mathematics},
	address = {Philadelphia, PA},
	abstract = {Solving sparse linear systems is a key task in a number of computational problems, such as data analysis and simulations, and majorly determines overall execution time. Choosing a suitable iterative solver algorithm, however, can significantly improve time-to-completion. We present a deep learning approach designed to predict the optimal iterative solver for a given sparse linear problem. For this, we detail useful linear system features to drive the prediction process, the metrics we use to quantify the iterative solvers{\textquoteright} time-to-approximation performance and a comprehensive experimental evaluation of the prediction quality of the neural network. Using a hyperparameter optimization and an ablation study on the SuiteSparse matrix collection we have inferred the importance of distinct features, achieving a top-1 classification accuracy of 60\%.},
	doi = {10.1137/1.978161197714110.1137/1.9781611977141.2},
	url = {https://epubs.siam.org/doi/10.1137/1.9781611977141.2},
	author = {Funk, Yannick and G{\"o}tz, Markus and Anzt, Hartwig},
	editor = {Li, Xiaoye S. and Teranishi, Keita}
}
@article {,
	title = {Providing performance portable numerics for Intel GPUs},
	journal = {Concurrency and Computation: Practice and Experience},
	volume = {17},
	year = {2022},
	month = {2022-10},
	abstract = {With discrete Intel GPUs entering the high-performance computing landscape, there is an urgent need for production-ready software stacks for these platforms. In this article, we report how we enable the Ginkgo math library to execute on Intel GPUs by developing a kernel backed based on the DPC++ programming environment. We discuss conceptual differences between the CUDA and DPC++ programming models and describe workflows for simplified code conversion. We evaluate the performance of basic and advanced sparse linear algebra routines available in Ginkgo{\textquoteright}s DPC++ backend in the hardware-specific performance bounds and compare against routines providing the same functionality that ship with Intel{\textquoteright}s oneMKL vendor library.},
	keywords = {Ginkgo, Intel GPUs, math library, oneAPI, SpMV},
	issn = {1532-0626},
	doi = {10.1002/cpe.7400},
	url = {https://onlinelibrary.wiley.com/doi/full/10.1002/cpe.7400},
	author = {Tsai, Yu-Hsiang M. and Cojean, Terry and Anzt, Hartwig}
}
@inproceedings {,
	title = {Reshaping Geostatistical Modeling and Prediction for Extreme-Scale Environmental Applications},
	journal = {2022 International Conference for High Performance Computing, Networking, Storage and Analysis (SC22)},
	year = {2022},
	month = {2022-11},
	publisher = {IEEE Press},
	address = {Dallas, TX},
	abstract = {We extend the capability of space-time geostatistical modeling using algebraic approximations, illustrating application-expected accuracy worthy of double precision from majority low-precision computations and low-rank matrix approximations. We exploit the mathematical structure of the dense covariance matrix whose inverse action and determinant are repeatedly required in Gaussian log-likelihood optimization. Geostatistics augments first-principles modeling approaches for the prediction of environmental phenomena given the availability of measurements at a large number of locations; however, traditional Cholesky-based approaches grow cubically in complexity, gating practical extension to continental and global datasets now available. We combine the linear algebraic contributions of mixed-precision and low-rank computations within a tilebased Cholesky solver with on-demand casting of precisions and dynamic runtime support from PaRSEC to orchestrate tasks and data movement. Our adaptive approach scales on various systems and leverages the Fujitsu A64FX nodes of Fugaku to achieve up to 12X performance speedup against the highly optimized dense Cholesky implementation.},
	keywords = {climate/weather prediction, dynamic runtime systems, high performance computing., low- rank matrix approximations, mixed-precision computations, space-time geospatial statistics, Task-based programming models},
	isbn = {9784665454445},
	url = {https://dl.acm.org/doi/abs/10.5555/3571885.3571888},
	author = {Cao, Qinglei and Abdulah, Sameh and Rabab Alomairy and Pei, Yu and Pratik Nag and George Bosilca and Dongarra, Jack and Genton, Marc G. and Keyes, David and Ltaief, Hatem and Sun, Ying}
}
@article {,
	title = {Resiliency in numerical algorithm design for extreme scale simulations},
	journal = {The International Journal of High Performance Computing Applications},
	volume = {36371337212766180823},
	year = {2022},
	month = {2022-03},
	pages = {251 - 285},
	keywords = {Fault tolerance, Numerical algorithms, parallel computer architecture, resilience},
	issn = {1094-3420},
	doi = {10.1177/10943420211055188},
	url = {http://journals.sagepub.com/doi/10.1177/10943420211055188http://journals.sagepub.com/doi/pdf/10.1177/10943420211055188http://journals.sagepub.com/doi/pdf/10.1177/10943420211055188http://journals.sagepub.com/doi/full-xml/10.1177/10943420211055188},
	author = {Agullo, Emmanuel and Altenbernd, Mirco and Anzt, Hartwig and Bautista-Gomez, Leonardo and Benacchio, Tommaso and Bonaventura, Luca and Bungartz, Hans-Joachim and Chatterjee, Sanjay and Ciorba, Florina M and DeBardeleben, Nathan and Drzisga, Daniel and Eibl, Sebastian and Engelmann, Christian and Gansterer, Wilfried N and Giraud, Luc and G{\"o}ddeke, Dominik and Heisig, Marco and J{\'e}z{\'e}quel, Fabienne and Kohl, Nils and Li, Xiaoye Sherry and Lion, Romain and Mehl, Miriam and Mycek, Paul and Obersteiner, Michael and Quintana-Ort{\'\i}, Enrique S and Rizzi, Francesco and R{\"u}de, Ulrich and Schulz, Martin and Fung, Fred and Speck, Robert and Stals, Linda and Teranishi, Keita and Thibault, Samuel and Th{\"o}nnes, Dominik and Wagner, Andreas and Wohlmuth, Barbara}
}
@article {,
	title = {Accelerating FFT towards Exascale Computing},
	year = {2021},
	publisher = {NVIDIA GPU Technology Conference (GTC2021)},
	author = {Alan Ayala and Stanimire Tomov and Haidar, Azzam and Stoyanov, M. and Cayrols, Sebastien and Li, Jiali and George Bosilca and Jack Dongarra}
}
@conference {,
	title = {Accelerating Multi - Process Communication for Parallel 3-D FFT},
	booktitle = {2021 Workshop on Exascale MPI (ExaMPI)},
	year = {2021},
	month = {2021-12},
	publisher = {IEEE},
	organization = {IEEE},
	address = {St. Louis, MO, USA},
	abstract = {Today largest and most powerful supercomputers in the world are built on heterogeneous platforms; and using the combined power of multi-core CPUs and GPUs, has had a great impact accelerating large-scale applications. However, on these architectures, parallel algorithms, such as the Fast Fourier Transform (FFT), encounter that inter-processor communication become a bottleneck and limits their scalability. In this paper, we present techniques for speeding up multi-process communication cost during the computation of FFTs, considering hybrid network connections as those expected on upcoming exascale machines. Among our techniques, we present algorithmic tuning, making use of phase diagrams; parametric tuning, using different FFT settings; and MPI distribution tuning based on FFT size and computational resources available. We present several experiments obtained on Summit supercomputer at Oak Ridge National Laboratory, using up to 40,960 IBM Power9 cores and 6,144 NVIDIA V-100 GPUs.},
	doi = {10.1109/ExaMPI54564.2021.00011},
	url = {https://ieeexplore.ieee.org/document/9652837/},
	author = {Ayala, Alan and Tomov, Stan and Stoyanov, Miroslav and Haidar, Azzam and Dongarra, Jack}
}
@article {,
	title = {Efficient exascale discretizations: High-order finite element methods},
	journal = {The International Journal of High Performance Computing Applications},
	year = {2021},
	pages = {10943420211020803},
	abstract = {Efficient exploitation of exascale architectures requires rethinking of the numerical algorithms used in many large-scale applications. These architectures favor algorithms that expose ultra fine-grain parallelism and maximize the ratio of floating point operations to energy intensive data movement. One of the few viable approaches to achieve high efficiency in the area of PDE discretizations on unstructured grids is to use matrix-free/partially assembled high-order finite element methods, since these methods can increase the accuracy and/or lower the computational time due to reduced data motion. In this paper we provide an overview of the research and development activities in the Center for Efficient Exascale Discretizations (CEED), a co-design center in the Exascale Computing Project that is focused on the development of next-generation discretization software and algorithms to enable a wide range of finite element applications to run efficiently on future hardware. CEED is a research partnership involving more than 30 computational scientists from two US national labs and five universities, including members of the Nek5000, MFEM, MAGMA and PETSc projects. We discuss the CEED co-design activities based on targeted benchmarks, miniapps and discretization libraries and our work on performance optimizations for large-scale GPU architectures. We also provide a broad overview of research and development activities in areas such as unstructured adaptive mesh refinement algorithms, matrix-free linear solvers, high-order data visualization, and list examples of collaborations with several ECP and external applications.},
	keywords = {co-design, high-order discretizations, High-performance computing, PDEs, unstructured grids},
	doi = {10.1177/10943420211020803},
	author = {Kolev, Tzanio and Fischer, Paul and Min, Misun and Jack Dongarra and Brown, Jed and Dobrev, Veselin and Warburton, Tim and Stanimire Tomov and Shephard, Mark S and Abdelfattah, Ahmad and others}
}
@article {,
	title = {Gingko: A Sparse Linear Algebrea Library for HPC},
	year = {2021},
	month = {2021-04},
	publisher = {2021 ECP Annual Meeting},
	author = {Hartwig Anzt and Natalie Beams and Terry Cojean and Fritz G{\"o}bel and Thomas Gr{\"u}tzmacher and Aditya Kashi and Pratik Nayak and Tobias Ribizel and Yuhsiang M. Tsai}
}
@article {,
	title = {GPU algorithms for Efficient Exascale Discretizations},
	journal = {Parallel Computing},
	volume = {108},
	year = {2021},
	pages = {102841},
	abstract = {In this paper we describe the research and development activities in the Center for Efficient Exascale Discretization within the US Exascale Computing Project, targeting state-of-the-art high-order finite-element algorithms for high-order applications on GPU-accelerated platforms. We discuss the GPU developments in several components of the CEED software stack, including the libCEED, MAGMA, MFEM, libParanumal, and Nek projects. We report performance and capability improvements in several CEED-enabled applications on both NVIDIA and AMD GPU systems.},
	keywords = {Exascale applications, Finite element methods, GPU acceleration, high-order discretizations, High-performance computing},
	doi = {10.1016/j.parco.2021.102841},
	author = {Abdelfattah, Ahmad and Valeria Barra and Natalie Beams and Bleile, Ryan and Brown, Jed and Camier, Jean-Sylvain and Carson, Robert and Chalmers, Noel and Dobrev, Veselin and Dudouit, Yohann and others}
}
@techreport {,
	title = {Interim Report on Benchmarking FFT Libraries on High Performance Systems},
	journal = {Innovative Computing Laboratory Technical Report},
	number = {ICL-UT-21-03},
	year = {2021},
	month = {2021-07},
	publisher = {University of Tennessee},
	type = {ICL Tech Report},
	abstract = {The Fast Fourier Transform (FFT) is used in many applications such as molecular dynamics, spectrum estimation, fast convolution and correlation, signal modulation, and many wireless multimedia applications. FFTs are also heavily used in ECP applications, such as EXAALT, Copa, ExaSky-HACC, ExaWind, WarpX, and many others. As these applications{\textquoteright} accuracy and speed depend on the performance of the FFTs, we designed an FFT benchmark to mea- sure performance and scalability of currently available FFT packages and present the results from a pre-Exascale platform. Our benchmarking also stresses the overall capacity of system interconnect; thus, it may be considered as an indicator of the bisection bandwidth, communication contention noise, and the software overheads in MPI collectives that are of interest to many other ECP applications and libraries.

This FFT benchmarking project aims to show the strengths and weaknesses of multiple FFT libraries and to indicate what can be done to improve their performance. In particular, we believe that the benchmarking results could help design and implement a fast and robust FFT library for 2D and 3D inputs, while targeting large-scale heterogeneous systems with multicore processors and hardware accelerators that are a co-designed in tandem with ECP applications. Our work involves studying and analyzing state-of-the-art FFT software both from vendors and available as open-source codes to better understand their performance.},
	author = {Alan Ayala and Stanimire Tomov and Piotr Luszczek and Cayrols, Sebastien and Ragghianti, Gerald and Jack Dongarra}
}
@book {,
	title = {Lecture Notes in Computer Science: High Performance Computing},
	volume = {12761},
	year = {2021},
	publisher = {Springer International Publishing},
	organization = {Springer International Publishing},
	abstract = {This book constitutes the refereed post-conference proceedings of 9 workshops held at the 35th International ISC High Performance 2021 Conference, in Frankfurt, Germany, in June-July 2021: Second International Workshop on the Application of Machine Learning Techniques to Computational Fluid Dynamics and Solid Mechanics Simulations and Analysis; HPC-IODC: HPC I/O in the Data Center Workshop; Compiler-assisted Correctness Checking and Performance Optimization for HPC; Machine Learning on HPC Systems; 4th International Workshop on Interoperability of Supercomputing and Cloud Technologies; 2nd International Workshop on Monitoring and Operational Data Analytics; 16th Workshop on Virtualization in High--Performance Cloud Computing; Deep Learning on Supercomputers;  5th International Workshop on In Situ Visualization. The 35 papers included in this volume were carefully reviewed and selected. They cover all aspects of research, development, and application of large-scale, high performance experimental and commercial systems. Topics include high-performance computing (HPC), computer architecture and hardware, programming models, system software, performance analysis and modeling, compiler analysis and optimization techniques, software sustainability, scientific applications, deep learning.},
	isbn = {978-3-030-90538-5},
	doi = {10.1007/978-3-030-90539-2},
	author = {Heike Jagode and Anzt, Hartwig and Ltaief, Hatem and Piotr Luszczek}
}
@conference {,
	title = {Leveraging PaRSEC Runtime Support to Tackle Challenging 3D Data-Sparse Matrix Problems},
	booktitle = {35th IEEE International Parallel \& Distributed Processing Symposium (IPDPS 2021)},
	year = {2021},
	month = {2021-05},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Portland, OR},
	abstract = {The task-based programming model associated with dynamic runtime systems has gained popularity for challenging problems because of workload imbalance, heterogeneous resources, or extreme concurrency. During the last decade, lowrank matrix approximations, where the main idea consists of exploiting data sparsity typically by compressing off-diagonal tiles up to an application-specific accuracy threshold, have been adopted to address the curse of dimensionality at extreme scale. In this paper, we create a bridge between the runtime and the linear algebra by communicating knowledge of the data sparsity to the runtime. We design and implement this synergistic approach with high user productivity in mind, in the context of the PaRSEC runtime system and the HiCMA numerical library. This requires to extend PaRSEC with new features to integrate rank information into the dataflow so that proper decisions can be taken at runtime. We focus on the tile low-rank (TLR) Cholesky factorization for solving 3D data-sparse covariance matrix problems arising in environmental applications. In particular, we employ the 3D exponential model of Matern matrix kernel, which exhibits challenging nonuniform {\textasciiacute}high ranks in off-diagonal tiles. We first provide a dynamic data structure management driven by a performance model to reduce extra floating-point operations. Next, we optimize the memory footprint of the application by relying on a dynamic memory allocator, and supported by a rank-aware data distribution to cope with the workload imbalance. Finally, we expose further parallelism using kernel recursive formulations to shorten the critical path. Our resulting high-performance implementation outperforms existing data-sparse TLR Cholesky factorization by up to 7-fold on a large-scale distributed-memory system, while minimizing the memory footprint up to a 44-fold factor. This multidisciplinary work highlights the need to empower runtime systems beyond their original duty of task scheduling for servicing next-generation low-rank matrix algebra libraries.},
	keywords = {asynchronous executions and load balancing, dynamic runtime system, environmental applications, High-performance computing, low-rank matrix computations, task-based programming model, user productivity},
	author = {Qinglei Cao and Yu Pei and Kadir Akbudak and George Bosilca and Hatem Ltaief and David Keyes and Jack Dongarra}
}
@article {,
	title = {libCEED: Fast algebra for high-order element-based discretizations},
	journal = {Journal of Open Source Software},
	volume = {6},
	number = {63},
	year = {2021},
	pages = {2945},
	abstract = {Finite element methods are widely used to solve partial differential equations (PDE) in science and engineering, but their standard implementation (Arndt et al., 2020; Kirk et al., 2006; Logg et al., 2012) relies on assembling sparse matrices. Sparse matrix multiplication and triangular operations perform a scalar multiply and add for each nonzero entry, just 2 floating point operations (flops) per scalar that must be loaded from memory (Williams et al., 2009). Modern hardware is capable of nearly 100 flops per scalar streamed from memory (Rupp, 2020) so sparse matrix operations cannot achieve more than about 2\% utilization of arithmetic units.
Matrix assembly becomes even more problematic when the polynomial degree p of the basis functions is increased, resulting in O(pd) storage and O(p2d) compute per degree of freedom (DoF) in d dimensions. Methods pioneered by the spectral element community (Deville et al., 2002; Orszag, 1980) exploit problem structure to reduce costs to O(1) storage and O(p)
compute per DoF, with very high utilization of modern CPUs and GPUs. Unfortunately, highquality implementations have been relegated to applications and intrusive frameworks that are often difficult to extend to new problems or incorporate into legacy applications, especially when strong preconditioners are required.
libCEED, the Code for Efficient Extensible Discretization (Abdelfattah et al., 2021), is a lightweight library that provides a purely algebraic interface for linear and nonlinear operators and preconditioners with element-based discretizations. libCEED provides portable performance via run-time selection of implementations optimized for CPUs and GPUs, including
support for just-in-time (JIT) compilation. It is designed for convenient use in new and legacy software, and offers interfaces in C99 (International Standards Organisation, 1999), Fortran77 (ANSI, 1978), Python (Python, 2021), Julia (Bezanson et al., 2017), and Rust (Rust, 2021). Users and library developers can integrate libCEED at a low level into existing applications in
place of existing matrix-vector products without significant refactoring of their own discretization infrastructure. Alternatively, users can utilize integrated libCEED support in MFEM (Anderson et al., 2020; MFEM, 2021).
In addition to supporting applications and discretization libraries, libCEED provides a platform for performance engineering and co-design, as well as an algebraic interface for solvers research like adaptive p-multigrid, much like how sparse matrix libraries enable development and deployment of algebraic multigrid solvers},
	keywords = {finite elements, high-order methods, High-performance computing, matrix-free, spectral elements},
	doi = {10.21105/joss.02945},
	url = {https://doi.org/10.21105/joss.02945},
	author = {Jed Brown and Ahmad Abdelfattah and Valeria Barra and Natalie Beams and Jean-Sylvain Camier and Veselin Dobrev and Yohann Dudouit and Leila Ghaffari and Tzanio Kolev and David Medina and Will Pazner and Thilina Ratnayaka and Jeremy Thompson and Stanimire Tomov}
}
@inproceedings {,
	title = {Scalability Issues in FFT Computation},
	journal = {International Conference on Parallel Computing Technologies},
	year = {2021},
	pages = {279{\textendash}287},
	publisher = {Springer},
	abstract = {The fast Fourier transform (FFT), is one the most important tools in mathematics, and it is widely required by several applications of science and engineering. State-of-the-art parallel implementations of the FFT algorithm, based on Cooley-Tukey developments, are known to be communication-bound, which causes critical issues when scaling the computational and architectural capabilities. In this paper, we study the main performance bottleneck of FFT computations on hybrid CPU and GPU systems at large-scale. We provide numerical simulations and potential acceleration techniques that can be easily integrated into FFT distributed libraries. We present different experiments on performance scalability and runtime analysis on the world{\textquoteright}s most powerful supercomputers today: Summit, using up to 6,144 NVIDIA V100 GPUs, and Fugaku, using more than one million Fujitsu A64FX cores.},
	keywords = {Hybrid systems, Parallel FFT, scalability},
	isbn = {978-3-030-86359-3},
	doi = {10.1007/978-3-030-86359-3_21},
	author = {Alan Ayala and Stanimire Tomov and Stoyanov, Miroslav and Jack Dongarra}
}
@article {,
	title = {A Set of Batched Basic Linear Algebra Subprograms and LAPACK Routines},
	journal = {ACM Transactions on Mathematical Software (TOMS)},
	volume = {47},
	number = {3},
	year = {2021},
	pages = {1{\textendash}23},
	abstract = {This article describes a standard API for a set of Batched Basic Linear Algebra Subprograms (Batched BLAS or BBLAS). The focus is on many independent BLAS operations on small matrices that are grouped together and processed by a single routine, called a Batched BLAS routine. The matrices are grouped together in uniformly sized groups, with just one group if all the matrices are of equal size. The aim is to provide more efficient, but portable, implementations of algorithms on high-performance many-core platforms. These include multicore and many-core CPU processors, GPUs and coprocessors, and other hardware accelerators with floating-point compute facility. As well as the standard types of single and double precision, we also include half and quadruple precision in the standard. In particular, half precision is used in many very large scale applications, such as those associated with machine learning.},
	keywords = {Computations on matrices, Mathematical analysis, Mathematics of computing, Numerical analysis},
	doi = {10.1145/3431921},
	author = {Abdelfattah, Ahmad and Costa, Timothy and Jack Dongarra and Mark Gates and Haidar, Azzam and Hammarling, Sven and Higham, Nicholas J and Kurzak, Jakub and Piotr Luszczek and Stanimire Tomov and others}
}
@techreport {,
	title = {SLATE Performance Improvements: QR and Eigenvalues},
	journal = {SLATE Working Notes},
	number = {17, ICL-UT-21-02},
	year = {2021},
	month = {2021-04},
	author = {Kadir Akbudak and Paul Bagwell and Sebastien Cayrols and Mark Gates and Dalal Sukkari and Asim YarKhan and Jack Dongarra}
}
@techreport {,
	title = {SLATE Port to AMD and Intel Platforms},
	journal = {SLATE Working Notes},
	number = {16, ICL-UT-21-01},
	year = {2021},
	month = {2021-04},
	author = {Ahmad Abdelfattah and Mohammed Al Farhan and Cade Brown and Mark Gates and Dalal Sukkari and Asim YarKhan and Jack Dongarra}
}
@article {,
	title = {A survey of numerical linear algebra methods utilizing mixed-precision arithmetic},
	journal = {The International Journal of High Performance Computing Applications},
	volume = {35},
	number = {4},
	year = {2021},
	pages = {344{\textendash}369},
	abstract = {The efficient utilization of mixed-precision numerical linear algebra algorithms can offer attractive acceleration to scientific computing applications. Especially with the hardware integration of low-precision special-function units designed for machine learning applications, the traditional numerical algorithms community urgently needs to reconsider the floating point formats used in the distinct operations to efficiently leverage the available compute power. In this work, we provide a comprehensive survey of mixed-precision numerical linear algebra routines, including the underlying concepts, theoretical background, and experimental results for both dense and sparse linear algebra problems.},
	keywords = {GPUs, High-performance computing, linear algebra, Mixed-precision arithmetic, numerical mathematics},
	doi = {10.1177/10943420211003313},
	author = {Abdelfattah, Ahmad and Anzt, Hartwig and Boman, Erik G and Carson, Erin and Cojean, Terry and Jack Dongarra and Fox, Alyson and Mark Gates and Higham, Nicholas J and Li, Xiaoye S and others}
}
@techreport {,
	title = {CEED ECP Milestone Report: Improve Performance and Capabilities of CEED-Enabled ECP Applications on Summit/Sierra},
	journal = {ECP Milestone Reports},
	year = {2020},
	month = {2020-05},
	publisher = {Zenodo},
	doi = {https://doi.org/10.5281/zenodo.3860804},
	url = {https://doi.org/10.5281/zenodo.3860804},
	author = {Kolev, Tzanio and Fischer, Paul and Abdelfattah, Ahmad and Ananthan, Shreyas and Valeria Barra and Natalie Beams and Bleile, Ryan and Brown, Jed and Carson, Robert and Camier, Jean-Sylvain and Churchfield, Matthew and Dobrev, Veselin and Jack Dongarra and Dudouit, Yohann and Karakus, Ali and Kerkemeier, Stefan and Lan, YuHsiang and Medina, David and Merzari, Elia and Min, Misun and Parker, Scott and Ratnayaka, Thilina and Smith, Cameron and Sprague, Michael and Stitt, Thomas and Thompson, Jeremy and Tomboulides, Ananias and Stanimire Tomov and Tomov, Vladimir and Vargas, Arturo and Warburton, Tim and Weiss, Kenneth}
}
@article {,
	title = {Clover: Computational Libraries Optimized via Exascale Research},
	year = {2020},
	month = {2020-02},
	publisher = {2020 Exascale Computing Project Annual Meeting},
	address = {Houston, TX},
	author = {Mark Gates and Stanimire Tomov and Hartwig Anzt and Piotr Luszczek and Jack Dongarra}
}
@conference {,
	title = {Design, Optimization, and Benchmarking of Dense Linear Algebra Algorithms on AMD GPUs},
	booktitle = {2020 IEEE High Performance Extreme Computing Virtual Conference},
	year = {2020},
	month = {2020-09},
	publisher = {IEEE},
	organization = {IEEE},
	abstract = {Dense linear algebra (DLA) has historically been in the vanguard of software that must be adapted first to hardware changes. This is because DLA is both critical to the accuracy and performance of so many different types of applications, and because they have proved to be outstanding vehicles for finding and implementing solutions to the problems that novel architectures pose. Therefore, in this paper we investigate the portability of the MAGMA DLA library to the latest AMD GPUs. We use auto tools to convert the CUDA code in MAGMA to the HeterogeneousComputing Interface for Portability (HIP) language. MAGMA provides LAPACK for GPUs and benchmarks for fundamental DLA routines ranging from BLAS to dense factorizations, linear systems and eigen-problem solvers. We port these routines to HIP and quantify currently achievable performance through the MAGMA benchmarks for the main workload algorithms on MI25 and MI50 AMD GPUs. Comparison with performance roofline models and theoretical expectations are used to identify current limitations and directions for future improvements.},
	author = {Cade Brown and Ahmad Abdelfattah and Stanimire Tomov and Jack Dongarra}
}
@techreport {,
	title = {Design, Optimization, and Benchmarking of Dense Linear Algebra Algorithms on AMD GPUs},
	journal = {Innovative Computing Laboratory Technical Report},
	number = {ICL-UT-20-12},
	year = {2020},
	month = {2020-08},
	publisher = {University of Tennessee},
	abstract = {Dense linear algebra (DLA) has historically been in the vanguard of software that must be adapted first to hardware changes. This is because DLA is both critical to the accuracy and performance of so many different types of applications, and because they have proved to be outstanding vehicles for finding and implementing solutions to the problems that novel architectures pose. Therefore, in this paper we investigate the portability of the MAGMA DLA library to the latest AMD GPUs. We use auto tools to convert the CUDA code in MAGMA to the HeterogeneousComputing Interface for Portability (HIP) language. MAGMA provides LAPACK for GPUs and benchmarks for fundamental DLA routines ranging from BLAS to dense factorizations, linear systems and eigen-problem solvers. We port these routines to HIP and quantify currently achievable performance through the MAGMA benchmarks for the main workload algorithms on MI25 and MI50 AMD GPUs. Comparison with performance roofline models and theoretical expectations are used to identify current limitations and directions for future improvements.},
	keywords = {AMD GPUs, GPU computing, HIP Runtime, HPC, numerical linear algebra, Portability},
	author = {Cade Brown and Ahmad Abdelfattah and Stanimire Tomov and Jack Dongarra}
}
@article {,
	title = {Evaluating Asynchronous Schwarz Solvers on GPUs},
	journal = {International Journal of High Performance Computing Applications},
	year = {2020},
	month = {2020-08},
	abstract = {With the commencement of the exascale computing era, we realize that the majority of the leadership supercomputers are heterogeneous and massively parallel. Even a single node can contain multiple co-processors such as GPUs and multiple CPU cores. For example, ORNL{\textquoteright}s Summit accumulates six NVIDIA Tesla V100 GPUs and 42 IBM Power9 cores on each node. Synchronizing across compute resources of multiple nodes can be prohibitively expensive. Hence, it is necessary to develop and study asynchronous algorithms that circumvent this issue of bulk-synchronous computing. In this study, we examine the asynchronous version of the abstract Restricted Additive Schwarz method as a solver. We do not explicitly synchronize, but allow the communication between the sub-domains to be completely asynchronous, thereby removing the bulk synchronous nature of the algorithm.

We accomplish this by using the one-sided Remote Memory Access (RMA) functions of the MPI standard. We study the benefits of using such an asynchronous solver over its synchronous counterpart. We also study the communication patterns governed by the partitioning and the overlap between the sub-domains on the global solver. Finally, we show that this concept can render attractive performance benefits over the synchronous counterparts even for a well-balanced problem.},
	keywords = {abstract Schwarz methods, Asynchronous solvers, exascale, GPUs, multicore processors, parallel numerical linear algebra},
	doi = {https://doi.org/10.1177/1094342020946814},
	author = {Pratik Nayak and Terry Cojean and Hartwig Anzt}
}
@conference {,
	title = {Evaluating the Performance of NVIDIA{\textquoteright}s A100 Ampere GPU for Sparse and Batched Computations},
	booktitle = {2020 IEEE/ACM Workshop on Performance Modeling, Benchmarking and Simulation of High Performance Computer Systems (PMBS)},
	year = {2020},
	month = {2020-11},
	publisher = {IEEE},
	organization = {IEEE},
	abstract = {GPU accelerators have become an important backbone for scientific high performance-computing, and the performance advances obtained from adopting new GPU hardware are significant. In this paper we take a first look at NVIDIA{\textquoteright}s newest server-line GPU, the A100 architecture, part of the Ampere generation. Specifically, we assess its performance for sparse and batch computations, as these routines are relied upon in many scientific applications, and compare to the p},
	keywords = {Batched linear algebra, NVIDIA A100 GPU, sparse linear algebra, Sparse Matrix Vector Product},
	author = {Hartwig Anzt and Yuhsiang M. Tsai and Ahmad Abdelfattah and Terry Cojean and Jack Dongarra}
}
@conference {,
	title = {Extreme-Scale Task-Based Cholesky Factorization Toward Climate and Weather Prediction Applications},
	booktitle = {Platform for Advanced Scientific Computing Conference (PASC20)},
	year = {2020},
	month = {2020-06},
	publisher = {ACM},
	organization = {ACM},
	address = {Geneva, Switzerland},
	abstract = {Climate and weather can be predicted statistically via geospatial Maximum Likelihood Estimates (MLE), as an alternative to running large ensembles of forward models. The MLE-based iterative optimization procedure requires the solving of large-scale linear systems that performs a Cholesky factorization on a symmetric positive-definite covariance matrix---a demanding dense factorization in terms of memory footprint and computation. We propose a novel solution to this problem: at the mathematical level, we reduce the computational requirement by exploiting the data sparsity structure of the matrix off-diagonal tiles by means of low-rank approximations; and, at the programming-paradigm level, we integrate PaRSEC, a dynamic, task-based runtime to reach unparalleled levels of efficiency for solving extreme-scale linear algebra matrix operations. The resulting solution leverages fine-grained computations to facilitate asynchronous execution while providing a flexible data distribution to mitigate load imbalance. Performance results are reported using 3D synthetic datasets up to 42M geospatial locations on 130, 000 cores, which represent a cornerstone toward fast and accurate predictions of environmental applications.},
	doi = {https://doi.org/10.1145/3394277.3401846},
	author = {Qinglei Cao and Yu Pei and Kadir Akbudak and Aleksandr Mikhalev and George Bosilca and Hatem Ltaief and David Keyes and Jack Dongarra}
}
@techreport {1461,
	title = {FFT-ECP API and High-Performance Library Prototype for 2-D and 3-D FFTs on Large-Scale Heterogeneous Systems with GPUs},
	journal = {ECP Milestone Report},
	number = {FFT-ECP STML13-27},
	year = {2020},
	note = {revision 01-2020},
	month = {2020-01},
	publisher = {Innovative Computing Laboratory, University of Tennessee},
	type = {ECP WBS 2.3.3.13 Milestone Report},
	author = {Stanimire Tomov and Alan Ayala and Azzam Haidar and Jack Dongarra}
}
@article {,
	title = {Ginkgo: A High Performance Numerical Linear Algebra Library},
	journal = {Journal of Open Source Software},
	volume = {5},
	year = {2020},
	month = {2020-08},
	abstract = {Ginkgo is a production-ready sparse linear algebra library for high performance computing on GPU-centric architectures with a high level of performance portability and focuses on software sustainability.

The library focuses on solving sparse linear systems and accommodates a large variety of matrix formats, state-of-the-art iterative (Krylov) solvers and preconditioners, which make the library suitable for a variety of scientific applications. Ginkgo supports many architectures such as multi-threaded CPU, NVIDIA GPUs, and AMD GPUs. The heavy use of modern C++ features simplifies the addition of new executor paradigms and algorithmic functionality without introducing significant performance overhead.

Solving linear systems is usually one of the most computationally and memory intensive aspects of any application. Hence there has been a significant amount of effort in this direction with software libraries such as UMFPACK (Davis, 2004) and CHOLMOD (Chen, Davis, Hager, \& Rajamanickam, 2008) for solving linear systems with direct methods and PETSc (Balay et al., 2020), Trilinos ({\textquotedblleft}The Trilinos Project Website,{\textquotedblright} 2020), Eigen (Guennebaud, Jacob, \& others, 2010) and many more to solve linear systems with iterative methods. With Ginkgo, we aim to ensure high performance while not compromising portability. Hence, we provide very efficient low level kernels optimized for different architectures and separate these kernels from the algorithms thereby ensuring extensibility and ease of use.

Ginkgo is also a part of the xSDK effort (Bartlett et al., 2017) and available as a Spack (Gamblin et al., 2015) package. xSDK aims to provide infrastructure for and interoperability between a collection of related and complementary software elements to foster rapid and efficient development of scientific applications using High Performance Computing. Within this effort, we provide interoperability with application libraries such as deal.ii (Arndt et al., 2019) and mfem (Anderson et al., 2020). Ginkgo provides wrappers within these two libraries so that they can take advantage of the features of Ginkgo.},
	doi = {https://doi.org/10.21105/joss.02260},
	author = {Hartwig Anzt and Terry Cojean and Yen-Chen Chen and Fritz Goebel and Thomas Gruetzmacher and Pratik Nayak and Tobias Ribizel and Yu-Hsiang Tsai}
}
@article {,
	title = {Ginkgo: A Node-Level Sparse Linear Algebra Library for HPC (Poster)},
	year = {2020},
	month = {2020-02},
	publisher = {2020 Exascale Computing Project Annual Meeting},
	address = {Houston, TX},
	author = {Hartwig Anzt and Terry Cojean and Yen-Chen Chen and Fritz Goebel and Thomas Gruetzmacher and Pratik Nayak and Tobias Ribizel and Yu-Hsiang Tsai and Jack Dongarra}
}
@conference {1481,
	title = {heFFTe: Highly Efficient FFT for Exascale},
	booktitle = {International Conference on Computational Science (ICCS 2020)},
	year = {2020},
	month = {2020-06},
	address = {Amsterdam, Netherlands},
	abstract = {Exascale computing aspires to meet the increasing demands from large scientific applications. Software targeting exascale is typically designed for heterogeneous architectures; henceforth, it is not only important to develop well-designed software, but also make it aware of the hardware architecture and efficiently exploit its power. Currently, several and diverse applications, such as those part of the Exascale Computing Project (ECP) in the United States, rely on efficient computation of the Fast Fourier Transform (FFT). In this context, we present the design and implementation of heFFTe (Highly Efficient FFT for Exascale) library, which targets the upcoming exascale supercomputers. We provide highly (linearly) scalable GPU kernels that achieve more than 40{\texttimes} speedup with respect to local kernels from CPU state-of-the-art libraries, and over 2{\texttimes} speedup for the whole FFT computation. A communication model for parallel FFTs is also provided to analyze the bottleneck for large-scale problems. We show experiments obtained on Summit supercomputer at Oak Ridge National Laboratory, using up to 24,576 IBM Power9 cores and 6,144 NVIDIA V-100 GPUs.},
	keywords = {exascale, FFT, gpu, scalable algorithm},
	doi = {https://doi.org/10.1007/978-3-030-50371-0_19},
	author = {Alan Ayala and Stanimire Tomov and Azzam Haidar and Jack Dongarra}
}
@article {,
	title = {heFFTe: Highly Efficient FFT for Exascale (Poster)},
	year = {2020},
	month = {2020-10},
	publisher = {NVIDIA GPU Technology Conference (GTC2020)},
	author = {Alan Ayala and Stanimire Tomov and Azzam Haidar and Jack Dongarra}
}
@article {,
	title = {heFFTe: Highly Efficient FFT for Exascale (Poster)},
	year = {2020},
	month = {2020-02},
	publisher = {SIAM Conference on Parallel Processing for Scientific Computing (SIAM PP20)},
	address = {Seattle, WA},
	abstract = {Considered one of the top 10 algorithms of the 20th century, the Fast Fourier Transform (FFT) is widely used by applications in science and engineering. Large scale parallel applications targeting exascale, such as those part of the DOE Exascale Computing Project (ECP), are designed for heterogeneous architectures and, currently, more than a dozen ECP applications use FFTs in their codes. To address the applications needs, we developed the highly efficient FFTs for exascale (heFFTe) library. The heFFTe library release features very good weak and strong scalability and performance that is close to 90\% of the roofline peak performance. We present these performance results on the Summit supercomputer. heFFTe is also integrated in a number of applications and we present how the overall performance gets improved by using hFFTe. Performance model, limitations, and challenges are discussed for current and upcoming computer architectures.},
	author = {Alan Ayala and Stanimire Tomov and Azzam Haidar and Jack Dongarra}
}
@article {,
	title = {heFFTe: Highly Efficient FFT for Exascale (Poster)},
	year = {2020},
	month = {2020-02},
	publisher = {2020 Exascale Computing Project Annual Meeting},
	address = {Houston, TX},
	author = {Alan Ayala and Stanimire Tomov and Jack Dongarra and Azzam Haidar}
}
@conference {,
	title = {High-Order Finite Element Method using Standard and Device-Level Batch GEMM on GPUs},
	booktitle = {2020 IEEE/ACM 11th Workshop on Latest Advances in Scalable Algorithms for Large-Scale Systems (ScalA)},
	year = {2020},
	month = {2020-11},
	publisher = {IEEE},
	organization = {IEEE},
	abstract = {We present new GPU implementations of the tensor contractions arising from basis-related computations for highorder finite element methods. We consider both tensor and nontensor bases. In the case of tensor bases, we introduce new kernels based on a series of fused device-level matrix multiplications (GEMMs), specifically designed to utilize the fast memory of the GPU. For non-tensor bases, we develop a tuned framework for choosing standard batch-BLAS GEMMs that will maximize performance across groups of elements. The implementations are included in a backend of the libCEED library. We present benchmark results for the diffusion and mass operators using libCEED integration through the MFEM finite element library and compare to those of the previously best-performing GPU backends for stand-alone basis computations. In tensor cases, we see improvements of approximately 10-30\% for some cases, particularly for higher basis orders. For the non-tensor tests, the new batch-GEMMs implementation is twice as fast as what was previously available for basis function order greater than five and greater than approximately 105 degrees of freedom in the mesh; up to ten times speedup is seen for eighth-order basis functions.},
	keywords = {Batched linear algebra, finite elements, gpu, high-order methods, matrix-free FEM, Tensor contractions},
	author = {Natalie Beams and Ahmad Abdelfattah and Stanimire Tomov and Jack Dongarra and Tzanio Kolev and Yohann Dudouit}
}
@booklet {,
	title = {hipMAGMA v1.0},
	year = {2020},
	month = {2020-03},
	publisher = {Zenodo},
	doi = {10.5281/zenodo.3908549},
	url = {https://doi.org/10.5281/zenodo.3908549},
	author = {Cade Brown and Ahmad Abdelfattah and Stanimire Tomov and Jack Dongarra}
}
@booklet {,
	title = {hipMAGMA v2.0},
	year = {2020},
	month = {2020-07},
	publisher = {Zenodo},
	doi = {10.5281/zenodo.3928667},
	url = {https://doi.org/10.5281/zenodo.3928667},
	author = {Cade Brown and Ahmad Abdelfattah and Stanimire Tomov and Jack Dongarra}
}
@article {,
	title = {Integrating Deep Learning in Domain Science at Exascale (MagmaDNN)},
	year = {2020},
	month = {2020-12},
	publisher = {DOD HPCMP seminar},
	address = {virtual},
	abstract = {We will present some of the current challenges in the design and integration of deep learning AI with traditional HPC simulations. We evaluate existing packages for readiness to run efficiently deep learning models and applications on large scale HPC systems, identify challenges, and propose new asynchronous parallelization and optimization techniques for current large-scale heterogeneous systems and up-coming exascale systems. These developments, along with existing HPC AI software capabilities, have been integrated in MagmaDNN, an open source HPC deep learning framework. 
Many deep learning frameworks are targeted towards data scientists and fall short in providing quality integration into existing HPC workflows. This paper discusses the necessities of an HPC deep learning framework and how these can be provided, e.g., as in MagmaDNN, through a deep integration with existing HPC libraries such as MAGMA and its modular memory management, MPI, CuBLAS, CuDNN, MKL, and HIP. Advancements are also illustrated through the use of algorithmic enhancements in reduced and mixed-precision and asynchronous optimization methods. Finally, we present illustrations and potential solutions on enhancing traditional compute and data intensive applications at ORNL and UTK with AI. The approaches and future challenges are illustrated on materials science, imaging, and climate applications.},
	author = {Stanimire Tomov and Kwai Wong and Jack Dongarra and Rick Archibald and Edmond Chow and Eduardo D{\textquoteright}Azevedo and Markus Eisenbach and Rocco Febbo and Florent Lopez and Daniel Nichols and Junqi Yin}
}
@techreport {,
	title = {Integrating Deep Learning in Domain Sciences at Exascale},
	journal = {Innovative Computing Laboratory Technical Report},
	number = {ICL-UT-20-10},
	year = {2020},
	month = {2020-08},
	publisher = {University of Tennessee},
	abstract = {This paper presents some of the current challenges in designing deep learning artificial intelligence (AI) and integrating it with traditional high-performance computing (HPC) simulations. We evaluate existing packages for their ability to run deep learning models and applications on large-scale HPC systems e ciently, identify challenges, and propose new asynchronous parallelization and optimization techniques for current large-scale heterogeneous systems and upcoming exascale systems. These developments, along with existing HPC AI software capabilities, have been integrated into MagmaDNN, an open-source HPC deep learning framework. Many deep learning frameworks are targeted at data scientists and fall short in providing quality integration into existing HPC workflows. This paper discusses the necessities of an HPC deep learning framework and how those needs can be provided (e.g., as in MagmaDNN) through a deep integration with existing HPC libraries, such as MAGMA and its modular memory management, MPI, CuBLAS, CuDNN, MKL, and HIP. Advancements are also illustrated through the use of algorithmic enhancements in reduced- and mixed-precision, as well as asynchronous optimization methods. Finally, we present illustrations and potential solutions for enhancing traditional compute- and data-intensive applications at ORNL and UTK with AI. The approaches and future challenges are illustrated in materials science, imaging, and climate applications.},
	author = {Rick Archibald and Edmond Chow and Eduardo D{\textquoteright}Azevedo and Jack Dongarra and Markus Eisenbach and Rocco Febbo and Florent Lopez and Daniel Nichols and Stanimire Tomov and Kwai Wong and Junqi Yin}
}
@conference {,
	title = {Integrating Deep Learning in Domain Sciences at Exascale},
	booktitle = {2020 Smoky Mountains Computational Sciences and Engineering Conference (SMC 2020)},
	year = {2020},
	month = {2020-08},
	abstract = {This paper presents some of the current challenges in designing deep learning artificial intelligence (AI) and integrating it with traditional high-performance computing (HPC) simulations. We evaluate existing packages for their ability to run deep learning models and applications on large-scale HPC systems e ciently, identify challenges, and propose new asynchronous parallelization and optimization techniques for current large-scale heterogeneous systems and upcoming exascale systems. These developments, along with existing HPC AI software capabilities, have been integrated into MagmaDNN, an open-source HPC deep learning framework. Many deep learning frameworks are targeted at data scientists and fall short in providing quality integration into existing HPC workflows. This paper discusses the necessities of an HPC deep learning framework and how those needs can be provided (e.g., as in MagmaDNN) through a deep integration with existing HPC libraries, such as MAGMA and its modular memory management, MPI, CuBLAS, CuDNN, MKL, and HIP. Advancements are also illustrated through the use of algorithmic enhancements in reduced- and mixed-precision, as well as asynchronous optimization methods. Finally, we present illustrations and potential solutions for enhancing traditional compute- and data-intensive applications at ORNL and UTK with AI. The approaches and future challenges are illustrated in materials science, imaging, and climate applications.},
	author = {Rick Archibald and Edmond Chow and Eduardo D{\textquoteright}Azevedo and Jack Dongarra and Markus Eisenbach and Rocco Febbo and Florent Lopez and Daniel Nichols and Stanimire Tomov and Kwai Wong and Junqi Yin}
}
@inbook {1213,
	title = {Interoperable Convergence of Storage, Networking, and Computation},
	booktitle = {Advances in Information and Communication: Proceedings of the 2019 Future of Information and Communication Conference (FICC)},
	number = {2},
	year = {2020},
	pages = {667-690},
	publisher = {Springer International Publishing},
	organization = {Springer International Publishing},
	abstract = {In every form of digital store-and-forward communication, intermediate forwarding nodes are computers, with attendant memory and processing resources. This has inevitably stimulated efforts to create a wide-area infrastructure that goes beyond simple store-and-forward to create a platform that makes more general and varied use of the potential of this collection of increasingly powerful nodes. Historically, these efforts predate the advent of globally routed packet networking. The desire for a converged infrastructure of this kind has only intensified over the last 30 years, as memory, storage, and processing resources have increased in both density and speed while simultaneously decreasing in cost. Although there is a general consensus that it should be possible to define and deploy such a dramatically more capable wide-area platform, a great deal of investment in research prototypes has yet to produce a credible candidate architecture. Drawing on technical analysis, historical examples, and case studies, we present an argument for the hypothesis that in order to realize a distributed system with the kind of convergent generality and deployment scalability that might qualify as "future-defining," we must build it from a small set of simple, generic, and limited abstractions of the low level resources (processing, storage and network) of its intermediate nodes.},
	keywords = {active networks, distributed cloud, distributed processing, distributed storage, edge computing, network convergence, network layering, scalability},
	isbn = {978-3-030-12385-7},
	author = {Micah Beck and Terry Moore and Piotr Luszczek and Anthony Danalis},
	editor = {Kohei Arai and Rahul Bhatia}
}
@conference {1480,
	title = {Investigating the Benefit of FP16-Enabled Mixed-Precision Solvers for  Symmetric Positive Definite Matrices using GPUs},
	booktitle = {International Conference on Computational Science (ICCS 2020)},
	year = {2020},
	month = {2020-06},
	publisher = {Springer, Cham},
	organization = {Springer, Cham},
	address = {Amsterdam, Netherlands},
	abstract = {Half-precision computation refers to performing floating-point operations in a 16-bit format. While half-precision has been driven largely by machine learning applications, recent algorithmic advances in numerical linear algebra have discovered beneficial use cases for half precision in accelerating the solution of linear systems of equations at higher precisions. In this paper, we present a high-performance, mixed-precision linear solver (Ax = b) for symmetric positive definite systems in double-precision using graphics processing units (GPUs). The solver is based on a mixed-precision Cholesky factorization that utilizes the high-performance tensor core units in CUDA-enabled GPUs. Since the Cholesky factors are affected by the low precision, an iterative refinement (IR) solver is required to recover the solution back to double-precision accuracy. Two different types of IR solvers are discussed on a wide range of test matrices. A preprocessing step is also developed, which scales and shifts the matrix, if necessary, in order to preserve its positive-definiteness in lower precisions. Our experiments on the V100 GPU show that performance speedups are up to 4.7{\texttimes} against a direct double-precision solver. However, matrix properties such as the condition number and the eigenvalue distribution can affect the convergence rate, which would consequently affect the overall performance.},
	doi = {https://doi.org/10.1007/978-3-030-50417-5_18},
	author = {Ahmad Abdelfattah and Stanimire Tomov and Jack Dongarra}
}
@article {,
	title = {Load-Balancing Sparse Matrix Vector Product Kernels on GPUs},
	journal = {ACM Transactions on Parallel Computing},
	volume = {7},
	year = {2020},
	month = {2020-03},
	abstract = {Efficient processing of Irregular Matrices on Single Instruction, Multiple Data (SIMD)-type architectures is a persistent challenge. Resolving it requires innovations in the development of data formats, computational techniques, and implementations that strike a balance between thread divergence, which is inherent for Irregular Matrices, and padding, which alleviates the performance-detrimental thread divergence but introduces artificial overheads. To this end, in this article, we address the challenge of designing high performance sparse matrix-vector product (SpMV) kernels designed for Nvidia Graphics Processing Units (GPUs). We present a compressed sparse row (CSR) format suitable for unbalanced matrices. We also provide a load-balancing kernel for the coordinate (COO) matrix format and extend it to a hybrid algorithm that stores part of the matrix in SIMD-friendly Ellpack format (ELL) format. The ratio between the ELL- and the COO-part is determined using a theoretical analysis of the nonzeros-per-row distribution. For the over 2,800 test matrices available in the Suite Sparse matrix collection, we compare the performance against SpMV kernels provided by NVIDIA{\textquoteright}s cuSPARSE library and a heavily-tuned sliced ELL (SELL-P) kernel that prevents unnecessary padding by considering the irregular matrices as a combination of matrix blocks stored in ELL format.},
	doi = {https://doi.org/10.1145/3380930},
	author = {Hartwig Anzt and Terry Cojean and Chen Yen-Chen and Jack Dongarra and Goran Flegar and Pratik Nayak and Stanimire Tomov and Yuhsiang M. Tsai and Weichung Wang}
}
@article {,
	title = {MAGMA Templates for Scalable Linear Algebra on Emerging Architectures},
	journal = {The International Journal of High Performance Computing Applications},
	volume = {34},
	year = {2020},
	month = {2020-11},
	pages = {645-658},
	abstract = {With the acquisition and widespread use of more resources that rely on accelerator/wide vector{\textendash}based computing, there has been a strong demand for science and engineering applications to take advantage of these latest assets. This, however, has been extremely challenging due to the diversity of systems to support their extreme concurrency, complex memory hierarchies, costly data movement, and heterogeneous node architectures. To address these challenges, we design a programming model and describe its ease of use in the development of a new MAGMA Templates library that delivers high-performance scalable linear algebra portable on current and emerging architectures. MAGMA Templates derives its performance and portability by (1) building on existing state-of-the-art linear algebra libraries, like MAGMA, SLATE, Trilinos, and vendor-optimized math libraries, and (2) providing access (seamlessly to the users) to the latest algorithms and architecture-specific optimizations through a single, easy-to-use C++-based API.},
	issn = {1094-3420},
	doi = {https://doi.org/10.1177/1094342020938421},
	author = {Mohammed Al Farhan and Ahmad Abdelfattah and Stanimire Tomov and Mark Gates and Dalal Sukkari and Azzam Haidar and Robert Rosenberg and Jack Dongarra}
}
@article {,
	title = {Matrix Multiplication on Batches of Small Matrices in Half and Half-Complex Precisions},
	journal = {Journal of Parallel and Distributed Computing},
	volume = {145},
	year = {2020},
	month = {2020-11},
	pages = {188-201},
	abstract = {Machine learning and artificial intelligence (AI) applications often rely on performing many small matrix operations{\textemdash}in particular general matrix{\textendash}matrix multiplication (GEMM). These operations are usually performed in a reduced precision, such as the 16-bit floating-point format (i.e., half precision or FP16). The GEMM operation is also very important for dense linear algebra algorithms, and half-precision GEMM operations can be used in mixed-precision linear solvers. Therefore, high-performance batched GEMM operations in reduced precision are significantly important, not only for deep learning frameworks, but also for scientific applications that rely on batched linear algebra, such as tensor contractions and sparse direct solvers.

This paper presents optimized batched GEMM kernels for graphics processing units (GPUs) in FP16 arithmetic. The paper addresses both real and complex half-precision computations on the GPU. The proposed design takes advantage of the Tensor Core technology that was recently introduced in CUDA-enabled GPUs. With eight tuning parameters introduced in the design, the developed kernels have a high degree of flexibility that overcomes the limitations imposed by the hardware and software (in the form of discrete configurations for the Tensor Core APIs). For real FP16 arithmetic, performance speedups are observed against cuBLAS for sizes up to 128, and range between  and . For the complex FP16 GEMM kernel, the speedups are between  and  thanks to a design that uses the standard interleaved matrix layout, in contrast with the planar layout required by the vendor{\textquoteright}s solution. The paper also discusses special optimizations for extremely small matrices, where even higher performance gains are achievable.},
	doi = {https://doi.org/10.1016/j.jpdc.2020.07.001},
	author = {Ahmad Abdelfattah and Stanimire Tomov and Jack Dongarra}
}
@conference {,
	title = {Multiprecision Block-Jacobi for Iterative Triangular Solves},
	booktitle = {European Conference on Parallel Processing (Euro-Par 2020)},
	year = {2020},
	month = {2020-08},
	publisher = {Springer},
	organization = {Springer},
	abstract = {Recent research efforts have shown that Jacobi and block-Jacobi relaxation methods can be used as an effective and highly parallel approach for the solution of sparse triangular linear systems arising in the application of ILU-type preconditioners. Simultaneously, a few independent works have focused on designing efficient high performance adaptive-precision block-Jacobi preconditioning (block-diagonal scaling), in the context of the iterative solution of sparse linear systems, on manycore architectures. In this paper, we bridge the gap between relaxation methods based on regular splittings and preconditioners by demonstrating that iterative refinement can be leveraged to construct a relaxation method from the preconditioner. In addition, we exploit this insight to construct a highly-efficient sparse triangular system solver for graphics processors that combines iterative refinement with the block-Jacobi preconditioner available in the Ginkgo library.},
	keywords = {Block-Jacobi, graphics processing units (GPUs), incomplete factorization preconditioning, multiprecision, sparse linear algebra},
	doi = {https://doi.org/10.1007/978-3-030-57675-2_34},
	author = {Fritz Goebel and Hartwig Anzt and Terry Cojean and Goran Flegar and Enrique S. Quintana-Orti}
}
@conference {,
	title = {Scalable Data Generation for Evaluating Mixed-Precision Solvers},
	booktitle = {2020 IEEE High Performance Extreme Computing Conference (HPEC)},
	year = {2020},
	month = {2020-09},
	publisher = {IEEE},
	organization = {IEEE},
	address = { Waltham, MA, USA},
	abstract = {We present techniques of generating data for mixed precision solvers that allows to test those solvers in a scalable manner. Our techniques focus on mixed precision hardware and software where both the solver and the hardware can take advantage of mixing multiple floating precision formats. This allows taking advantage of recently released generation of hardware platforms that focus on ML and DNN workloads but can also be utilized for HPC applications if a new breed of algorithms is combined with the custom floating-point formats to deliver performance levels beyond the standard IEEE data types while delivering a comparable accuracy of the results.},
	doi = {https://doi.org/10.1109/HPEC43674.2020.9286145},
	author = {Piotr Luszczek and Yaohung Tsai and Neil Lindquist and Hartwig Anzt and Jack Dongarra}
}
@article {,
	title = {A Set of Batched Basic Linear Algebra Subprograms},
	journal = {ACM Transactions on Mathematical Software},
	year = {2020},
	month = {2020-10},
	abstract = {This paper describes a standard API for a set of Batched Basic Linear Algebra Subprograms (Batched BLAS or BBLAS). The focus is on many independent BLAS operations on small matrices that are grouped together and processed by a single routine, called a Batched BLAS routine. The matrices are grouped together in uniformly sized groups, with just one group if all the matrices are of equal size. The aim is to provide more efficient, but portable, implementations of algorithms on high-performance many-core platforms. These include multicore and many-core CPU processors, GPUs and coprocessors, and other hardware accelerators with floating-point compute facility. As well as the standard types of single and double precision, we also include half and quadruple precision in the standard. In particular half precision is used in many very large scale applications, such as those associated with machine learning.},
	author = {Ahmad Abdelfattah and Timothy Costa and Jack Dongarra and Mark Gates and Azzam Haidar and Sven Hammarling and Nicholas J. Higham and Jakub Kurzak and Piotr Luszczek and Stanimire Tomov and Mawussi Zounon}
}
@conference {,
	title = {Sparse Linear Algebra on AMD and NVIDIA GPUs{\textemdash}The Race is On},
	booktitle = {ISC High Performance},
	year = {2020},
	month = {2020-06},
	publisher = {Springer},
	organization = {Springer},
	abstract = {Efficiently processing sparse matrices is a central and performance-critical part of many scientific simulation codes. Recognizing the adoption of manycore accelerators in HPC, we evaluate in this paper the performance of the currently best sparse matrix-vector product (SpMV) implementations on high-end GPUs from AMD and NVIDIA. Specifically, we optimize SpMV kernels for the CSR, COO, ELL, and HYB format taking the hardware characteristics of the latest GPU technologies into account. We compare for 2,800 test matrices the performance of our kernels against AMD{\textquoteright}s hipSPARSE library and NVIDIA{\textquoteright}s cuSPARSE library, and ultimately assess how the GPU technologies from AMD and NVIDIA compare in terms of SpMV performance.},
	keywords = {AMD, GPUs, nVidia, sparse matrix vector product (SpMV)},
	doi = {https://doi.org/10.1007/978-3-030-50743-5_16},
	author = {Yuhsiang M. Tsai and Terry Cojean and Hartwig Anzt}
}
@techreport {,
	title = {A Survey of Numerical Methods Utilizing Mixed Precision Arithmetic},
	journal = {SLATE Working Notes},
	number = {15, ICL-UT-20-08},
	year = {2020},
	month = {2020-07},
	publisher = {University of Tennessee},
	type = {SLATE Working Notes},
	author = {Ahmad Abdelfattah and Hartwig Anzt and Erik Boman and Erin Carson and Terry Cojean and Jack Dongarra and Mark Gates and Thomas Gruetzmacher and Nicholas J. Higham and Sherry Li and Neil Lindquist and Yang Liu and Jennifer Loe and Piotr Luszczek and Pratik Nayak and Sri Pranesh and Siva Rajamanickam and Tobias Ribizel and Barry Smith and Kasia Swirydowicz and Stephen Thomas and Stanimire Tomov and Yaohung Tsai and Ichitaro Yamazaki and Urike Meier Yang}
}
@conference {,
	title = {Task Bench: A Parameterized Benchmark for Evaluating Parallel Runtime Performance},
	booktitle = {International Conference for High Performance Computing Networking, Storage, and Analysis (SC20)},
	year = {2020},
	month = {2020-11},
	publisher = {ACM},
	organization = {ACM},
	abstract = {We present Task Bench, a parameterized benchmark designed to explore the performance of distributed programming systems under a variety of application scenarios. Task Bench dramatically lowers the barrier to benchmarking and comparing multiple programming systems by making the implementation for a given system orthogonal to the benchmarks themselves: every benchmark constructed with Task Bench runs on every Task Bench implementation. Furthermore, Task Bench{\textquoteright}s parameterization enables a wide variety of benchmark scenarios that distill the key characteristics of larger applications.

To assess the effectiveness and overheads of the tested systems, we introduce a novel metric, minimum effective task granularity (METG). We conduct a comprehensive study with 15 programming systems on up to 256 Haswell nodes of the Cori supercomputer. Running at scale, 100μs-long tasks are the finest granularity that any system runs efficiently with current technologies. We also study each system{\textquoteright}s scalability, ability to hide communication and mitigate load imbalance.},
	url = {https://dl.acm.org/doi/10.5555/3433701.3433783},
	author = {Elliott Slaughter and Wei Wu and Yuankun Fu and Legend Brandenburg and Nicolai Garcia and Wilhem Kautz and Emily Marx and Kaleb S. Morris and Qinglei Cao and George Bosilca and Seema Mirchandaney and Wonchan Lee and Sean Treichler and Patrick McCormick and Alex Aiken}
}
@article {1319,
	title = {Adaptive Precision in Block-Jacobi Preconditioning for Iterative Sparse Linear System Solvers},
	journal = {Concurrency and Computation: Practice and Experience},
	volume = {31},
	number = {6},
	year = {2019},
	month = {2019-03},
	pages = {e4460},
	abstract = {Summary We propose an adaptive scheme to reduce communication overhead caused by data movement by selectively storing the diagonal blocks of a block-Jacobi preconditioner in different precision formats (half, single, or double). This specialized preconditioner can then be combined with any Krylov subspace method for the solution of sparse linear systems to perform all arithmetic in double precision. We assess the effects of the adaptive precision preconditioner on the iteration count and data transfer cost of a preconditioned conjugate gradient solver. A preconditioned conjugate gradient method is, in general, a memory bandwidth-bound algorithm, and therefore its execution time and energy consumption are largely dominated by the costs of accessing the problem{\textquoteright}s data in memory. Given this observation, we propose a model that quantifies the time and energy savings of our approach based on the assumption that these two costs depend linearly on the bit length of a floating point number. Furthermore, we use a number of test problems from the SuiteSparse matrix collection to estimate the potential benefits of the adaptive block-Jacobi preconditioning scheme.},
	keywords = {adaptive precision, block-Jacobi preconditioning, communication reduction, energy efficiency, Krylov subspace methods, sparse linear systems},
	doi = {https://doi.org/10.1002/cpe.4460},
	url = {https://onlinelibrary.wiley.com/doi/abs/10.1002/cpe.4460},
	author = {Hartwig Anzt and Jack Dongarra and Goran Flegar and Nicholas J. Higham and Enrique S. Quintana-Orti}
}
@article {1262,
	title = {Algorithms and Optimization Techniques for High-Performance Matrix-Matrix Multiplications of Very Small Matrices},
	journal = {Parallel Computing},
	volume = {81},
	year = {2019},
	month = {2019-01},
	pages = {1{\textendash}21},
	abstract = {Expressing scientific computations in terms of BLAS, and in particular the general dense matrix-matrix multiplication (GEMM), is of fundamental importance for obtaining high performance portability across architectures. However, GEMMs for small matrices of sizes smaller than 32 are not sufficiently optimized in existing libraries. We consider the computation of many small GEMMs and its performance portability for a wide range of computer architectures, including Intel CPUs, ARM, IBM, Intel Xeon Phi, and GPUs. These computations often occur in applications like big data analytics, machine learning, high-order finite element methods (FEM), and others. The GEMMs are grouped together in a single batched routine. For these cases, we present algorithms and their optimization techniques that are specialized for the matrix sizes and architectures of interest. We derive a performance model and show that the new developments can be tuned to obtain performance that is within 90\% of the optimal for any of the architectures of interest. For example, on a V100 GPU for square matrices of size 32, we achieve an execution rate of about 1600 gigaFLOP/s in double-precision arithmetic, which is 95\% of the theoretically derived peak for this computation on a V100 GPU. We also show that these results outperform currently available state-of-the-art implementations such as vendor-tuned math libraries, including Intel MKL and NVIDIA CUBLAS, as well as open-source libraries like OpenBLAS and Eigen.},
	keywords = {Autotuning, Batched GEMM, HPC, Matrix-matrix product, optimization, Small matrices},
	doi = {https://doi.org/10.1016/j.parco.2018.10.003},
	author = {Ian Masliah and Ahmad Abdelfattah and Azzam Haidar and Stanimire Tomov and Marc Baboulin and Jo{\"e}l Falcou and Jack Dongarra}
}
@conference {1368,
	title = {Approximate and Exact Selection on GPUs},
	booktitle = {2019 IEEE International Parallel and Distributed Processing Symposium Workshops},
	year = {2019},
	month = {2019-05},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Rio de Janeiro, Brazil},
	abstract = {We present a novel algorithm for parallel selection on GPUs. The algorithm requires no assumptions on the input data distribution, and has a much lower recursion depth compared to many state-of-the-art algorithms. We implement the algorithm for different GPU generations, always using the respectively-available low-level communication features, and assess the performance on server-line hardware. The computational complexity of our SampleSelect algorithm is comparable to specialized algorithms designed for - and exploiting the characteristics of - "pleasant" data distributions. At the same time, as the SampleSelect does not work on the actual values but the ranks of the elements only, it is robust to the input data and can complete significantly faster for adversarial data distributions. Additionally to the exact SampleSelect, we address the use case of approximate selection by designing a variant that radically reduces the computational cost while preserving high approximation accuracy.},
	doi = {10.1109/IPDPSW.2019.00088},
	author = {Tobias Ribizel and Hartwig Anzt}
}
@conference {1367,
	title = {Are we Doing the Right Thing? {\textendash} A Critical Analysis of the Academic HPC Community},
	booktitle = {2019 IEEE International Parallel and Distributed Processing Symposium Workshops (IPDPSW)},
	year = {2019},
	month = {2019-05},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Rio de Janeiro, Brazil},
	abstract = {Like in any other research field, academically surviving in the High Performance Computing (HPC) community generally requires to publish papers, in the bast case many of them and in high-ranked journals or at top-tier conferences. As a result, the number of scientific papers published each year in this relatively small community easily outnumbers what a single researcher can read. At the same time, many of the proposed and analyzed strategies, algorithms, and hardware-optimized implementations never make it beyond the prototype stage, as they are abandoned once they served the single purpose of yielding (another) publication. In a time and field where high-quality manpower is a scarce resource, this is extremely inefficient. In this position paper we promote a radical paradigm shift towards accepting high-quality software patches to community software packages as legitimate conference contributions. In consequence, the reputation and appointability of researchers is no longer based on the classical scientific metrics, but on the quality and documentation of open source software contributions - effectively improving and accelerating the collaborative development of community software.},
	doi = {10.1109/IPDPSW.2019.00122},
	author = {Hartwig Anzt and Goran Flegar}
}
@techreport {1433,
	title = {CEED ECP Milestone Report: Performance Tuning of CEED Software and 1st and 2nd Wave Apps},
	year = {2019},
	month = {2019-10},
	publisher = {Zenodo},
	doi = {https://doi.org/10.5281/zenodo.3477618},
	author = {Stanimire Tomov and Ahmad Abdelfattah and Valeria Barra and Natalie Beams and Jed Brown and Jean-Sylvain Camier and Veselin Dobrev and Jack Dongarra and Yohann Dudouit and Paul Fischer and Ali Karakus and Stefan Kerkemeier and Tzanio Kolev and YuHsiang Lan and Elia Merzari and Misun Min and Aleks Obabko and Scott Parker and Thilina Ratnayaka and Jeremy Thompson and Ananias Tomboulides and Vladimir Tomov and Tim Warburton}
}
@techreport {1434,
	title = {CEED ECP Milestone Report: Public release of CEED 2.0},
	year = {2019},
	month = {2019-04},
	publisher = {Zenodo},
	doi = {10.5281/zenodo.2641316},
	url = {https://doi.org/10.5281/zenodo.2641316},
	author = {Jed Brown and Ahmad Abdelfattah and Valeria Barra and Veselin Dobrev and Yohann Dudouit and Paul Fischer and Tzanio Kolev and David Medina and Misun Min and Thilina Ratnayaka and Cameron Smith and Jeremy Thompson and Stanimire Tomov and Vladimir Tomov and Tim Warburton}
}
@article {1305,
	title = {Checkpointing Strategies for Shared High-Performance Computing Platforms},
	journal = {International Journal of Networking and Computing},
	volume = {9},
	number = {1},
	year = {2019},
	pages = {28{\textendash}52},
	abstract = {Input/output (I/O) from various sources often contend for scarcely available bandwidth. For example, checkpoint/restart (CR) protocols can help to ensure application progress in failure-prone environments. However, CR I/O alongside an application{\textquoteright}s normal, requisite I/O can increase I/O contention and might negatively impact performance. In this work, we consider different aspects (system-level scheduling policies and hardware) that optimize the overall performance of concurrently executing CR-based applications that share I/O resources. We provide a theoretical model and derive a set of necessary constraints to minimize the global waste on a given platform. Our results demonstrate that Young/Daly{\textquoteright}s optimal checkpoint interval, despite providing a sensible metric for a single, undisturbed application, is not sufficient to optimally address resource contention at scale. We show that by combining optimal checkpointing periods with contention-aware system-level I/O scheduling strategies, we can significantly improve overall application performance and maximize the platform throughput. Finally, we evaluate how specialized hardware, namely burst buffers, may help to mitigate the I/O contention problem. Overall, these results provide critical analysis and direct guidance on how to design efficient, CR ready, large -scale platforms without a large investment in the I/O subsystem.},
	issn = {2185-2847},
	url = {http://www.ijnc.org/index.php/ijnc/article/view/195},
	author = {Thomas Herault and Yves Robert and Aurelien Bouteiller and Dorian Arnold and Kurt Ferreira and George Bosilca and Jack Dongarra}
}
@techreport {1399,
	title = {A Collection of White Papers from the BDEC2 Workshop in Poznan, Poland},
	journal = {Innovative Computing Laboratory Technical Report},
	number = {ICL-UT-19-10},
	year = {2019},
	month = {2019-05},
	publisher = {University of Tennessee, Knoxville},
	author = {Gabriel Antoniu and Alexandru Costan and Ovidiu Marcu and Maria S. P{\'e}rez and Nenad Stojanovic and Rosa M. Badia and Miguel V{\'a}zquez and Sergi Girona and Micah Beck and Terry Moore and Piotr Luszczek and Ezra Kissel and Martin Swany and Geoffrey Fox and Vibhatha Abeykoon and Selahattin Akkas and Kannan Govindarajan and Gurhan Gunduz and Supun Kamburugamuve and Niranda Perera and Ahmet Uyar and Pulasthi Wickramasinghe and Chathura Widanage and Maria Girone and Toshihiro Hanawa and Richard Moreno and Ariel Oleksiak and Martin Swany and Ryousei Takano and M.P. van Haarlem and J. van Leeuwen and J.B.R. Oonk and T. Shimwell and L.V.E. Koopmans}
}
@techreport {1408,
	title = {A Collection of White Papers from the BDEC2 Workshop in San Diego, CA},
	journal = {Innovative Computing Laboratory Technical Report},
	number = {ICL-UT-19-13},
	year = {2019},
	month = {2019-10},
	publisher = {University of Tennessee},
	author = {Ilkay Altintas and Kyle Marcus and Volkan Vural and Shweta Purawat and Daniel Crawl and Gabriel Antoniu and Alexandru Costan and Ovidiu Marcu and Prasanna Balaprakash and Rongqiang Cao and Yangang Wang and Franck Cappello and Robert Underwood and Sheng Di and Justin M. Wozniak and Jon C. Calhoun and Cong Xu and Antonio Lain and Paolo Faraboschi and Nic Dube and Dejan Milojicic and Balazs Gerofi and Maria Girone and Viktor Khristenko and Tony Hey and Erza Kissel and Yu Liu and Richard Loft and Pekka Manninen and Sebastian von Alfthan and Takemasa Miyoshi and Bruno Raffin and Olivier Richard and Denis Trystram and Maryam Rahnemoonfar and Robin Murphy and Joel Saltz and Kentaro Sano and Rupak Roy and Kento Sato and Jian Guo and Jen s Domke and Weikuan Yu and Takaki Hatsui and Yasumasa Joti and Alex Szalay and William M. Tang and Michael R. Wyatt II and Michela Taufer and Todd Gamblin and Stephen Herbein and Adam Moody and Dong H. Ahn and Rich Wolski and Chandra Krintz and Fatih Bakir and Wei-tsung Lin and Gareth George}
}
@article {1313,
	title = {Co-Scheduling HPC Workloads on Cache-Partitioned CMP Platforms},
	journal = {International Journal of High Performance Computing Applications},
	volume = {33},
	year = {2019},
	month = {2019-11},
	pages = {1221-1239},
	abstract = {With the recent advent of many-core architectures such as chip multiprocessors (CMPs), the number of processing units accessing a global shared memory is constantly increasing. Co-scheduling techniques are used to improve application throughput on such architectures, but sharing resources often generates critical interferences. In this article, we focus on the interferences in the last level of cache (LLC) and use the Cache Allocation Technology (CAT) recently provided by Intel to partition the LLC and give each co-scheduled application their own cache area. We consider m iterative HPC applications running concurrently and answer to the following questions: (i) How to precisely model the behavior of these applications on the cache-partitioned platform? and (ii) how many cores and cache fractions should be assigned to each application to maximize the platform efficiency? Here, platform efficiency is defined as maximizing the performance either globally, or as guaranteeing a fixed ratio of iterations per second for each application. Through extensive experiments using CAT, we demonstrate the impact of cache partitioning when multiple HPC applications are co-scheduled onto CMP platforms.},
	keywords = {cache partitioning, chip multiprocessor, co-scheduling, HPC application},
	doi = {https://doi.org/10.1177/1094342019846956},
	author = {Guillaume Aupy and Anne Benoit and Brice Goglin and Lo{\"\i}c Pottier and Yves Robert}
}
@article {1369,
	title = {A Customized Precision Format Based on Mantissa Segmentation for Accelerating Sparse Linear Algebra},
	journal = {Concurrency and Computation: Practice and Experience},
	volume = {40319},
	year = {2019},
	month = {2019-01},
	issn = {1532-0626},
	doi = {https://doi.org/10.1002/cpe.5418},
	author = {Thomas Gruetzmacher and Terry Cojean and Goran Flegar and Fritz G{\"o}bel and Hartwig Anzt}
}
@techreport {1322,
	title = {Design and Implementation for FFT-ECP on Distributed Accelerated Systems},
	journal = {Innovative Computing Laboratory Technical Report},
	number = {ICL-UT-19-05},
	year = {2019},
	month = {2019-04},
	publisher = {University of Tennessee},
	type = {ECP WBS 2.3.3.09 Milestone Report},
	author = {Stanimire Tomov and Azzam Haidar and Alan Ayala and Daniel Schultz and Jack Dongarra}
}
@techreport {1396,
	title = {An Empirical View of SLATE Algorithms on Scalable Hybrid System},
	journal = {Innovative Computing Laboratory Technical Report},
	number = {ICL-UT-19-08},
	year = {2019},
	month = {2019-09},
	publisher = {University of Tennessee, Knoxville},
	author = {Asim YarKhan and Jakub Kurzak and Ahmad Abdelfattah and Jack Dongarra}
}
@conference {1323,
	title = {Fast Batched Matrix Multiplication for Small Sizes using Half Precision Arithmetic on GPUs},
	booktitle = {33rd IEEE International Parallel and Distributed Processing Symposium (IPDPS)},
	year = {2019},
	month = {2019-05},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Rio de Janeiro, Brazil},
	author = {Ahmad Abdelfattah and Stanimire Tomov and Jack Dongarra}
}
@article {1329,
	title = {FFT-ECP Fast Fourier Transform},
	year = {2019},
	month = {2019-01},
	publisher = {2019 ECP Annual Meeting (Research Poster)},
	address = {Houston, TX},
	author = {Stanimire Tomov and Azzam Haidar and Alan Ayala and Daniel Schultz and Jack Dongarra}
}
@techreport {1401,
	title = {FFT-ECP Implementation Optimizations and Features Phase},
	journal = {Innovative Computing Laboratory Technical Report},
	number = {ICL-UT-19-12},
	year = {2019},
	month = {2019-10},
	publisher = {University of Tennessee},
	author = {Stanimire Tomov and Azzam Haidar and Alan Ayala and Hejer Shaiek and Jack Dongarra}
}
@article {1385,
	title = {GPUDirect MPI Communications and Optimizations to Accelerate FFTs on Exascale Systems},
	journal = {EuroMPI{\textquoteright}19 Posters, Zurich, Switzerland},
	number = {icl-ut-19-06},
	year = {2019},
	month = {2019-09},
	publisher = {ICL},
	type = {Extended Abstract},
	abstract = {Fast Fourier transforms (FFTs) are used in applications ranging from molecular dynamics and spectrum estimation to machine learn- ing, fast convolution and correlation, signal modulation, wireless multimedia applications, and others. However, FFTs are memory bound, and therefore, to accelerate them, it is crucial to avoid and optimize the FFTs{\textquoteright} communications. To this end, we present a 3-D FFT design for distributed graphics processing unit (GPU) systems that: (1) efficiently uses GPUs{\textquoteright} high bandwidth, (2) reduces global communications algorithmically, when possible, and (3) employs GPUDirect technologies as well as MPI optimizations in the development of high-performance FFTs for large-scale GPU-accelerated systems. We show that these developments and optimizations lead to very good strong scalability and a performance that is close to 90\% of the theoretical peak.},
	keywords = {CUDA-Aware MPI, ECP, FFT, FFT-ECP, gpu, GPUDirect},
	author = {Hejer Shaiek and Stanimire Tomov and Alan Ayala and Azzam Haidar and Jack Dongarra}
}
@conference {1403,
	title = {Impacts of Multi-GPU MPI Collective Communications on Large FFT Computation},
	booktitle = {Workshop on Exascale MPI (ExaMPI) at SC19},
	year = {2019},
	month = {2019-11},
	address = {Denver, CO},
	keywords = {Collective MPI, Exascale applications, FFT, Heterogeneous systems, scalable},
	author = {Alan Ayala and Stanimire Tomov and Xi Luo and Hejer Shaiek and Azzam Haidar and George Bosilca and Jack Dongarra}
}
@conference {1373,
	title = {Massively Parallel Automated Software Tuning},
	booktitle = {48th International Conference on Parallel Processing (ICPP 2019)},
	year = {2019},
	month = {2019-08},
	publisher = {ACM Press},
	organization = {ACM Press},
	address = {Kyoto, Japan},
	abstract = {This article presents an implementation of a distributed autotuning engine developed as part of the Bench-testing OpenN Software Autotuning Infrastructure project. The system is geared towards performance optimization of computational kernels for graphics processing units, and allows for the deployment of vast autotuning sweeps to massively parallel machines. The software implements dynamic work scheduling to distributed-memory resources and takes advantage of multithreading for parallel compilation and dispatches kernel launches to multiple accelerators. This paper lays out the main design principles of the system and discusses the basic mechanics of the initial implementation. Preliminary performance results are presented, encountered challenges are discussed, and the future directions are outlined.},
	doi = {https://doi.org/10.1145/3337821.3337908},
	author = {Jakub Kurzak and Yaohung Tsai and Mark Gates and Ahmad Abdelfattah and Jack Dongarra}
}
@conference {1327,
	title = {OpenDIEL: A Parallel Workflow Engine and DataAnalytics Framework},
	booktitle = {Practice and Experience in Advanced Research Computing (PEARC {\textquoteright}19)},
	year = {2019},
	month = {2019-07},
	publisher = {ACM},
	organization = {ACM},
	address = {Chicago, IL},
	author = {Frank Betancourt and Kwai Wong and Efosa Asemota and Quindell Marshall and Daniel Nichols and Stanimire Tomov}
}
@article {1374,
	title = {Optimizing Batch HGEMM on Small Sizes Using Tensor Cores},
	year = {2019},
	month = {2019-03},
	publisher = {GPU Technology Conference (GTC)},
	address = {San Jose, CA},
	author = {Ahmad Abdelfattah and Stanimire Tomov and Jack Dongarra}
}
@article {1377,
	title = {PAPI Software-Defined Events for in-Depth Performance Analysis},
	journal = {The International Journal of High Performance Computing Applications},
	volume = {33},
	year = {2019},
	month = {2019-11},
	pages = {1113-1127},
	abstract = {The methodology and standardization layer provided by the Performance Application Programming Interface (PAPI) has played a vital role in application profiling for almost two decades. It has enabled sophisticated performance analysis tool designers and performance-conscious scientists to gain insights into their applications by simply instrumenting their code using a handful of PAPI functions that {\textquotedblleft}just work{\textquotedblright} across different hardware components. In the past, PAPI development had focused primarily on hardware-specific performance metrics. However, the rapidly increasing complexity of software infrastructure poses new measurement and analysis challenges for the developers of large-scale applications. In particular, acquiring information regarding the behavior of libraries and runtimes{\textemdash}used by scientific applications{\textemdash}requires low-level binary instrumentation, or APIs specific to each library and runtime. No uniform API for monitoring events that originate from inside the software stack has emerged. In this article, we present our efforts to extend PAPI{\textquoteright}s role so that it becomes the de facto standard for exposing performance-critical events, which we refer to as software-defined events (SDEs), from different software layers. Upgrading PAPI with SDEs enables monitoring of both types of performance events{\textemdash}hardware- and software-related events{\textemdash}in a uniform way, through the same consistent PAPI. The goal of this article is threefold. First, we motivate the need for SDEs and describe our design decisions regarding the functionality we offer through PAPI{\textquoteright}s new SDE interface. Second, we illustrate how SDEs can be utilized by different software packages, specifically, by showcasing their use in the numerical linear algebra library MAGMA-Sparse, the tensor algebra library TAMM that is part of the NWChem suite, and the compiler-based performance analysis tool Byfl. Third, we provide a performance analysis of the overhead that results from monitoring SDEs and discuss the trade-offs between overhead and functionality.},
	url = {https://doi.org/10.1177/1094342019846287},
	author = {Heike Jagode and Anthony Danalis and Hartwig Anzt and Jack Dongarra}
}
@article {1437,
	title = {Parallel Selection on GPUs},
	journal = {Parallel Computing},
	volume = {91},
	year = {2019},
	month = {2020-03},
	abstract = {We present a novel parallel selection algorithm for GPUs capable of handling single rank selection (single selection) and multiple rank selection (multiselection). The algorithm requires no assumptions on the input data distribution, and has a much lower recursion depth compared to many state-of-the-art algorithms. We implement the algorithm for different GPU generations, always leveraging the respectively-available low-level communication features, and assess the performance on server-line hardware. The computational complexity of our SampleSelect algorithm is comparable to specialized algorithms designed for {\textendash} and exploiting the characteristics of {\textendash} {\textquotedblleft}pleasant{\textquotedblright} data distributions. At the same time, as the proposed SampleSelect algorithm does not work on the actual element values but on the element ranks of the elements only, it is robust to the input data and can complete significantly faster for adversarial data distributions. We also address the use case of approximate selection by designing a variant that radically reduces the computational cost while preserving high approximation accuracy.},
	keywords = {approximate selection, gpu, kth order statistics, multiselection, parallel selection algorithm},
	doi = {https://doi.org/10.1016/j.parco.2019.102588},
	url = {https://www.sciencedirect.com/science/article/pii/S0167819119301796},
	author = {Tobias Ribizel and Hartwig Anzt}
}
@conference {1436,
	title = {ParILUT {\textendash} A Parallel Threshold ILU for GPUs},
	booktitle = {IEEE International Parallel and Distributed Processing Symposium (IPDPS)},
	year = {2019},
	month = {2019-05},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Rio de Janeiro, Brazil},
	abstract = {In this paper, we present the first algorithm for computing threshold ILU factorizations on GPU architectures. The proposed ParILUT-GPU algorithm is based on interleaving parallel fixed-point iterations that approximate the incomplete factors for an existing nonzero pattern with a strategy that dynamically adapts the nonzero pattern to the problem characteristics. This requires the efficient selection of thresholds that separate the values to be dropped from the incomplete factors, and we design a novel selection algorithm tailored towards GPUs. All components of the ParILUT-GPU algorithm make heavy use of the features available in the latest NVIDIA GPU generations, and outperform existing multithreaded CPU implementations.},
	doi = {https://doi.org/10.1109/IPDPS.2019.00033},
	author = {Hartwig Anzt and Tobias Ribizel and Goran Flegar and Edmond Chow and Jack Dongarra}
}
@conference {1452,
	title = {Performance Analysis of Tile Low-Rank Cholesky Factorization Using PaRSEC Instrumentation Tools},
	booktitle = {Workshop on Programming and Performance Visualization Tools (ProTools 19) at SC19},
	year = {2019},
	month = {2019-11},
	publisher = {ACM},
	organization = {ACM},
	address = {Denver, CO},
	author = {Qinglei Cao and Yu Pei and Thomas Herault and Kadir Akbudak and Aleksandr Mikhalev and George Bosilca and Hatem Ltaief and David Keyes and Jack Dongarra}
}
@article {1270,
	title = {PLASMA: Parallel Linear Algebra Software for Multicore Using OpenMP},
	journal = {ACM Transactions on Mathematical Software},
	volume = {45},
	year = {2019},
	month = {2019-06},
	doi = {https://doi.org/10.1145/3264491},
	author = {Jack Dongarra and Mark Gates and Azzam Haidar and Jakub Kurzak and Piotr Luszczek and Panruo Wu and Ichitaro Yamazaki and Asim YarKhan and Maksims Abalenkovs and Negin Bagherpour and Sven Hammarling and Jakub Sistek}
}
@conference {1376,
	title = {Progressive Optimization of Batched LU Factorization on GPUs},
	booktitle = { IEEE High Performance Extreme Computing Conference (HPEC{\textquoteright}19)},
	year = {2019},
	month = {2019-09},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Waltham, MA},
	author = {Ahmad Abdelfattah and Stanimire Tomov and Jack Dongarra}
}
@conference {1316,
	title = {Reservation Strategies for Stochastic Jobs},
	booktitle = {33rd IEEE International Parallel and Distributed Processing Symposium (IPDPS 2019)},
	year = {2019},
	month = {2019-05},
	publisher = {IEEE Computer Society Press},
	organization = {IEEE Computer Society Press},
	address = {Rio de Janeiro, Brazil},
	author = {Guillaume Aupy and Ana Gainaru and Valentin Honor{\'e} and Padma Raghavan and Yves Robert and Hongyang Sun}
}
@article {1317,
	title = {Toward a Modular Precision Ecosystem for High-Performance Computing},
	journal = {The International Journal of High Performance Computing Applications},
	volume = {33},
	year = {2019},
	month = {2019-11},
	pages = {1069-1078},
	abstract = {With the memory bandwidth of current computer architectures being significantly slower than the (floating point) arithmetic performance, many scientific computations only leverage a fraction of the computational power in today{\textquoteright}s high-performance architectures. At the same time, memory operations are the primary energy consumer of modern architectures, heavily impacting the resource cost of large-scale applications and the battery life of mobile devices. This article tackles this mismatch between floating point arithmetic throughput and memory bandwidth by advocating a disruptive paradigm change with respect to how data are stored and processed in scientific applications. Concretely, the goal is to radically decouple the data storage format from the processing format and, ultimately, design a {\textquotedblleft}modular precision ecosystem{\textquotedblright} that allows for more flexibility in terms of customized data access. For memory-bounded scientific applications, dynamically adapting the memory precision to the numerical requirements allows for attractive resource savings. In this article, we demonstrate the potential of employing a modular precision ecosystem for the block-Jacobi preconditioner and the PageRank algorithm{\textemdash}two applications that are popular in the communities and at the same characteristic representatives for the field of numerical linear algebra and data analytics, respectively.},
	keywords = {conjugate gradient, GPUs, Jacobi method, Modular precision, multicore processors, PageRank, parallel numerical linear algebra},
	issn = {1094-3420},
	doi = {https://doi.org/10.1177/1094342019846547},
	author = {Hartwig Anzt and Goran Flegar and Thomas Gruetzmacher and Enrique S. Quintana-Orti}
}
@article {1438,
	title = {Towards a New Peer Review Concept for Scientific Computing ensuring Technical Quality, Software Sustainability, and Result Reproducibility},
	journal = {Proceedings in Applied Mathematics and Mechanics},
	volume = {19},
	year = {2019},
	month = {2019-11},
	abstract = {In this position paper we argue for implementing an alternative peer review process for scientific computing contributions that promotes high quality scientific software developments as fully-recognized conference submission. The idea is based on leveraging the code reviewers{\textquoteright} feedback on scientific software contributions to community software developments as a third-party review involvement. Providing open access to this technical review would complement the scientific review of the contribution, efficiently reduce the workload of the undisclosed reviewers, improve the algorithm implementation quality and software sustainability, and ensure full reproducibility of the reported results. Using this process creates incentives to publish scientific algorithms in open source software {\textendash} instead of designing prototype algorithms with the unique purpose of publishing a paper. In addition, the comments and suggestions of the community being archived in the versioning control systems ensure that also community reviewers are receiving credit for the review contributions {\textendash} unlike reviewers in the traditional peer review process. Finally, it reflects the particularity of the scientific computing community using conferences rather than journals as the main publication venue.},
	issn = {1617-7061},
	doi = {https://doi.org/10.1002/pamm.201900490},
	author = {Hartwig Anzt and Terry Cojean and Eileen Kuhn}
}
@conference {1318,
	title = {Towards Continuous Benchmarking},
	booktitle = {Platform for Advanced Scientific Computing Conference (PASC 2019)},
	year = {2019},
	month = {2019-06},
	publisher = {ACM Press},
	organization = {ACM Press},
	address = {Zurich, Switzerland},
	abstract = {We present an automated performance evaluation framework that enables an automated workflow for testing and performance evaluation of software libraries. Integrating this component into an ecosystem enables sustainable software development, as a community effort, via a web application for interactively evaluating the performance of individual software components. The performance evaluation tool is based exclusively on web technologies, which removes the burden of downloading performance data or installing additional software. We employ this framework for the Ginkgo software ecosystem, but the framework can be used with essentially any software project, including the comparison between different software libraries. The Continuous Integration (CI) framework of Ginkgo is also extended to automatically run a benchmark suite on predetermined HPC systems, store the state of the machine and the environment along with the compiled binaries, and collect results in a publicly accessible performance data repository based on Git. The Ginkgo performance explorer (GPE) can be used to retrieve the performance data from the repository, and visualizes it in a web browser. GPE also implements an interface that allows users to write scripts, archived in a Git repository, to extract particular data, compute particular metrics, and visualize them in many different formats (as specified by the script). The combination of these approaches creates a workflow which enables performance reproducibility and software sustainability of scientific software. In this paper, we present example scripts that extract and visualize performance data for Ginkgo{\textquoteright}s SpMV kernels that allow users to identify the optimal kernel for specific problem characteristics.},
	isbn = {9781450367707},
	doi = {https://doi.org/10.1145/3324989.3325719},
	author = {Hartwig Anzt and Yen Chen Chen and Terry Cojean and Jack Dongarra and Goran Flegar and Pratik Nayak and Enrique S. Quintana-Orti and Yuhsiang M. Tsai and Weichung Wang}
}
@conference {1435,
	title = {Towards Half-Precision Computation for Complex Matrices: A Case Study for Mixed Precision Solvers on GPUs},
	booktitle = {ScalA19: 10th Workshop on Latest Advances in Scalable Algorithms for Large-Scale Systems},
	year = {2019},
	month = {2019-11},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Denver, CO},
	keywords = {Half precision, mixed-precision solvers, Tensor cores FP16 arithmetic},
	author = {Ahmad Abdelfattah and Stanimire Tomov and Jack Dongarra}
}
@article {1220,
	title = {Variable-Size Batched Gauss-Jordan Elimination for Block-Jacobi Preconditioning on Graphics Processors},
	journal = {Parallel Computing},
	volume = {81},
	year = {2019},
	month = {2019-01},
	pages = {131-146},
	abstract = {In this work, we address the efficient realization of block-Jacobi preconditioning on graphics processing units (GPUs). This task requires the solution of a collection of small and independent linear systems. To fully realize this implementation, we develop a variable-size batched matrix inversion kernel that uses Gauss-Jordan elimination (GJE) along with a variable-size batched matrix{\textendash}vector multiplication kernel that transforms the linear systems{\textquoteright} right-hand sides into the solution vectors. Our kernels make heavy use of the increased register count and the warp-local communication associated with newer GPU architectures. Moreover, in the matrix inversion, we employ an implicit pivoting strategy that migrates the workload (i.e., operations) to the place where the data resides instead of moving the data to the executing cores. We complement the matrix inversion with extraction and insertion strategies that allow the block-Jacobi preconditioner to be set up rapidly. The experiments on NVIDIA{\textquoteright}s K40 and P100 architectures reveal that our variable-size batched matrix inversion routine outperforms the CUDA basic linear algebra subroutine (cuBLAS) library functions that provide the same (or even less) functionality. We also show that the preconditioner setup and preconditioner application cost can be somewhat offset by the faster convergence of the iterative solver.},
	keywords = {Batched algorithms, Block-Jacobi, Gauss{\textendash}Jordan elimination, Graphics processor, matrix inversion, sparse linear systems},
	doi = {https://doi.org/10.1016/j.parco.2017.12.006},
	author = {Hartwig Anzt and Jack Dongarra and Goran Flegar and Enrique S. Quintana-Orti}
}
@techreport {1229,
	title = {Algorithms and Optimization Techniques for High-Performance Matrix-Matrix Multiplications of Very Small Matrices},
	journal = {Innovative Computing Laboratory Technical Report},
	number = {ICL-UT-18-09},
	year = {2018},
	month = {2018-09},
	publisher = {Innovative Computing Laboratory, University of Tennessee},
	abstract = {Expressing scientific computations in terms of BLAS, and in particular the general dense matrix-matrix multiplication (GEMM), is of fundamental importance for obtaining high performance portability across architectures. However, GEMMs for small matrices of sizes smaller than 32 are not sufficiently optimized in existing libraries. We consider the computation of many small GEMMs and its performance portability for a wide range of computer architectures, including Intel CPUs, ARM, IBM, Intel Xeon Phi, and GPUs. These computations often occur in applications like big data analytics, machine learning, high-order finite element methods (FEM), and others. The GEMMs are grouped together in a single batched routine. For these cases, we present algorithms and their optimization techniques that are specialized for the matrix sizes and architectures of interest. We derive a performance model and show that the new developments can be tuned to obtain performance that is within 90\% of the optimal for any of the architectures of interest. For example, on a V100 GPU for square matrices of size 32, we achieve an execution rate of about 1; 600 gigaFLOP/s in double-precision arithmetic, which is 95\% of the theoretically derived peak for this computation on a V100 GPU. We also show that these results outperform currently available state-of-the-art implementations such as vendor-tuned math libraries, including Intel MKL and NVIDIA CUBLAS, as well as open-source libraries like OpenBLAS and Eigen.},
	author = {Ian Masliah and Ahmad Abdelfattah and Azzam Haidar and Stanimire Tomov and Marc Baboulin and Jo{\"e}l Falcou and Jack Dongarra}
}
@article {1260,
	title = {Analysis and Design Techniques towards High-Performance and Energy-Efficient Dense Linear Solvers on GPUs},
	journal = {IEEE Transactions on Parallel and Distributed Systems},
	volume = {29},
	year = {2018},
	month = {2018-12},
	pages = {2700{\textendash}2712},
	abstract = {Graphics Processing Units (GPUs) are widely used in accelerating dense linear solvers. The matrix factorizations, which dominate the runtime for these solvers, are often designed using a hybrid scheme, where GPUs perform trailing matrix updates, while the CPUs perform the panel factorizations. Consequently, hybrid solutions require high-end CPUs and optimized CPU software in order to deliver high performance. Furthermore, they lack the energy efficiency inherent for GPUs due to the use of less energy-efficient CPUs, as well as CPU-GPU communications. This paper presents analysis and design techniques that overcome the shortcomings of the hybrid algorithms, and allow the design of high-performance and energy-efficient dense LU and Cholesky factorizations that use GPUs only. The full GPU solution eliminates the need for a high-end CPU and optimized CPU software, which leads to a better energy efficiency. We discuss different design choices, and introduce optimized GPU kernels for panel factorizations. The developed solutions achieve 90+ percent of the performance of optimized hybrid solutions, while improving the energy efficiency by 50 percent. They outperform the vendor library by 30-50 percent in single precision, and 15-50 percent in double precision. We also show that hybrid designs trail the proposed solutions in performance when optimized CPU software is not available.},
	keywords = {Dense linear solvers, energy efficiency, GPU computing},
	doi = {10.1109/TPDS.2018.2842785},
	author = {Ahmad Abdelfattah and Azzam Haidar and Stanimire Tomov and Jack Dongarra}
}
@conference {1195,
	title = {Analyzing Performance of BiCGStab with Hierarchical Matrix on GPU Clusters},
	booktitle = {IEEE International Parallel and Distributed Processing Symposium (IPDPS)},
	year = {2018},
	month = {2018-05},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Vancouver, BC, Canada},
	abstract = {ppohBEM is an open-source software package im- plementing the boundary element method. One of its main software tasks is the solution of the dense linear system of equations, for which, ppohBEM relies on another software package called HACApK. To reduce the cost of solving the linear system, HACApK hierarchically compresses the coefficient matrix using adaptive cross approximation. This hierarchical compression greatly reduces the storage and time complexities of the solver and enables the solution of large-scale boundary value problems. To extend the capability of ppohBEM, in this paper, we carefully port the HACApK{\textquoteright}s linear solver onto GPU clusters. Though the potential of the GPUs has been widely accepted in high-performance computing, it is still a challenge to utilize the GPUs for a solver, like HACApK{\textquoteright}s, that requires fine-grained computation and global communication. First, to utilize the GPUs, we integrate the batched GPU kernel that was recently released in the MAGMA software package. We discuss several techniques to improve the performance of the batched kernel. We then study various techniques to address the inter-GPU communication and study their effects on state-of- the-art GPU clusters. We believe that the techniques studied in this paper are of interest to a wide range of software packages running on GPUs, especially with the increasingly complex node architectures and the growing costs of the communication. We also hope that our efforts to integrate the GPU kernel or to setup the inter-GPU communication will influence the design of the future-generation batched kernels or the communication layer within a software stack.},
	author = {Ichitaro Yamazaki and Ahmad Abdelfattah and Akihiro Ida and Satoshi Ohshima and Stanimire Tomov and Rio Yokota and Jack Dongarra}
}
@article {1209,
	title = {Batched One-Sided Factorizations of Tiny Matrices Using GPUs: Challenges and Countermeasures},
	journal = {Journal of Computational Science},
	volume = {26},
	year = {2018},
	month = {2018-05},
	pages = {226{\textendash}236},
	abstract = {The use of batched matrix computations recently gained a lot of interest for applications, where the same operation is applied to many small independent matrices. The batched computational pattern is frequently encountered in applications of data analytics, direct/iterative solvers and preconditioners, computer vision, astrophysics, and more, and often requires specific designs for vectorization and extreme parallelism to map well on today{\textquoteright}s high-end many-core architectures. This has led to the development of optimized software for batch computations, and to an ongoing community effort to develop standard interfaces for batched linear algebra software. Furthering these developments, we present GPU design and optimization techniques for high-performance batched one-sided factorizations of millions of tiny matrices (of size 32 and less). We quantify the effects and relevance of different techniques in order to select the best-performing LU, QR, and Cholesky factorization designs. While we adapt common optimization techniques, such as optimal memory traffic, register blocking, and concurrency control, we also show that a different mindset and techniques are needed when matrices are tiny, and in particular, sub-vector/warp in size. The proposed routines are part of the MAGMA library and deliver significant speedups compared to their counterparts in currently available vendor-optimized libraries. Notably, we tune the developments for the newest V100 GPU from NVIDIA to show speedups of up to 11.8{\texttimes}.},
	keywords = {batch computation, GPU computing, matrix factorization},
	doi = {https://doi.org/10.1016/j.jocs.2018.01.005},
	author = {Ahmad Abdelfattah and Azzam Haidar and Stanimire Tomov and Jack Dongarra}
}
@article {1211,
	title = {Big Data and Extreme-Scale Computing: Pathways to Convergence - Toward a Shaping Strategy for a Future Software and Data Ecosystem for Scientific Inquiry},
	journal = {The International Journal of High Performance Computing Applications},
	volume = {32},
	year = {2018},
	month = {2018-07},
	pages = {435{\textendash}479},
	abstract = {Over the past four years, the Big Data and Exascale Computing (BDEC) project organized a series of five international workshops that aimed to explore the ways in which the new forms of data-centric discovery introduced by the ongoing revolution in high-end data analysis (HDA) might be integrated with the established, simulation-centric paradigm of the high-performance computing (HPC) community. Based on those meetings, we argue that the rapid proliferation of digital data generators, the unprecedented growth in the volume and diversity of the data they generate, and the intense evolution of the methods for analyzing and using that data are radically reshaping the landscape of scientific computing. The most critical problems involve the logistics of wide-area, multistage workflows that will move back and forth across the computing continuum, between the multitude of distributed sensors, instruments and other devices at the networks edge, and the centralized resources of commercial clouds and HPC centers. We suggest that the prospects for the future integration of technological infrastructures and research ecosystems need to be considered at three different levels. First, we discuss the convergence of research applications and workflows that establish a research paradigm that combines both HPC and HDA, where ongoing progress is already motivating efforts at the other two levels. Second, we offer an account of some of the problems involved with creating a converged infrastructure for peripheral environments, that is, a shared infrastructure that can be deployed throughout the network in a scalable manner to meet the highly diverse requirements for processing, communication, and buffering/storage of massive data workflows of many different scientific domains. Third, we focus on some opportunities for software ecosystem convergence in big, logically centralized facilities that execute large-scale simulations and models and/or perform large-scale data analytics. We close by offering some conclusions and recommendations for future investment and policy review.},
	doi = {https://doi.org/10.1177/1094342018778123},
	author = {Mark Asch and Terry Moore and Rosa M. Badia and Micah Beck and Pete Beckman and Thierry Bidot and Fran{\c c}ois Bodin and Franck Cappello and Alok Choudhary and Bronis R. de Supinski and Ewa Deelman and Jack Dongarra and Anshu Dubey and Geoffrey Fox and Haohuan Fu and Sergi Girona and Michael Heroux and Yutaka Ishikawa and Kate Keahey and David Keyes and William T. Kramer and Jean-Fran{\c c}ois Lavignon and Yutong Lu and Satoshi Matsuoka and Bernd Mohr and St{\'e}phane Requena and Joel Saltz and Thomas Schulthess and Rick Stevens and Martin Swany and Alexander Szalay and William Tang and Ga{\"e}l Varoquaux and Jean-Pierre Vilotte and Robert W. Wisniewski and Zhiwei Xu and Igor Zacharov}
}
@techreport {1397,
	title = {A Collection of White Papers from the BDEC2 Workshop in Bloomington, IN},
	journal = {Innovative Computing Laboratory Technical Report},
	number = {ICL-UT-18-15},
	year = {2018},
	month = {2018-11},
	publisher = {University of Tennessee, Knoxville},
	author = {James Ahrens and Christopher M. Biwer and Alexandru Costan and Gabriel Antoniu and Maria S. P{\'e}rez and Nenad Stojanovic and Rosa Badia and Oliver Beckstein and Geoffrey Fox and Shantenu Jha and Micah Beck and Terry Moore and Sunita Chandrasekaran and Carlos Costa and Thierry Deutsch and Luigi Genovese and Tarek El-Ghazawi and Ian Foster and Dennis Gannon and Toshihiro Hanawa and Tevfik Kosar and William Kramer and Madhav V. Marathe and Christopher L. Barrett and Takemasa Miyoshi and Alex Pothen and Ariful Azad and Judy Qiu and Bo Peng and Ravi Teja and Sahil Tyagi and Chathura Widanage and Jon Koskey and Maryam Rahnemoonfar and Umakishore Ramachandran and Miles Deegan and William Tang and Osamu Tatebe and Michela Taufer and Michel Cuende and Ewa Deelman and Trilce Estrada and Rafael Ferreira Da Silva and Harrel Weinstein and Rodrigo Vargas and Miwako Tsuji and Kevin G. Yager and Wanling Gao and Jianfeng Zhan and Lei Wang and Chunjie Luo and Daoyi Zheng and Xu Wen and Rui Ren and Chen Zheng and Xiwen He and Hainan Ye and Haoning Tang and Zheng Cao and Shujie Zhang and Jiahui Dai}
}
@article {1198,
	title = {Co-Scheduling Amdhal Applications on Cache-Partitioned Systems},
	journal = {International Journal of High Performance Computing Applications},
	volume = {32},
	year = {2018},
	month = {2018-01},
	pages = {123{\textendash}138},
	abstract = {Cache-partitioned architectures allow subsections of the shared last-level cache (LLC) to be exclusively reserved for some applications. This technique dramatically limits interactions between applications that are concurrently executing on a multicore machine. Consider n applications that execute concurrently, with the objective to minimize the makespan, defined as the maximum completion time of the n applications. Key scheduling questions are as follows: (i) which proportion of cache and (ii) how many processors should be given to each application? In this article, we provide answers to (i) and (ii) for Amdahl applications. Even though the problem is shown to be NP-complete, we give key elements to determine the subset of applications that should share the LLC (while remaining ones only use their smaller private cache). Building upon these results, we design efficient heuristics for Amdahl applications. Extensive simulations demonstrate the usefulness of co-scheduling when our efficient cache partitioning strategies are deployed.},
	keywords = {cache partitioning, co-scheduling, complexity results},
	doi = {https://doi.org/10.1177/1094342017710806},
	author = {Guillaume Aupy and Anne Benoit and Sicheng Dai and Lo{\"\i}c Pottier and Padma Raghavan and Yves Robert and Manu Shantharam}
}
@conference {1217,
	title = {Co-Scheduling HPC Workloads on Cache-Partitioned CMP Platforms},
	booktitle = {Cluster 2018},
	year = {2018},
	month = {2018-09},
	publisher = {IEEE Computer Society Press},
	organization = {IEEE Computer Society Press},
	address = {Belfast, UK},
	author = {Guillaume Aupy and Anne Benoit and Brice Goglin and Lo{\"\i}c Pottier and Yves Robert}
}
@inproceedings {1259,
	title = {The Design of Fast and Energy-Efficient Linear Solvers: On the Potential of Half-Precision Arithmetic and Iterative Refinement Techniques},
	journal = {International Conference on Computational Science (ICCS 2018)},
	volume = {10860},
	year = {2018},
	month = {2018-06},
	pages = {586{\textendash}600},
	publisher = {Springer},
	address = {Wuxi, China},
	abstract = {As parallel computers approach exascale, power efficiency in high-performance computing (HPC) systems is of increasing concern. Exploiting both the hardware features and algorithms is an effective solution to achieve power efficiency, and to address the energy constraints in modern and future HPC systems. In this work, we present a novel design and implementation of an energy-efficient solution for dense linear systems of equations, which are at the heart of large-scale HPC applications. The proposed energy-efficient linear system solvers are based on two main components: (1) iterative refinement techniques, and (2) reduced-precision computing features in modern accelerators and coprocessors. While most of the energy efficiency approaches aim to reduce the consumption with a minimal performance penalty, our method improves both the performance and the energy efficiency. Compared to highly-optimized linear system solvers, our kernels deliver the same accuracy solution up to   2{\texttimes}  faster and reduce the energy consumption up to half on Intel Knights Landing (KNL) architectures. By efficiently using the Tensor Cores available in the NVIDIA V100 PCIe GPUs, the speedups can be up to   4{\texttimes} , with more than 80\% reduction in the energy consumption.},
	doi = {https://doi.org/10.1007/978-3-319-93698-7_45},
	url = {https://rdcu.be/bcKSC},
	author = {Azzam Haidar and Ahmad Abdelfattah and Mawussi Zounon and Panruo Wu and Srikara Pranesh and Stanimire Tomov and Jack Dongarra}
}
@article {1208,
	title = {A Guide for Achieving High Performance with Very Small Matrices on GPUs: A Case Study of Batched LU and Cholesky Factorizations},
	journal = {IEEE Transactions on Parallel and Distributed Systems},
	volume = {29},
	year = {2018},
	month = {2018-05},
	pages = {973{\textendash}984},
	abstract = {We present a high-performance GPU kernel with a substantial speedup over vendor libraries for very small matrix computations. In addition, we discuss most of the challenges that hinder the design of efficient GPU kernels for small matrix algorithms. We propose relevant algorithm analysis to harness the full power of a GPU, and strategies for predicting the performance, before introducing a proper implementation. We develop a theoretical analysis and a methodology for high-performance linear solvers for very small matrices. As test cases, we take the Cholesky and LU factorizations and show how the proposed methodology enables us to achieve a performance close to the theoretical upper bound of the hardware. This work investigates and proposes novel algorithms for designing highly optimized GPU kernels for solving batches of hundreds of thousands of small-size Cholesky and LU factorizations. Our focus on efficient batched Cholesky and batched LU kernels is motivated by the increasing need for these kernels in scientific simulations (e.g., astrophysics applications). Techniques for optimal memory traffic, register blocking, and tunable concurrency are incorporated in our proposed design. The proposed GPU kernels achieve performance speedups versus CUBLAS of up to 6x for the factorizations, using double precision arithmetic on an NVIDIA Pascal P100 GPU.},
	doi = {10.1109/TPDS.2017.2783929},
	author = {Azzam Haidar and Ahmad Abdelfattah and Mawussi Zounon and Stanimire Tomov and Jack Dongarra}
}
@article {1335,
	title = {Harnessing GPU{\textquoteright}s Tensor Cores Fast FP16 Arithmetic to Speedup Mixed-Precision Iterative Refinement Solvers and Achieve 74 Gflops/Watt on Nvidia V100},
	year = {2018},
	month = {2018-03},
	publisher = {GPU Technology Conference (GTC), Poster},
	address = {San Jose, CA},
	author = {Azzam Haidar and Ahmad Abdelfattah and Stanimire Tomov and Jack Dongarra}
}
@conference {1235,
	title = {High-Performance GPU Implementation of PageRank with Reduced Precision based on Mantissa Segmentation},
	booktitle = {8th Workshop on Irregular Applications: Architectures and Algorithms},
	year = {2018},
	author = {Anzt, Hartwig and Thomas Gruetzmacher and Enrique S. Quintana-Orti and Scheidegger, Florian}
}
@techreport {1203,
	title = {Implementation of the C++ API for Batch BLAS},
	journal = {SLATE Working Notes},
	number = {07, ICL-UT-18-04},
	year = {2018},
	month = {2018-06},
	publisher = {Innovative Computing Laboratory, University of Tennessee},
	author = {Ahmad Abdelfattah and Mark Gates and Jakub Kurzak and Piotr Luszczek and Jack Dongarra}
}
@article {1158,
	title = {Incomplete Sparse Approximate Inverses for Parallel Preconditioning},
	journal = {Parallel Computing},
	volume = {71},
	year = {2018},
	month = {2018-01},
	pages = {1{\textendash}22},
	abstract = {In this paper, we propose a new preconditioning method that can be seen as a generalization of block-Jacobi methods, or as a simplification of the sparse approximate inverse (SAI) preconditioners. The {\textquotedblleft}Incomplete Sparse Approximate Inverses{\textquotedblright} (ISAI) is in particular efficient in the solution of sparse triangular linear systems of equations. Those arise, for example, in the context of incomplete factorization preconditioning. ISAI preconditioners can be generated via an algorithm providing fine-grained parallelism, which makes them attractive for hardware with a high concurrency level. In a study covering a large number of matrices, we identify the ISAI preconditioner as an attractive alternative to exact triangular solves in the context of incomplete factorization preconditioning.},
	issn = {01678191},
	doi = {10.1016/j.parco.2017.10.003},
	url = {http://www.sciencedirect.com/science/article/pii/S016781911730176X},
	author = {Hartwig Anzt and Thomas Huckle and J{\"u}rgen Br{\"a}ckle and Jack Dongarra}
}
@conference {1233,
	title = {A Jaccard Weights Kernel Leveraging Independent Thread Scheduling on GPUs},
	booktitle = {SBAC-PAD},
	year = {2018},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Lyon, France},
	url = {https://ieeexplore.ieee.org/document/8645946},
	author = {Anzt, Hartwig and Jack Dongarra}
}
@article {1330,
	title = {MATEDOR: MAtrix, TEnsor, and Deep-learning Optimized Routines},
	year = {2018},
	month = {2018-11},
	publisher = {The International Conference for High Performance Computing, Networking, Storage, and Analysis (SC18), Research Poster},
	address = {Dallas, TX},
	author = {Ahmad Abdelfattah and Jack Dongarra and Azzam Haidar and Stanimire Tomov and Ichitaro Yamazaki}
}
@article {1333,
	title = {MAtrix, TEnsor, and Deep-learning Optimized Routines (MATEDOR)},
	year = {2018},
	month = {2018-04},
	publisher = {NSF PI Meeting, Poster},
	address = {Washington, DC},
	doi = {https://doi.org/10.6084/m9.figshare.6174143.v3},
	author = {Azzam Haidar and Stanimire Tomov and Ahmad Abdelfattah and Ichitaro Yamazaki and Jack Dongarra}
}
@conference {1196,
	title = {Optimal Cooperative Checkpointing for Shared High-Performance Computing Platforms},
	booktitle = {2018 IEEE International Parallel and Distributed Processing Symposium Workshops (IPDPSW), Best Paper Award},
	year = {2018},
	month = {2018-05},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Vancouver, BC, Canada},
	abstract = {In high-performance computing environments, input/output (I/O) from various sources often contend for scarce available bandwidth. Adding to the I/O operations inherent to the failure-free execution of an application, I/O from checkpoint/restart (CR) operations (used to ensure progress in the presence of failures) place an additional burden as it increase I/O contention, leading to degraded performance. In this work, we consider a cooperative scheduling policy that optimizes the overall performance of concurrently executing CR-based applications which share valuable I/O resources. First, we provide a theoretical model and then derive a set of necessary constraints needed to minimize the global waste on the platform. Our results demonstrate that the optimal checkpoint interval, as defined by Young/Daly, despite providing a sensible metric for a single application, is not sufficient to optimally address resource contention at the platform scale. We therefore show that combining optimal checkpointing periods with I/O scheduling strategies can provide a significant improvement on the overall application performance, thereby maximizing platform throughput. Overall, these results provide critical analysis and direct guidance on checkpointing large-scale workloads in the presence of competing I/O while minimizing the impact on application performance.},
	doi = {10.1109/IPDPSW.2018.00127},
	author = {Thomas Herault and Yves Robert and Aurelien Bouteiller and Dorian Arnold and Kurt Ferreira and George Bosilca and Jack Dongarra}
}
@article {1219,
	title = {Optimization and Performance Evaluation of the IDR Iterative Krylov Solver on GPUs},
	journal = {The International Journal of High Performance Computing Applications},
	volume = {32},
	number = {2},
	year = {2018},
	month = {2018-03},
	pages = {220{\textendash}230},
	abstract = {In this paper, we present an optimized GPU implementation for the induced dimension reduction algorithm. We improve data locality, combine it with an efficient sparse matrix vector kernel, and investigate the potential of overlapping computation with communication as well as the possibility of concurrent kernel execution. A comprehensive performance evaluation is conducted using a suitable performance model. The analysis reveals efficiency of up to 90\%, which indicates that the implementation achieves performance close to the theoretically attainable bound.},
	keywords = {co-design, gpu, Induced dimension reduction (IDR), kernel fusion, kernel overlap, roofline performance model},
	doi = {https://doi.org/10.1177/1094342016646844},
	author = {Hartwig Anzt and Moritz Kreutzer and Eduardo Ponce and Gregory D. Peterson and Gerhard Wellein and Jack Dongarra}
}
@conference {1210,
	title = {Optimizing GPU Kernels for Irregular Batch Workloads: A Case Study for Cholesky Factorization},
	booktitle = {IEEE High Performance Extreme Computing Conference (HPEC{\textquoteright}18)},
	year = {2018},
	month = {2018-09},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Waltham, MA},
	abstract = {This paper introduces several frameworks for the design and implementation of high performance GPU kernels that target batch workloads with irregular sizes. Such workloads are ubiquitous in many scientific applications, including sparse direct solvers, astrophysics, and quantum chemistry. The paper addresses two main categories of frameworks, taking the Cholesky factorization as a case study. The first uses hostside kernel launches, and the second uses device-side launches. Within each category, different design options are introduced, with an emphasis on the advantages and the disadvantages of each approach. Our best performing design outperforms the state-of-the-art CPU implementation, scoring up to 4.7{\texttimes} speedup in double precision on a Pascal P100 GPU.},
	author = {Ahmad Abdelfattah and Azzam Haidar and Stanimire Tomov and Jack Dongarra}
}
@article {1190,
	title = {ParILUT - A New Parallel Threshold ILU},
	journal = {SIAM Journal on Scientific Computing},
	volume = {40},
	year = {2018},
	month = {2018-07},
	pages = {C503{\textendash}C519},
	publisher = {SIAM},
	abstract = {We propose a parallel algorithm for computing a threshold incomplete LU (ILU) factorization. The main idea is to interleave a parallel fixed-point iteration that approximates an incomplete factorization for a given sparsity pattern with a procedure that adjusts the pattern. We describe and test a strategy for identifying nonzeros to be added and nonzeros to be removed from the sparsity pattern. The resulting pattern may be different and more effective than that of existing threshold ILU algorithms. Also in contrast to other parallel threshold ILU algorithms, much of the new algorithm has fine-grained parallelism.},
	doi = {https://doi.org/10.1137/16M1079506},
	author = {Hartwig Anzt and Edmond Chow and Jack Dongarra}
}
@inbook {1261,
	title = {Scheduling for Fault-Tolerance: An Introduction},
	booktitle = {Topics in Parallel and Distributed Computing},
	year = {2018},
	pages = {143{\textendash}170},
	publisher = {Springer International Publishing},
	organization = {Springer International Publishing},
	isbn = { 978-3-319-93108-1},
	doi = {10.1007/978-3-319-93109-8},
	author = {Guillaume Aupy and Yves Robert}
}
@techreport {1275,
	title = {Software-Defined Events (SDEs) in MAGMA-Sparse},
	journal = {Innovative Computing Laboratory Technical Report},
	number = {ICL-UT-18-12},
	year = {2018},
	month = {2018-12},
	publisher = {University of Tennessee},
	author = {Heike Jagode and Anthony Danalis and Hartwig Anzt and Ichitaro Yamazaki and Mark Hoemmen and Erik Boman and Stanimire Tomov and Jack Dongarra}
}
@techreport {1204,
	title = {Solver Interface \& Performance on Cori},
	journal = {Innovative Computing Laboratory Technical Report},
	number = {ICL-UT-18-05},
	year = {2018},
	month = {2018-06},
	publisher = {University of Tennessee},
	author = {Hartwig Anzt and Ichitaro Yamazaki and Mark Hoemmen and Erik Boman and Jack Dongarra}
}
@article {1334,
	title = {Tensor Contractions using Optimized Batch GEMM Routines},
	year = {2018},
	month = {2018-03},
	publisher = {GPU Technology Conference (GTC), Poster},
	address = {San Jose, CA},
	author = {Ahmad Abdelfattah and Azzam Haidar and Stanimire Tomov and Jack Dongarra}
}
@conference {1265,
	title = {Using GPU FP16 Tensor Cores Arithmetic to Accelerate Mixed-Precision Iterative Refinement Solvers and Reduce Energy Consumption},
	booktitle = {ISC High Performance (ISC{\textquoteright}18), Best Poster},
	year = {2018},
	month = {2018-06},
	address = {Frankfurt, Germany},
	author = {Azzam Haidar and Stanimire Tomov and Ahmad Abdelfattah and Mawussi Zounon and Jack Dongarra}
}
@article {1332,
	title = {Using GPU FP16 Tensor Cores Arithmetic to Accelerate Mixed-Precision Iterative Refinement Solvers and Reduce Energy Consumption},
	year = {2018},
	month = {2018-06},
	publisher = {ISC High Performance (ISC18), Best Poster Award},
	address = {Frankfurt, Germany},
	author = {Azzam Haidar and Stanimire Tomov and Ahmad Abdelfattah and Mawussi Zounon and Jack Dongarra}
}
@article {1221,
	title = {Using Jacobi Iterations and Blocking for Solving Sparse Triangular Systems in Incomplete Factorization Preconditioning},
	journal = {Journal of Parallel and Distributed Computing},
	volume = {119},
	year = {2018},
	month = {2018-11},
	pages = {219{\textendash}230},
	abstract = {When using incomplete factorization preconditioners with an iterative method to solve large sparse linear systems, each application of the preconditioner involves solving two sparse triangular systems. These triangular systems are challenging to solve efficiently on computers with high levels of concurrency. On such computers, it has recently been proposed to use Jacobi iterations, which are highly parallel, to approximately solve the triangular systems from incomplete factorizations. The effectiveness of this approach, however, is problem-dependent: the Jacobi iterations may not always converge quickly enough for all problems. Thus, as a necessary and important step to evaluate this approach, we experimentally test the approach on a large number of realistic symmetric positive definite problems. We also show that by using block Jacobi iterations, we can extend the range of problems for which such an approach can be effective. For block Jacobi iterations, it is essential for the blocking to be cognizant of the matrix structure.},
	doi = {https://doi.org/10.1016/j.jpdc.2018.04.017},
	author = {Edmond Chow and Hartwig Anzt and Jennifer Scott and Jack Dongarra}
}
@conference {1234,
	title = {Variable-Size Batched Condition Number Calculation on GPUs},
	booktitle = {SBAC-PAD},
	year = {2018},
	month = {2018-09},
	address = {Lyon, France},
	url = {https://ieeexplore.ieee.org/document/8645907},
	author = {Hartwig Anzt and Jack Dongarra and Goran Flegar and Thomas Gruetzmacher}
}
@article {1341,
	title = {Accelerating Tensor Contractions in High-Order FEM with MAGMA Batched},
	year = {2017},
	month = {2017-03},
	publisher = {SIAM Conference on Computer Science and Engineering (SIAM CSE17), Presentation},
	address = {Atlanta, GA},
	author = {Ahmad Abdelfattah and Marc Baboulin and Veselin Dobrev and Jack Dongarra and Christopher Earl and Jo{\"e}l Falcou and Azzam Haidar and Ian Karlin and Tzanio Kolev and Ian Masliah and Stanimire Tomov}
}
@article {1176,
	title = {Argobots: A Lightweight Low-Level Threading and Tasking Framework},
	journal = {IEEE Transactions on Parallel and Distributed Systems},
	year = {2017},
	month = {2017-10},
	abstract = {In the past few decades, a number of user-level threading and tasking models have been proposed in the literature to address the shortcomings of OS-level threads, primarily with respect to cost and flexibility. Current state-of-the-art user-level threading and tasking models, however, are either too specific to applications or architectures or are not as powerful or flexible. In this paper, we present Argobots, a lightweight, low-level threading and tasking framework that is designed as a portable and performant substrate for high-level programming models or runtime systems. Argobots offers a carefully designed execution model that balances generality of functionality with providing a rich set of controls to allow specialization by the user or high-level programming model. We describe the design, implementation, and optimization of Argobots and present integrations with three example high-level models: OpenMP, MPI, and co-located I/O service. Evaluations show that (1) Argobots outperforms existing generic threading runtimes; (2) our OpenMP runtime offers more efficient interoperability capabilities than production OpenMP runtimes do; (3) when MPI interoperates with Argobots instead of Pthreads, it enjoys reduced synchronization costs and better latency hiding capabilities; and (4) I/O service with Argobots reduces interference with co-located applications, achieving performance competitive with that of the Pthreads version.},
	keywords = {Argobots, context switch, I/O, interoperability, lightweight, MPI, OpenMP, stackable scheduler, tasklet, user-level thread},
	doi = {10.1109/TPDS.2017.2766062},
	url = {http://ieeexplore.ieee.org/document/8082139/},
	author = {Sangmin Seo and Abdelhalim Amer and Pavan Balaji and Cyril Bordage and George Bosilca and Alex Brooks and Philip Carns and Adrian Castello and Damien Genet and Thomas Herault and Shintaro Iwasaki and Prateek Jindal and Sanjay Kale and Sriram Krishnamoorthy and Jonathan Lifflander and Huiwei Lu and Esteban Meneses and Mar Snir and Yanhua Sun and Kenjiro Taura and Pete Beckman}
}
@conference {1099,
	title = {Assuming failure independence: are we right to be wrong?},
	booktitle = {The 3rd International Workshop on Fault Tolerant Systems (FTS)},
	year = {2017},
	month = {2017-09},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Honolulu, Hawaii},
	abstract = {This paper revisits the failure temporal independence hypothesis which is omnipresent in the analysis of resilience methods for HPC. We explain why a previous approach is incorrect, and we propose a new method to detect failure cascades, i.e., series of non-independent consecutive failures. We use this new method to assess whether public archive failure logs contain failure cascades. Then we design and compare several cascadeaware checkpointing algorithms to quantify the maximum gain that could be obtained, and we report extensive simulation results with archive and synthetic failure logs. Altogether, there are a few logs that contain cascades, but we show that the gain that can be achieved from this knowledge is not significant. The conclusion is that we can wrongly, but safely, assume failure independence!},
	author = {Guillaume Aupy and Yves Robert and Frederic Vivien}
}
@inproceedings {998,
	title = {Batched Gauss-Jordan Elimination for Block-Jacobi Preconditioner Generation on GPUs},
	journal = {Proceedings of the 8th International Workshop on Programming Models and Applications for Multicores and Manycores},
	year = {2017},
	month = {2017-02},
	pages = {1{\textendash}10},
	publisher = {ACM},
	address = {New York, NY, USA},
	abstract = {In this paper, we design and evaluate a routine for the efficient generation of block-Jacobi preconditioners on graphics processing units (GPUs). Concretely, to exploit the architecture of the graphics accelerator, we develop a batched Gauss-Jordan elimination CUDA kernel for matrix inversion that embeds an implicit pivoting technique and handles the entire inversion process in the GPU registers. In addition, we integrate extraction and insertion CUDA kernels to rapidly set up the block-Jacobi preconditioner.

Our experiments compare the performance of our implementation against a sequence of batched routines from the MAGMA library realizing the inversion via the LU factorization with partial pivoting. Furthermore, we evaluate the costs of different strategies for the block-Jacobi extraction and insertion steps, using a variety of sparse matrices from the SuiteSparse matrix collection. Finally, we assess the efficiency of the complete block-Jacobi preconditioner generation in the context of an iterative solver applied to a set of computational science problems, and quantify its benefits over a scalar Jacobi preconditioner.},
	keywords = {block-Jacobi preconditioner, Gauss-Jordan elimination, graphics processing units (GPUs), iterative methods, matrix inversion, sparse linear systems},
	isbn = {978-1-4503-4883-6},
	doi = {10.1145/3026937.3026940},
	url = {http://doi.acm.org/10.1145/3026937.3026940},
	author = {Hartwig Anzt and Jack Dongarra and Goran Flegar and Enrique S. Quintana-Orti}
}
@techreport {1156,
	title = {BDEC Pathways to Convergence: Toward a Shaping Strategy for a Future Software and Data Ecosystem for Scientific Inquiry},
	journal = {Innovative Computing Laboratory Technical Report},
	number = {ICL-UT-17-08},
	year = {2017},
	month = {2017-11},
	publisher = {University of Tennessee},
	url = {http://www.exascale.org/bdec/report},
	editor = {Terry Moore and Mark Asch}
}
@inbook {1167,
	title = {Bringing High Performance Computing to Big Data Algorithms},
	booktitle = {Handbook of Big Data Technologies},
	year = {2017},
	publisher = {Springer},
	organization = {Springer},
	isbn = {978-3-319-49339-8},
	doi = {10.1007/978-3-319-49340-4},
	author = {Hartwig Anzt and Jack Dongarra and Mark Gates and Jakub Kurzak and Piotr Luszczek and Stanimire Tomov and Ichitaro Yamazaki}
}
@techreport {1175,
	title = {C++ API for Batch BLAS},
	journal = {SLATE Working Notes},
	number = {04, ICL-UT-17-12},
	year = {2017},
	month = {2017-12},
	publisher = {University of Tennessee},
	author = {Ahmad Abdelfattah and Konstantin Arturov and Cris Cecka and Jack Dongarra and Chip Freitag and Mark Gates and Azzam Haidar and Jakub Kurzak and Piotr Luszczek and Stanimire Tomov and Panruo Wu}
}
@techreport {1081,
	title = {C++ API for BLAS and LAPACK},
	journal = {SLATE Working Notes},
	number = {02, ICL-UT-17-03},
	year = {2017},
	note = {Revision 02-21-2018},
	month = {2017-06},
	publisher = {Innovative Computing Laboratory, University of Tennessee},
	author = {Mark Gates and Piotr Luszczek and Ahmad Abdelfattah and Jakub Kurzak and Jack Dongarra and Konstantin Arturov and Cris Cecka and Chip Freitag}
}
@conference {1094,
	title = {Co-Scheduling Algorithms for Cache-Partitioned Systems},
	booktitle = {19th Workshop on Advances in Parallel and Distributed Computational Models},
	year = {2017},
	month = {2017-05},
	publisher = {IEEE Computer Society Press},
	organization = {IEEE Computer Society Press},
	address = {Orlando, FL},
	abstract = {Cache-partitioned architectures allow subsections of the shared last-level cache (LLC) to be exclusively reserved for some applications. This technique dramatically limits interactions between applications that are concurrently executing on a multicore machine. Consider n applications that execute concurrently, with the objective to minimize the makespan, defined as the maximum completion time of the n applications. Key scheduling questions are: (i) which proportion of cache and (ii) how many processors should be given to each application? Here, we assign rational numbers of processors to each application, since they can be shared across applications through multi-threading. In this paper, we provide answers to (i) and (ii) for perfectly parallel applications. Even though the problem is shown to be NP-complete, we give key elements to determine the subset of applications that should share the LLC (while remaining ones only use their smaller private cache). Building upon these results, we design efficient heuristics for general applications. Extensive simulations demonstrate the usefulness of co-scheduling when our efficient cache partitioning strategies are deployed.},
	keywords = {Computational modeling, Degradation, Interference, Mathematical model, Program processors, Supercomputers, Throughput},
	doi = {10.1109/IPDPSW.2017.60},
	author = {Guillaume Aupy and Anne Benoit and Lo{\"\i}c Pottier and Padma Raghavan and Yves Robert and Manu Shantharam}
}
@article {1103,
	title = {Factorization and Inversion of a Million Matrices using GPUs: Challenges and Countermeasures},
	journal = {Procedia Computer Science},
	volume = {108},
	year = {2017},
	month = {2017-06},
	pages = {606{\textendash}615},
	abstract = {This paper presents new algorithmic approaches and optimization techniques for LU factorization and matrix inversion of millions of very small matrices using GPUs. These problems appear in many scientific applications including astrophysics and generation of block-Jacobi preconditioners. We show that, for very small problem sizes, design and optimization of GPU kernels require a mindset different from the one usually used when designing LAPACK algorithms for GPUs. Techniques for optimal memory traffic, register blocking, and tunable concurrency are incorporated in our proposed design. We also take advantage of the small matrix sizes to eliminate the intermediate row interchanges in both the factorization and inversion kernels. The proposed GPU kernels achieve performance speedups vs. CUBLAS of up to 6{\texttimes} for the factorization, and 14{\texttimes} for the inversion, using double precision arithmetic on a Pascal P100 GPU.},
	doi = {https://doi.org/10.1016/j.procs.2017.05.250},
	author = {Ahmad Abdelfattah and Azzam Haidar and Stanimire Tomov and Jack Dongarra}
}
@article {1102,
	title = {Fast Cholesky Factorization on GPUs for Batch and Native Modes in MAGMA},
	journal = {Journal of Computational Science},
	volume = {20},
	year = {2017},
	month = {2017-05},
	pages = {85{\textendash}93},
	abstract = {This paper presents a GPU-accelerated Cholesky factorization for two different modes of operation. The first one is the batch mode, where many independent factorizations on small matrices can be performed concurrently. This mode supports fixed size and variable size problems, and is found in many scientific applications. The second mode is the native mode, where one factorization is performed on a large matrix without any CPU involvement, which allows the CPU do other useful work. We show that, despite the different workloads, both modes of operation share a common code-base that uses the GPU only. We also show that the developed routines achieve significant speedups against a multicore CPU using the MKL library, and against a GPU implementation by cuSOLVER. This work is part of the MAGMA library.},
	keywords = {GPU computing; Cholesky factorization; Batched execution},
	doi = {https://doi.org/10.1016/j.jocs.2016.12.009},
	author = {Ahmad Abdelfattah and Azzam Haidar and Stanimire Tomov and Jack Dongarra}
}
@article {1163,
	title = {Flexible Batched Sparse Matrix Vector Product on GPUs},
	year = {2017},
	month = {2017-11},
	publisher = {ScalA{\textquoteright}17: 8th Workshop on Latest Advances in Scalable Algorithms for Large-Scale Systems},
	address = {Denver, Colorado},
	author = {Hartwig Anzt and Collins, Gary and Jack Dongarra and Goran Flegar and Enrique S. Quintana-Orti}
}
@conference {,
	title = {Flexible Batched Sparse Matrix-Vector Product on GPUs},
	booktitle = {8th Workshop on Latest Advances in Scalable Algorithms for Large-Scale Systems (ScalA {\textquoteright}17)},
	year = {2017},
	month = {2017-11},
	publisher = {ACM Press},
	organization = {ACM Press},
	address = {Denver, CO},
	abstract = {	
We propose a variety of batched routines for concurrently processing a large collection of small-size, independent sparse matrix-vector products (SpMV) on graphics processing units (GPUs). These batched SpMV kernels are designed to be flexible in order to handle a batch of matrices which differ in size, nonzero count, and nonzero distribution. Furthermore, they support three most commonly used sparse storage formats: CSR, COO and ELL. Our experimental results on a state-of-the-art GPU reveal performance improvements of up to 25X compared to non-batched SpMV routines.},
	doi = {http://dx.doi.org/10.1145/3148226.3148230},
	author = {Hartwig Anzt and Gary Collins and Jack Dongarra and Goran Flegar and Enrique S. Quintana-Orti}
}
@conference {1142,
	title = {High-performance Cholesky Factorization for GPU-only Execution},
	booktitle = {Proceedings of the General Purpose GPUs (GPGPU-10)},
	year = {2017},
	month = {2017-02},
	publisher = {ACM},
	organization = {ACM},
	address = {Austin, TX},
	abstract = {We present our performance analysis, algorithm designs, and the optimizations needed for the development of high-performance GPU-only algorithms, and in particular, for the dense Cholesky factorization. In contrast to currently promoted designs that solve parallelism challenges on multicore architectures by representing algorithms as Directed Acyclic Graphs (DAGs), where nodes are tasks of fine granularity and edges are the dependencies between the tasks, our designs explicitly target manycore architectures like GPUs and feature coarse granularity tasks (that can be hierarchically split into fine grain data-parallel subtasks). Furthermore, in contrast to hybrid algorithms that schedule difficult to parallelize tasks on CPUs, we develop highly-efficient code for entirely GPU execution. GPU-only codes remove the expensive CPU-to-GPU communications and the tuning challenges related to slow CPU and/or low CPU-to-GPU bandwidth. We show that on latest GPUs, like the P100, this becomes so important that the GPU-only code even outperforms the hybrid MAGMA algorithms when the CPU tasks and communications can not be entirely overlapped with GPU computations. We achieve up to 4,300 GFlop/s in double precision on a P100 GPU, which is about 7-8{\texttimes} faster than high-end multicore CPUs, e.g., two 10-cores Intel Xeon E5-2650 v3 Haswell CPUs, where MKL runs up to about 500-600 Gflop/s. The new algorithm also outperforms significantly the GPU-only implementation currently available in the NVIDIA cuSOLVER library.},
	doi = {https://doi.org/10.1145/3038228.3038237},
	author = {Azzam Haidar and Ahmad Abdelfattah and Stanimire Tomov and Jack Dongarra}
}
@techreport {1130,
	title = {MAGMA-sparse Interface Design Whitepaper},
	journal = {Innovative Computing Laboratory Technical Report},
	number = {ICL-UT-17-05},
	year = {2017},
	month = {2017-09},
	type = {Technical Report},
	abstract = {In this report we describe the logic and interface we develop for the MAGMA-sparse library
to allow for easy integration as third-party library into a top-level software ecosystem. The
design choices are based on extensive consultation with other software library developers, in
particular the Trilinos software development team. The interface documentation is at this point
not exhaustive, but a first proposal for setting a standard. Although the interface description
targets the MAGMA-sparse software module, we hope that the design choices carry beyond this
specific library, and are attractive for adoption in other packages.
This report is not intended as static document, but will be updated over time to reflect the agile
software development in the ECP 1.3.3.11 STMS11-PEEKS project.},
	author = {Hartwig Anzt and Erik Boman and Jack Dongarra and Goran Flegar and Mark Gates and Mike Heroux and Mark Hoemmen and Jakub Kurzak and Piotr Luszczek and Sivasankaran Rajamanickam and Stanimire Tomov and Stephen Wood and Ichitaro Yamazaki}
}
@conference {1084,
	title = {Novel HPC Techniques to Batch Execution of Many Variable Size BLAS Computations on GPUs},
	booktitle = {International Conference on Supercomputing (ICS {\textquoteright}17)},
	year = {2017},
	month = {2017-06},
	publisher = {ACM},
	organization = {ACM},
	address = {Chicago, Illinois},
	doi = {10.1145/3079079.3079103},
	url = {http://dl.acm.org/citation.cfm?id=3079103},
	author = {Ahmad Abdelfattah and Azzam Haidar and Stanimire Tomov and Jack Dongarra}
}
@inbook {1384,
	title = {Performance Analysis and Debugging Tools at Scale},
	booktitle = {Exascale Scientific Applications: Scalability and Performance Portability},
	year = {2017},
	month = {2017-11},
	pages = {17-50},
	publisher = {Chapman \& Hall / CRC Press},
	organization = {Chapman \& Hall / CRC Press},
	chapter = {2},
	abstract = {This chapter explores present-day challenges and those likely to arise as new hardware and software technologies are introduced on the path to exascale. It covers some of the underlying hardware, software, and techniques that enable tools and debuggers. Performance tools and debuggers are critical components that enable computational scientists to fully exploit the computing power of While high-performance computing systems. Instrumentation is the insertion of code to perform measurement in a program. It is vital step in performance analysis, especially for parallel programs. The essence of a debugging tool is enabling observation, exploration, and control of program state, such that a developer can, for example, verify that what is currently occurring correlates to what is intended. The increased complexity and volume of performance and debugging data likely to be seen on exascale systems risks overwhelming tool users. Tools and debuggers may need to develop advanced techniques such as automated filtering and analysis to reduce the complexity seen by the user.},
	isbn = {9781315277400},
	doi = {https://doi.org/10.1201/b21930},
	author = {Scott Parker and John Mellor-Crummey and Dong H. Ahn and Heike Jagode and Holger Brunst and Sameer Shende and Allen D. Malony and David DelSignore and Ronny Tschuter and Ralph Castain and Kevin Harms and Philip Carns and Ray Loy and Kalyan Kumaran}
}
@techreport {1173,
	title = {PLASMA 17 Performance Report},
	journal = {Innovative Computing Laboratory Technical Report},
	number = {ICL-UT-17-11},
	year = {2017},
	month = {2017-06},
	publisher = {University of Tennessee},
	abstract = {PLASMA (Parallel Linear Algebra for Multicore Architectures) is a dense linear algebra package at the forefront of multicore computing. PLASMA is designed to deliver the highest possible performance from a system with multiple sockets of multicore processors. PLASMA achieves this objective by combining state of the art solutions in parallel algorithms, scheduling, and software engineering. PLASMA currently offers a collection of routines for solving linear systems of equations and least square problems.},
	author = {Maksims Abalenkovs and Negin Bagherpour and Jack Dongarra and Mark Gates and Azzam Haidar and Jakub Kurzak and Piotr Luszczek and Samuel Relton and Jakub Sistek and David Stevens and Panruo Wu and Ichitaro Yamazaki and Asim YarKhan and Mawussi Zounon}
}
@techreport {1172,
	title = {PLASMA 17.1 Functionality Report},
	journal = {Innovative Computing Laboratory Technical Report},
	number = {ICL-UT-17-10},
	year = {2017},
	month = {2017-06},
	publisher = {University of Tennessee},
	abstract = {PLASMA (Parallel Linear Algebra for Multicore Architectures) is a dense linear algebra package at the forefront of multicore computing. PLASMA is designed to deliver the highest possible performance from a system with multiple sockets of multicore processors. PLASMA achieves this objective by combining state of the art solutions in parallel algorithms, scheduling, and software engineering. PLASMA currently offers a collection of routines for solving linear systems of equations and least square problems.},
	author = {Maksims Abalenkovs and Negin Bagherpour and Jack Dongarra and Mark Gates and Azzam Haidar and Jakub Kurzak and Piotr Luszczek and Samuel Relton and Jakub Sistek and David Stevens and Panruo Wu and Ichitaro Yamazaki and Asim YarKhan and Mawussi Zounon}
}
@article {1067,
	title = {Preconditioned Krylov Solvers on GPUs},
	journal = {Parallel Computing},
	year = {2017},
	month = {2017-06},
	abstract = {In this paper, we study the effect of enhancing GPU-accelerated Krylov solvers with preconditioners. We consider the BiCGSTAB, CGS, QMR, and IDR(s) Krylov solvers. For a large set of test matrices, we assess the impact of Jacobi and incomplete factorization preconditioning on the solvers{\textquoteright} numerical stability and time-to-solution performance. We also analyze how the use of a preconditioner impacts the choice of the fastest solver.},
	keywords = {gpu, ILU, Jacobi, Krylov solvers, Preconditioning},
	issn = {01678191},
	doi = {10.1016/j.parco.2017.05.006},
	url = {http://www.sciencedirect.com/science/article/pii/S0167819117300777},
	author = {Hartwig Anzt and Mark Gates and Jack Dongarra and Moritz Kreutzer and Gerhard Wellein and Martin Kohler}
}
@techreport {1080,
	title = {Roadmap for the Development of a Linear Algebra Library for Exascale Computing: SLATE: Software for Linear Algebra Targeting Exascale},
	journal = {SLATE Working Notes},
	number = {01, ICL-UT-17-02},
	year = {2017},
	month = {2017-06},
	publisher = {Innovative Computing Laboratory, University of Tennessee},
	type = {SLATE Working Notes},
	author = {Ahmad Abdelfattah and Hartwig Anzt and Aurelien Bouteiller and Anthony Danalis and Jack Dongarra and Mark Gates and Azzam Haidar and Jakub Kurzak and Piotr Luszczek and Stanimire Tomov and Stephen Wood and Panruo Wu and Ichitaro Yamazaki and Asim YarKhan}
}
@techreport {1082,
	title = {Small Tensor Operations on Advanced Architectures for High-Order Applications},
	journal = {University of Tennessee Computer Science Technical Report},
	number = {UT-EECS-17-749},
	year = {2017},
	month = {2017-04},
	publisher = {Innovative Computing Laboratory, University of Tennessee},
	author = {Ahmad Abdelfattah and Marc Baboulin and Veselin Dobrev and Jack Dongarra and Azzam Haidar and Ian Karlin and Tzanio Kolev and Ian Masliah and Stanimire Tomov}
}
@inproceedings {1088,
	title = {Variable-Size Batched Gauss-Huard for Block-Jacobi Preconditioning},
	journal = {International Conference on Computational Science (ICCS 2017)},
	volume = {108},
	year = {2017},
	month = {2017-06},
	pages = {1783-1792},
	publisher = {Procedia Computer Science},
	address = {Zurich, Switzerland},
	abstract = {In this work we present new kernels for the generation and application of block-Jacobi precon-ditioners that accelerate the iterative solution of sparse linear systems on graphics processing units (GPUs). Our approach departs from the conventional LU factorization and decomposes the diagonal blocks of the matrix using the Gauss-Huard method. When enhanced with column pivoting, this method is as stable as LU with partial/row pivoting. Due to extensive use of GPU registers and integration of implicit pivoting, our variable size batched Gauss-Huard implementation outperforms the batched version of LU factorization. In addition, the application kernel combines the conventional two-stage triangular solve procedure, consisting of a backward solve followed by a forward solve, into a single stage that performs both operations simultaneously.},
	doi = {https://doi.org/10.1016/j.procs.2017.05.186},
	author = {Hartwig Anzt and Jack Dongarra and Goran Flegar and Enrique S. Quintana-Orti and Andres E. Thomas}
}
@conference {1160,
	title = {Variable-Size Batched LU for Small Matrices and Its Integration into Block-Jacobi Preconditioning},
	booktitle = {46th International Conference on Parallel Processing (ICPP)},
	year = {2017},
	month = {2017-08},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Bristol, United Kingdom},
	abstract = {We present a set of new batched CUDA kernels for the LU factorization of a large collection of independent problems of different size, and the subsequent triangular solves. All kernels heavily exploit the registers of the graphics processing unit (GPU) in order to deliver high performance for small problems. The development of these kernels is motivated by the need for tackling this embarrassingly parallel scenario in the context of block-Jacobi preconditioning that is relevant for the iterative solution of sparse linear systems.},
	keywords = {graphics processing units, Jacobian matrices, Kernel, linear systems, Parallel processing, Sparse matrices},
	doi = {10.1109/ICPP.2017.18},
	url = {http://ieeexplore.ieee.org/abstract/document/8025283/?reload=true},
	author = {Hartwig Anzt and Jack Dongarra and Goran Flegar and Enrique S. Quintana-Orti}
}
@article {,
	title = {With Extreme Computing, the Rules Have Changed},
	journal = {Computing in Science \& Engineering},
	volume = {19},
	year = {2017},
	month = {2017-05},
	pages = {52-62},
	abstract = {On the eve of exascale computing, traditional wisdom no longer applies. High-performance computing is gone as we know it. This article discusses a range of new algorithmic techniques emerging in the context of exascale computing, many of which defy the common wisdom of high-performance computing and are considered unorthodox, but could turn out to be a necessity in near future.},
	doi = {https://doi.org/10.1109/MCSE.2017.48},
	author = {Jack Dongarra and Stanimire Tomov and Piotr Luszczek and Jakub Kurzak and Mark Gates and Ichitaro Yamazaki and Hartwig Anzt and Azzam Haidar and Ahmad Abdelfattah}
}
@article {1342,
	title = {Accelerating Tensor Contractions for High-Order FEM on CPUs, GPUs, and KNLs},
	year = {2016},
	month = {2016-09},
	publisher = {moky Mountains Computational Sciences and Engineering Conference (SMC16), Poster},
	address = {Gatlinburg, TN},
	author = {Azzam Haidar and Ahmad Abdelfattah and Veselin Dobrev and Ian Karlin and Tzanio Kolev and Stanimire Tomov and Jack Dongarra}
}
@article {994,
	title = {Accelerating the Conjugate Gradient Algorithm with GPU in CFD Simulations},
	journal = {VECPAR},
	year = {2016},
	abstract = {This paper illustrates how GPU computing can be used to accelerate computational fluid dynamics (CFD) simulations. For sparse linear systems arising from finite volume discretization, we evaluate and optimize the performance of Conjugate Gradient (CG) routines designed for manycore accelerators and compare against an industrial CPU-based implementation. We also investigate how the recent advances in preconditioning, such as iterative Incomplete Cholesky (IC, as symmetric case of ILU) preconditioning, match the requirements for solving real world problems.},
	url = {http://hgpu.org/?p=16264},
	author = {Hartwig Anzt and Marc Baboulin and Jack Dongarra and Yvan Fournier and Frank Hulsemann and Amal Khabou and Yushan Wang}
}
@inproceedings {991,
	title = {Batched Generation of Incomplete Sparse Approximate Inverses on GPUs},
	journal = {Proceedings of the 7th Workshop on Latest Advances in Scalable Algorithms for Large-Scale Systems},
	year = {2016},
	month = {2016-11},
	pages = {49{\textendash}56},
	abstract = {Incomplete Sparse Approximate Inverses (ISAI) have recently been shown to be an attractive alternative to exact sparse triangular solves in the context of incomplete factorization preconditioning. In this paper we propose a batched GPU-kernel for the efficient generation of ISAI matrices. Utilizing only thread-local memory allows for computing the ISAI matrix with very small memory footprint. We demonstrate that this strategy is faster than the existing strategy for generating ISAI matrices, and use a large number of test matrices to assess the algorithm{\textquoteright}s efficiency in an iterative solver setting.},
	isbn = {978-1-5090-5222-6},
	doi = {10.1109/ScalA.2016.11},
	author = {Hartwig Anzt and Edmond Chow and Thomas Huckle and Jack Dongarra}
}
@techreport {988,
	title = {On block-asynchronous execution on GPUs},
	journal = {LAPACK Working Note},
	number = {291},
	year = {2016},
	month = {2016-11},
	abstract = {This paper experimentally investigates how GPUs execute instructions when used for general purpose computing (GPGPU). We use a light-weight realizing a vector operation to analyze which vector entries are updated subsequently, and identify regions where parallel execution can be expected. The results help us to understand how GPUs operate, and map this operation mode to the mathematical concept of asynchronism. In particular it helps to understand the effects that can occur when implementing a fixed-point method using in-place updates on GPU hardware.},
	url = {http://www.netlib.org/lapack/lawnspdf/lawn291.pdf},
	author = {Hartwig Anzt and Edmond Chow and Jack Dongarra}
}
@article {1344,
	title = {Cholesky Factorization on Batches of Matrices with Fixed and Variable Sizes},
	year = {2016},
	month = {2016-04},
	publisher = {GPU Technology Conference (GTC16), Poster},
	address = {San Jose, CA},
	author = {Ahmad Abdelfattah and Azzam Haidar and Stanimire Tomov and Jack Dongarra}
}
@conference {940,
	title = {On the Development of Variable Size Batched Computation for Heterogeneous Parallel Architectures},
	booktitle = {The 17th IEEE International Workshop on Parallel and Distributed Scientific and Engineering Computing (PDSEC 2016), IPDPS 2016},
	year = {2016},
	month = {2016-05},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Chicago, IL},
	abstract = {<p>
Many scientific applications, ranging from national security to medical advances, require solving a number of relatively small-size independent problems. As the size of each individual problem does not provide sufficient parallelism for the underlying hardware, especially accelerators, these problems must be solved concurrently as a batch in order to saturate the hardware with enough work, hence the name batched computation. A possible simplification is to assume a uniform size for all problems. However, real applications do not necessarily satisfy such assumption. Consequently, an efficient solution for variable-size batched computations is required.
</p>
<p>
This paper proposes a foundation for high performance variable-size batched matrix computation based on Graphics Processing Units (GPUs). Being throughput-oriented processors, GPUs favor regular computation and less divergence among threads, in order to achieve high performance. Therefore, the development of high performance numerical software for this kind of problems is challenging. As a case study, we developed efficient batched Cholesky factorization algorithms for relatively small matrices of different sizes. However, most of the strategies and the software developed, and in particular a set of variable size batched BLAS kernels, can be used in many other dense matrix factorizations, large scale sparse direct multifrontal solvers, and applications. We propose new interfaces and mechanisms to handle the irregular computation pattern on the GPU. According to the authors{\textquoteright} knowledge, this is the first attempt to develop high performance software for this class of problems. Using a K40c GPU, our performance tests show speedups of up to 2:5 against two Sandy Bridge CPUs (8-core each) running Intel MKL library.
</p>},
	keywords = {batched computation, GPUs, variable small sizes},
	author = {Ahmad Abdelfattah and Azzam Haidar and Stanimire Tomov and Jack Dongarra}
}
@inproceedings {996,
	title = {Domain Overlap for Iterative Sparse Triangular Solves on GPUs},
	journal = {Software for Exascale Computing - SPPEXA},
	volume = {113},
	year = {2016},
	month = {2016-09},
	pages = {527{\textendash}545},
	publisher = {Springer International Publishing},
	abstract = {Iterative methods for solving sparse triangular systems are an attractive alternative to exact forward and backward substitution if an approximation of the solution is acceptable. On modern hardware, performance benefits are available as iterative methods allow for better parallelization. In this paper, we investigate how block-iterative triangular solves can benefit from using overlap. Because the matrices are triangular, we use {\textquotedblleft}directed{\textquotedblright} overlap, depending on whether the matrix is upper or lower triangular. We enhance a GPU implementation of the block-asynchronous Jacobi method with directed overlap. For GPUs and other cases where the problem must be overdecomposed, i.e., more subdomains and threads than cores, there is a preference in processing or scheduling the subdomains in a specific order, following the dependencies specified by the sparse triangular matrix. For sparse triangular factors from incomplete factorizations, we demonstrate that moderate directed overlap with subdomain scheduling can improve convergence and time-to-solution.},
	doi = {10.1007/978-3-319-40528-5_24},
	author = {Hartwig Anzt and Edmond Chow and Daniel Szyld and Jack Dongarra},
	editor = {Hans-Joachim Bungartz and Philipp Neumann and Wolfgang E. Nagel}
}
@inproceedings {992,
	title = {Efficiency of General Krylov Methods on GPUs {\textendash} An Experimental Study},
	journal = {2016 IEEE International Parallel and Distributed Processing Symposium Workshops (IPDPSW)},
	year = {2016},
	month = {2016-05},
	pages = {683-691},
	abstract = {This paper compares different Krylov methods based on short recurrences with respect to their efficiency whenimplemented on GPUs. The comparison includes BiCGSTAB, CGS, QMR, and IDR using different shadow space dimensions. These methods are known for their good convergencecharacteristics. For a large set of test matrices taken from theUniversity of Florida Matrix Collection, we evaluate the methods{\textquoteright}performance against different target metrics: convergence, number of sparse matrix-vector multiplications, and executiontime. We also analyze whether the methods are "orthogonal"in terms of problem suitability. We propose best practicesfor choosing methods in a "black box" scenario, where noinformation about the optimal solver is available.},
	keywords = {algorithmic bombardment, BiCGSTAB, CGS, Convergence, Electric breakdown, gpu, graphics processing units, Hardware, IDR(s), Krylov solver, Libraries, linear systems, QMR, Sparse matrices},
	doi = {10.1109/IPDPSW.2016.45},
	author = {Hartwig Anzt and Jack Dongarra and Moritz Kreutzer and Gerhard Wellein and Martin Kohler}
}
@conference {937,
	title = {Efficiency of General Krylov Methods on GPUs {\textendash} An Experimental Study},
	booktitle = {The Sixth International Workshop on Accelerators and Hybrid Exascale Systems (AsHES)},
	year = {2016},
	month = {2016-05},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Chicago, IL},
	abstract = {This paper compares different Krylov methods based on short recurrences with respect to their efficiency when implemented on GPUs. The comparison includes BiCGSTAB, CGS, QMR, and IDR using different shadow space dimensions. These methods are known for their good convergence characteristics. For a large set of test matrices taken from the University of Florida Matrix Collection, we evaluate the methods{\textquoteright}
performance against different target metrics: convergence, number of sparse matrix-vector multiplications, and execution time. We also analyze whether the methods are {\textquotedblleft}orthogonal{\textquotedblright} in terms of problem suitability. We propose best practices for choosing methods in a {\textquotedblleft}black box{\textquotedblright} scenario, where no information about the optimal solver is available.},
	keywords = {algorithmic bombardment, BiCGSTAB, CGS, gpu, IDR(s), Krylov solver, QMR},
	doi = {10.1109/IPDPSW.2016.45},
	author = {Hartwig Anzt and Jack Dongarra and Moritz Kreutzer and Gerhard Wellein and Martin Kohler}
}
@article {989,
	title = {Fine-grained Bit-Flip Protection for Relaxation Methods},
	journal = {Journal of Computational Science},
	year = {2016},
	month = {2016-11},
	abstract = {Resilience is considered a challenging under-addressed issue that the high performance computing community (HPC) will have to face in order to produce reliable Exascale systems by the beginning of the next decade. As part of a push toward a resilient HPC ecosystem, in this paper we propose an error-resilient iterative solver for sparse linear systems based on stationary component-wise relaxation methods. Starting from a plain implementation of the Jacobi iteration, our approach introduces a low-cost component-wise technique that detects bit-flips, rejecting some component updates, and turning the initial synchronized solver into an asynchronous iteration. Our experimental study with sparse incomplete factorizations from a collection of real-world applications, and a practical GPU implementation, exposes the convergence delay incurred by the fault-tolerant implementation and its practical performance.},
	keywords = {Bit flips, Fault tolerance, High Performance Computing, iterative solvers, Jacobi method, sparse linear systems},
	doi = {https://doi.org/10.1016/j.jocs.2016.11.013},
	author = {Hartwig Anzt and Jack Dongarra and Enrique S. Quintana-Orti}
}
@conference {939,
	title = {Heterogeneous Streaming},
	booktitle = {The Sixth International Workshop on Accelerators and Hybrid Exascale Systems (AsHES), IPDPS 2016},
	year = {2016},
	month = {2016-05},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Chicago, IL},
	abstract = {This paper introduces a new heterogeneous streaming library called hetero Streams (hStreams). We show how a simple FIFO streaming model can be applied to heterogeneous systems that include manycore coprocessors and multicore CPUs. This model supports concurrency across nodes, among tasks within a node, and between data transfers and computation. We give examples for different approaches, show how the implementation can be layered, analyze overheads among layers, and apply those models to parallelize applications using simple, intuitive interfaces. We compare the features and versatility of hStreams, OpenMP, CUDA Streams1 and OmpSs. We show how the use of hStreams makes it easier for scientists to identify tasks and easily expose concurrency among them, and how it enables tuning experts and runtime systems to tailor execution for different heterogeneous targets. Practical application examples are taken from the field of numerical linear algebra, commercial structural simulation software, and a seismic processing application.},
	keywords = {plasma},
	author = {Chris J. Newburn and Gaurav Bansal and Michael Wood and Luis Crivelli and Judit Planas and Alejandro Duran and Paulo Souza and Leonardo Borges and Piotr Luszczek and Stanimire Tomov and Jack Dongarra and Hartwig Anzt and Mark Gates and Azzam Haidar and Yulu Jia and Khairul Kabir and Ichitaro Yamazaki and Jesus Labarta}
}
@conference {964,
	title = {High-performance Matrix-matrix Multiplications of Very Small Matrices},
	booktitle = {22nd International European Conference on Parallel and Distributed Computing (Euro-Par{\textquoteright}16)},
	year = {2016},
	month = {2016-08},
	publisher = {Springer International Publishing},
	organization = {Springer International Publishing},
	address = {Grenoble, France},
	abstract = {The use of the general dense matrix-matrix multiplication (GEMM) is fundamental for obtaining high performance in many scientific computing applications. GEMMs for small matrices (of sizes less than 32) however, are not sufficiently optimized in existing libraries. In this paper we consider the case of many small GEMMs on either CPU or GPU architectures. This is a case that often occurs in applications like big data analytics, machine learning, high-order FEM, and others. The GEMMs are grouped together in a single batched routine. We present specialized for these cases algorithms and optimization techniques to obtain performance that is within 90\% of the optimal. We show that these results outperform currently available state-of-the-art implementations and vendor-tuned math libraries.},
	author = {Ian Masliah and Ahmad Abdelfattah and Azzam Haidar and Stanimire Tomov and Jo{\"e}l Falcou and Jack Dongarra}
}
@techreport {929,
	title = {High-Performance Tensor Contractions for GPUs},
	journal = {University of Tennessee Computer Science Technical Report},
	number = {UT-EECS-16-738},
	year = {2016},
	month = {2016-01},
	publisher = {University of Tennessee},
	abstract = {We present a computational framework for high-performance tensor contractions on GPUs. High-performance is difficult to obtain using existing libraries, especially for many  independent contractions where each contraction is very small, e.g., sub-vector/warp in size. However, using our framework to batch contractions plus application-specifics, we demonstrate close to peak performance results. In particular, to accelerate large scale tensor-formulated high-order finite element method (FEM) simulations, which is the main focus and motivation for this work, we represent contractions as tensor index reordering plus matrix-matrix multiplications (GEMMs). This is a key factor to achieve algorithmically many-fold acceleration (vs. not using it) due to possible reuse of data loaded in fast memory. In addition to using this context knowledge, we design tensor data-structures, tensor algebra interfaces, and new tensor contraction algorithms and implementations to achieve 90+\% of a theoretically derived peak on GPUs. On a K40c GPU for contractions resulting in GEMMs on square matrices of size 8 for example, we are 2.8{\texttimes} faster than CUBLAS, and 8.5{\texttimes} faster than MKL on 16 cores of Intel Xeon ES-2670 (Sandy Bridge) 2.60GHz CPUs. Finally, we apply autotuning and code generation techniques to simplify tuning and provide an architecture-aware, user-friendly interface.},
	author = {Ahmad Abdelfattah and Marc Baboulin and Veselin Dobrev and Jack Dongarra and Christopher Earl and Jo{\"e}l Falcou and Azzam Haidar and Ian Karlin and Tzanio Kolev and Ian Masliah and Stanimire Tomov}
}
@conference {942,
	title = {High-Performance Tensor Contractions for GPUs},
	booktitle = {International Conference on Computational Science (ICCS{\textquoteright}16)},
	year = {2016},
	month = {2016-06},
	address = {San Diego, CA},
	abstract = {We present a computational framework for high-performance tensor contractions on GPUs. High-performance is difficult to obtain using existing libraries, especially for many independent contractions where each contraction is very small, e.g., sub-vector/warp in size. However, using our framework to batch contractions plus application-specifics, we demonstrate close to peak performance results. In particular, to accelerate large scale tensor-formulated high-order finite element method (FEM) simulations, which is the main focus and motivation for this work, we represent contractions as tensor index reordering plus matrix-matrix multiplications (GEMMs). This is a key factor to achieve algorithmically many-fold acceleration (vs. not using it) due to possible reuse of data loaded in fast memory. In addition to using this context knowledge, we design tensor data-structures, tensor algebra interfaces, and new tensor contraction algorithms and implementations to achieve 90+\% of a theoretically derived peak on GPUs. On a K40c GPU for contractions resulting in GEMMs on square matrices of size 8 for example, we are 2.8{\texttimes} faster than CUBLAS, and 8.5{\texttimes} faster than MKL on 16 cores of Intel Xeon E5-2670 (Sandy Bridge) 2.60GHz CPUs. Finally, we apply autotuning and code generation techniques to simplify tuning and provide an architecture-aware, user-friendly interface.},
	keywords = {Applications, Batched linear algebra, FEM, gpu, Tensor contractions, Tensor HPC},
	author = {Ahmad Abdelfattah and Marc Baboulin and Veselin Dobrev and Jack Dongarra and Christopher Earl and Jo{\"e}l Falcou and Azzam Haidar and Ian Karlin and Tzanio Kolev and Ian Masliah and Stanimire Tomov}
}
@article {1472,
	title = {Linear Algebra Software for Large-Scale Accelerated Multicore Computing},
	journal = {Acta Numerica},
	volume = {25},
	year = {2016},
	month = {2016-05},
	pages = {1-160},
	abstract = {Many crucial scientific computing applications, ranging from national security to medical advances, rely on high-performance linear algebra algorithms and technologies, underscoring their importance and broad impact. Here we present the state-of-the-art design and implementation practices for the acceleration of the predominant linear algebra algorithms on large-scale accelerated multicore systems. Examples are given with fundamental dense linear algebra algorithms {\textendash} from the LU, QR, Cholesky, and LDLT factorizations needed for solving linear systems of equations, to eigenvalue and singular value decomposition (SVD) problems. The implementations presented are readily available via the open-source PLASMA and MAGMA libraries, which represent the next generation modernization of the popular LAPACK library for accelerated multicore systems. To generate the extreme level of parallelism needed for the efficient use of these systems, algorithms of interest are redesigned and then split into well-chosen computational tasks. The task execution is scheduled over the computational components of a hybrid system of multicore CPUs with GPU accelerators and/or Xeon Phi coprocessors, using either static scheduling or light-weight runtime systems. The use of light-weight runtime systems keeps scheduling overheads low, similar to static scheduling, while enabling the expression of parallelism through sequential-like code. This simplifies the development effort and allows exploration of the unique strengths of the various hardware components. Finally, we emphasize the development of innovative linear algebra algorithms using three technologies {\textendash} mixed precision arithmetic, batched operations, and asynchronous iterations {\textendash} that are currently of high interest for accelerated multicore systems.},
	doi = {10.1017/S0962492916000015},
	author = {Ahmad Abdelfattah and Hartwig Anzt and Jack Dongarra and Mark Gates and Azzam Haidar and Jakub Kurzak and Piotr Luszczek and Stanimire Tomov and undefined and Asim YarKhan}
}
@conference {963,
	title = {LU, QR, and Cholesky Factorizations: Programming Model, Performance Analysis and Optimization Techniques for the Intel Knights Landing Xeon Phi},
	booktitle = {IEEE High Performance Extreme Computing Conference (HPEC{\textquoteright}16)},
	year = {2016},
	month = {2016-09},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Waltham, MA},
	abstract = {A wide variety of heterogeneous compute resources, ranging from multicore CPUs to GPUs and coprocessors, are available to modern computers, making it challenging to design unified numerical libraries that efficiently and productively use all these varied resources. For example, in order to efficiently use Intel{\textquoteright}s Knights Langing (KNL) processor, the next-generation of Xeon Phi architectures, one must design and schedule an application in multiple degrees of parallelism and task grain sizes in order to obtain efficient performance. We propose a productive and portable programming model that allows us to write a serial-looking code, which, however, achieves parallelism and scalability by using a lightweight runtime environment to manage the resource-specific workload, and to control the dataflow and the parallel execution. This is done through multiple techniques ranging from multi-level data partitioning to adaptive task grain sizes, and dynamic task scheduling. In addition, our task abstractions enable unified algorithmic development across all the heterogeneous resources. Finally, we outline the strengths and the effectiveness of this approach {\textendash} especially in regards to hardware trends and ease of programming high-performance numerical software that current applications need {\textendash} in order to motivate current work and future directions for the next generation of parallel programming models for high-performance linear algebra libraries on heterogeneous systems.},
	author = {Azzam Haidar and Stanimire Tomov and Konstantin Arturov and Murat Guney and Shane Story and Jack Dongarra}
}
@techreport {,
	title = {MAGMA Batched: A Batched BLAS Approach for Small Matrix Factorizations and Applications on GPUs},
	journal = {Innovative Computing Laboratory Technical Report},
	number = {ICL-UT-16-02},
	year = {2016},
	month = {2016-08},
	publisher = {University of Tennessee},
	abstract = {A particularly challenging class of problems arising in many applications, called batched problems, involves linear algebra operations on many small-sized matrices. We proposed and designed batched BLAS (Basic Linear Algebra Subroutines), Level-2 GEMV and Level-3 GEMM, to solve them. We illustrate how batched GEMV and GEMM to be able to assist batched advance factorization (e.g. bi-diagonalization) and other BLAS routines (e.g. triangular solve) to achieve optimal performance on GPUs. Our solutions achieved up to 2.8-3{\texttimes} speedups compared to CUBLAS and MKL solutions, wherever possible. We illustrated the batched methodology on a real-world Hydrodynamic application by reformulating the tensor operations into batched BLAS GEMV and GEMM operations. A 2.5{\texttimes} speedup and a 1.4{\texttimes} greenup are obtained by changing 10\% of the code. We accelerated and scaled it on Titan supercomputer to 4096 nodes.},
	author = {Tingxing Dong and Azzam Haidar and Piotr Luszczek and Stanimire Tomov and Ahmad Abdelfattah and Jack Dongarra}
}
@article {990,
	title = {On the performance and energy efficiency of sparse linear algebra on GPUs},
	journal = {International Journal of High Performance Computing Applications},
	year = {2016},
	month = {2016-10},
	abstract = {In this paper we unveil some performance and energy efficiency frontiers for sparse computations on GPU-based supercomputers. We compare the resource efficiency of different sparse matrix{\textendash}vector products (SpMV) taken from libraries such as cuSPARSE and MAGMA for GPU and Intel{\textquoteright}s MKL for multicore CPUs, and develop a GPU sparse matrix{\textendash}matrix product (SpMM) implementation that handles the simultaneous multiplication of a sparse matrix with a set of vectors in block-wise fashion. While a typical sparse computation such as the SpMV reaches only a fraction of the peak of current GPUs, we show that the SpMM succeeds in exceeding the memory-bound limitations of the SpMV. We integrate this kernel into a GPU-accelerated Locally Optimal Block Preconditioned Conjugate Gradient (LOBPCG) eigensolver. LOBPCG is chosen as a benchmark algorithm for this study as it combines an interesting mix of sparse and dense linear algebra operations that is typical for complex simulation applications, and allows for hardware-aware optimizations. In a detailed analysis we compare the performance and energy efficiency against a multi-threaded CPU counterpart. The reported performance and energy efficiency results are indicative of sparse computations on supercomputers.},
	doi = {10.1177/1094342016672081},
	url = {http://hpc.sagepub.com/content/early/2016/10/05/1094342016672081.abstract},
	author = {Hartwig Anzt and Stanimire Tomov and Jack Dongarra}
}
@conference {944,
	title = {Performance, Design, and Autotuning of Batched GEMM for GPUs},
	booktitle = {The International Supercomputing Conference (ISC High Performance 2016)},
	year = {2016},
	month = {2016-06},
	address = {Frankfurt, Germany},
	abstract = {The general matrix-matrix multiplication (GEMM) is the most important numerical kernel in dense linear algebra, and is the key component for obtaining high performance in most LAPACK routines. As batched computations on relatively small problems continue to gain interest in many scientific applications, a need arises for a high performance GEMM kernel for batches of small matrices. Such a kernel should be well designed and tuned to handle small sizes, and to maintain high performance for realistic test cases found in the higher level LAPACK routines, and scientific computing applications in general. This paper presents a high performance batched GEMM kernel on Graphics Processing Units (GPUs). We address batched problems with both fixed and variable sizes, and show that specialized GEMM designs and
a comprehensive autotuning process are needed to handle problems of small sizes. For most performance tests reported in this paper, the proposed kernels outperform state-of-the-art approaches using a K40c GPU.},
	keywords = {Autotuning, Batched GEMM, GEMM, GPU computing, HPC},
	author = {Ahmad Abdelfattah and Azzam Haidar and Stanimire Tomov and Jack Dongarra}
}
@inbook {997,
	title = {Performance, Design, and Autotuning of Batched GEMM for GPUs},
	booktitle = {High Performance Computing: 31st International Conference, ISC High Performance 2016, Frankfurt, Germany, June 19-23, 2016, Proceedings},
	number = {9697},
	year = {2016},
	pages = {21{\textendash}38},
	publisher = {Springer International Publishing},
	organization = {Springer International Publishing},
	abstract = {The general matrix-matrix multiplication (GEMM) is the most important numerical kernel in dense linear algebra, and is the key component for obtaining high performance in most LAPACK routines. As batched computations on relatively small problems continue to gain interest in many scientific applications, a need arises for a high performance GEMM kernel for batches of small matrices. Such a kernel should be well designed and tuned to handle small sizes, and to maintain high performance for realistic test cases found in the higher level LAPACK routines, and scientific computing applications in general.

This paper presents a high performance batched GEMM kernel on Graphics Processing Units (GPUs). We address batched problems with both fixed and variable sizes, and show that specialized GEMM designs and a comprehensive autotuning process are needed to handle problems of small sizes. For most performance tests reported in this paper, the proposed kernels outperform state-of-the-art approaches using a K40c GPU.},
	isbn = {978-3-319-41321-1},
	doi = {10.1007/978-3-319-41321-1_2},
	url = {http://dx.doi.org/10.1007/978-3-319-41321-1_2},
	author = {Ahmad Abdelfattah and Azzam Haidar and Stanimire Tomov and Jack Dongarra},
	editor = {Julian M. Kunkel and Pavan Balaji and Jack Dongarra}
}
@techreport {934,
	title = {Performance, Design, and Autotuning of Batched GEMM for GPUs},
	journal = {University of Tennessee Computer Science Technical Report},
	number = {UT-EECS-16-739},
	year = {2016},
	month = {2016-02},
	publisher = {University of Tennessee},
	abstract = {Abstract. The general matrix-matrix multiplication (GEMM) is the most important numerical kernel in dense linear algebra. It is the key
component for obtaining high performance in most LAPACK routines. As batched computations on relatively small problems continue to gain interest in many scientific applications, there becomes a need to have a high performance GEMM kernel for a batch of small matrices. Such kernel should be well designed and tuned to handle small sizes, and to maintain high performance for realistic test cases found in the higher level LAPACK routines, and scientific computing applications in general. This paper presents a high performance batched GEMM kernel on Graphics Processing Units (GPUs). We address batched problems with both xed and variable sizes, and show that specialized GEMM designs and a comprehensive autotuning process are needed to handle problems of small sizes. For most performance test reported in this paper, the proposed kernels outperform state-of-the-art approaches using a K40c GPU.},
	keywords = {Autotuning, Batched GEMM, GEMM, GPU computing, HPC},
	author = {Ahmad Abdelfattah and Azzam Haidar and Stanimire Tomov and Jack Dongarra}
}
@article {985,
	title = {Performance optimization of Sparse Matrix-Vector Multiplication for multi-component PDE-based applications using GPUs},
	journal = {Concurrency and Computation: Practice and Experience},
	volume = {28},
	year = {2016},
	month = {2016-05},
	pages = {3447 - 3465},
	abstract = {Simulations of many multi-component PDE-based applications, such as petroleum reservoirs or reacting flows, are dominated by the solution, on each time step and within each Newton step, of large sparse linear systems. The standard solver is a preconditioned Krylov method. Along with application of the preconditioner, memory-bound Sparse Matrix-Vector Multiplication (SpMV) is the most time-consuming operation in such solvers. Multi-species models produce Jacobians with a dense block structure, where the block size can be as large as a few dozen. Failing to exploit this dense block structure vastly underutilizes hardware capable of delivering high performance on dense BLAS operations. This paper presents a GPU-accelerated SpMV kernel for block-sparse matrices. Dense matrix-vector multiplications within the sparse-block structure leverage optimization techniques from the KBLAS library, a high performance library for dense BLAS kernels. The design ideas of KBLAS can be applied to block-sparse matrices. Furthermore, a technique is proposed to balance the workload among thread blocks when there are large variations in the lengths of nonzero rows. Multi-GPU performance is highlighted. The proposed SpMV kernel outperforms existing state-of-the-art implementations using matrices with real structures from different applications.},
	doi = {10.1002/cpe.v28.1210.1002/cpe.3874},
	url = {http://onlinelibrary.wiley.com/doi/10.1002/cpe.3874/full},
	author = {Ahmad Abdelfattah and Hatem Ltaeif and David Keyes and Jack Dongarra}
}
@conference {943,
	title = {Performance Tuning and Optimization Techniques of Fixed and Variable Size Batched Cholesky Factorization on GPUs},
	booktitle = {International Conference on Computational Science (ICCS{\textquoteright}16)},
	year = {2016},
	month = {2016-06},
	address = {San Diego, CA},
	abstract = {<p>Solving a large number of relatively small linear systems has recently drawn more attention in the HPC community, due to the importance of such computational workloads in many scientific applications, including sparse multifrontal solvers. Modern hardware accelerators and their architecture require a set of optimization techniques that are very different from the ones used in solving one relatively large matrix. In order to impose concurrency on such throughput-oriented architectures, a common practice is to batch the solution of these matrices as one task offloaded to the underlying hardware, rather than solving them individually.</p>

<p>This paper presents a high performance batched Cholesky factorization on large sets of relatively small matrices using Graphics Processing Units (GPUs), and addresses both fixed and variable size batched problems. We investigate various algorithm designs and optimization techniques, and show that it is essential to combine kernel design with performance tuning in order to achieve the best possible performance. We compare our approaches against state-of-the-art CPU solutions as well as GPU-based solutions using existing libraries, and show that, on a K40c GPU for example, our kernels are more than 2 faster.</p>},
	keywords = {batched computation, Cholesky Factorization, GPUs, Tuning},
	author = {Ahmad Abdelfattah and Azzam Haidar and Stanimire Tomov and Jack Dongarra}
}
@article {932,
	title = {Scheduling Computational Workflows on Failure-prone Platforms},
	journal = {International Journal of Networking and Computing},
	volume = {6},
	number = {1},
	year = {2016},
	pages = {2-26},
	abstract = {We study the scheduling of computational workflows on compute resources that experience exponentially distributed failures. When a failure occurs, rollback and recovery is used to resume the execution from the last checkpointed state. The scheduling problem is to minimize the expected execution time by deciding in which order to execute the tasks in the workflow and deciding for each task whether to checkpoint it or not after it completes. We give a polynomialtime optimal algorithm for fork DAGs (Directed Acyclic Graphs) and show that the problem is NP-complete with join DAGs. We also investigate the complexity of the simple case in which no task is checkpointed. Our main result is a polynomial-time algorithm to compute the expected execution time of a workflow, with a given task execution order and specified to-be-checkpointed tasks. Using this algorithm as a basis, we propose several heuristics for solving the scheduling problem. We evaluate these heuristics for representative workflow configurations.},
	keywords = {checkpointing, fault-tolerance, reliability, scheduling, workflow},
	issn = { 2185-2847},
	author = {Guillaume Aupy and Anne Benoit and Henri Casanova and Yves Robert}
}
@article {995,
	title = {Updating Incomplete Factorization Preconditioners for Model Order Reduction},
	journal = {Numerical Algorithms},
	volume = {73},
	number = {3},
	year = {2016},
	month = {2016-02},
	pages = {611{\textendash}630},
	abstract = {When solving a sequence of related linear systems by iterative methods, it is common to reuse the preconditioner for several systems, and then to recompute the preconditioner when the matrix has changed significantly. Rather than recomputing the preconditioner from scratch, it is potentially more efficient to update the previous preconditioner. Unfortunately, it is not always known how to update a preconditioner, for example, when the preconditioner is an incomplete factorization. A recently proposed iterative algorithm for computing incomplete factorizations, however, is able to exploit an initial guess, unlike existing algorithms for incomplete factorizations. By treating a previous factorization as an initial guess to this algorithm, an incomplete factorization may thus be updated. We use a sequence of problems from model order reduction. Experimental results using an optimized GPU implementation show that updating a previous factorization can be inexpensive and effective, making solving sequences of linear systems a potential niche problem for the iterative incomplete factorization algorithm.},
	keywords = {key publication},
	doi = {10.1007/s11075-016-0110-2},
	author = {Hartwig Anzt and Edmond Chow and Jens Saak and Jack Dongarra}
}
@conference {880,
	title = {Accelerating Collaborative Filtering for Implicit Feedback Datasets using GPUs},
	booktitle = {2015 IEEE International Conference on Big Data (IEEE BigData 2015)},
	year = {2015},
	month = {2015-11},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Santa Clara, CA},
	abstract = {In this paper we accelerate the Alternating Least Squares (ALS) algorithm used for generating product recommendations on the basis of implicit feedback datasets. We approach the algorithm with concepts proven to be successful in High Performance Computing. This includes the formulation of the algorithm as a mix of cache-optimized algorithm-specific kernels and standard BLAS routines, acceleration via graphics processing units (GPUs), use of parallel batched kernels, and autotuning to identify performance winners. For benchmark datasets, the multi-threaded CPU implementation we propose achieves more than a 10 times speedup over the implementations available in the GraphLab and Spark MLlib software packages. For the GPU implementation, the parameters of an algorithm-specific kernel were optimized using a comprehensive autotuning sweep. This results in an additional 2 times speedup over our CPU implementation.},
	author = {Mark Gates and Hartwig Anzt and Jakub Kurzak and Jack Dongarra}
}
@conference {914,
	title = {Accelerating the LOBPCG method on GPUs using a blocked Sparse Matrix Vector Product},
	booktitle = {Spring Simulation Multi-Conference 2015 (SpringSim{\textquoteright}15)},
	year = {2015},
	month = {2015-04},
	publisher = {SCS},
	organization = {SCS},
	address = {Alexandria, VA},
	abstract = {This paper presents a heterogeneous CPU-GPU implementation for a sparse iterative eigensolver the Locally Optimal Block Preconditioned Conjugate Gradient (LOBPCG). For the key routine generating the Krylov search spaces via the product of a sparse matrix and a block of vectors, we propose a GPU kernel based on a modied sliced ELLPACK format. Blocking a set of vectors and processing them simultaneously accelerates the computation of a set of consecutive SpMVs significantly. Comparing the performance against similar routines from Intel{\textquoteright}s MKL and NVIDIA{\textquoteright}s cuSPARSE library we identify appealing performance improvements. We integrate it into the highly optimized LOBPCG implementation. Compared to the BLOBEX CPU implementation running on two eight-core Intel Xeon E5-2690s, we accelerate the computation of a small set of eigenvectors using NVIDIA{\textquoteright}s K40 GPU by typically more than an order of magnitude.},
	author = {Hartwig Anzt and Stanimire Tomov and Jack Dongarra}
}
@article {866,
	title = {Acceleration of GPU-based Krylov solvers via Data Transfer Reduction},
	journal = {International Journal of High Performance Computing Applications},
	year = {2015},
	author = {Hartwig Anzt and William Sawyer and Stanimire Tomov and Piotr Luszczek and Jack Dongarra}
}
@conference {912,
	title = {Adaptive Precision Solvers for Sparse Linear Systems},
	booktitle = {3rd International Workshop on Energy Efficient Supercomputing (E2SC {\textquoteright}15)},
	year = {2015},
	month = {2015-11},
	publisher = {ACM},
	organization = {ACM},
	address = {Austin, TX},
	author = {Hartwig Anzt and Jack Dongarra and Enrique S. Quintana-Orti}
}
@conference {865,
	title = {Asynchronous Iterative Algorithm for Computing Incomplete Factorizations on GPUs},
	booktitle = {International Supercomputing Conference (ISC 2015)},
	year = {2015},
	month = {2015-07},
	address = {Frankfurt, Germany},
	author = {Edmond Chow and Hartwig Anzt and Jack Dongarra}
}
@conference {895,
	title = {Batched Matrix Computations on Hardware Accelerators Based on GPUs},
	booktitle = {2015 SIAM Conference on Applied Linear Algebra (SIAM LA)},
	year = {2015},
	month = {2015-10},
	publisher = {SIAM},
	organization = {SIAM},
	address = {Atlanta, GA},
	abstract = {We will present techniques for small matrix computations on GPUs and their use for energy efficient, high-performance solvers. Work on small problems delivers high performance through improved data reuse. Many numerical libraries and applications need this functionality further developed. We describe the main factorizations LU, QR, and Cholesky for a set of small dense matrices in parallel. We achieve significant acceleration and reduced energy consumption against other solutions. Our techniques are of interest to GPU application developers in general.},
	author = {Azzam Haidar and Ahmad Abdelfattah and Stanimire Tomov and Jack Dongarra}
}
@conference {857,
	title = {Energy Efficiency and Performance Frontiers for Sparse Computations on GPU Supercomputers},
	booktitle = {Sixth International Workshop on Programming Models and Applications for Multicores and Manycores (PMAM {\textquoteright}15)},
	year = {2015},
	month = {2015-02},
	publisher = {ACM},
	organization = {ACM},
	address = {San Francisco, CA},
	abstract = {In this paper we unveil some energy efficiency and performance frontiers for sparse computations on GPU-based supercomputers. To do this, we consider state-of-the-art implementations of the sparse matrix-vector (SpMV) product in libraries like cuSPARSE, MKL, and MAGMA, and their use in the LOBPCG eigen-solver. LOBPCG is chosen as a benchmark for this study as it combines an interesting mix of sparse and dense linear algebra operations with potential for hardware-aware optimizations. Most notably, LOBPCG includes a blocking technique that is a common performance optimization for many applications. In particular, multiple memory-bound SpMV operations are blocked into a SpM-matrix product (SpMM), that achieves significantly higher performance than a sequence of SpMVs. We provide details about the GPU kernels we use for the SpMV, SpMM, and the LOBPCG implementation design, and study performance and energy consumption compared to CPU solutions. While a typical sparse computation like the SpMV reaches only a fraction of the peak of current GPUs, we show that the SpMM achieves up to a 6x performance improvement over the GPU{\textquoteright}s SpMV, and the GPU-accelerated LOBPCG based on this kernel is 3 to 5x faster than multicore CPUs with the same power draw, e.g., a K40 GPU vs. two Sandy Bridge CPUs (16 cores). In practice though, we show that currently available CPU implementations are much slower due to missed optimization opportunities. These performance results translate to similar improvements in energy consumption, and are indicative of today{\textquoteright}s frontiers in energy efficiency and performance for sparse computations on supercomputers.},
	isbn = {978-1-4503-3404-4},
	doi = {10.1145/2712386.2712387},
	author = {Hartwig Anzt and Stanimire Tomov and Jack Dongarra}
}
@article {873,
	title = {Experiences in autotuning matrix multiplication for energy minimization on GPUs},
	journal = {Concurrency in Computation: Practice and Experience},
	volume = {27},
	year = {2015},
	month = {2015-12},
	pages = {5096-5113},
	doi = {10.1002/cpe.3516},
	author = {Hartwig Anzt and Blake Haugen and Jakub Kurzak and Piotr Luszczek and Jack Dongarra}
}
@article {982,
	title = {Experiences in Autotuning Matrix Multiplication for Energy Minimization on GPUs},
	journal = {Concurrency and Computation: Practice and Experience},
	volume = {27},
	year = {2015},
	month = {12-Oct},
	pages = {5096 - 5113},
	abstract = {In this paper, we report extensive results and analysis of autotuning the computationally intensive graphics processing units kernel for dense matrix{\textendash}matrix multiplication in double precision. In contrast to traditional autotuning and/or optimization for runtime performance only, we also take the energy efficiency into account. For kernels achieving equal performance, we show significant differences in their energy balance. We also identify the memory throughput as the most influential metric that trades off performance and energy efficiency. As a result, the performance optimal case ends up not being the most efficient kernel in overall resource use.},
	keywords = {Autotuning, energy efficiency, hardware accelerators, matrix multiplication, power},
	doi = {10.1002/cpe.3516},
	url = {http://doi.wiley.com/10.1002/cpe.3516https://api.wiley.com/onlinelibrary/tdm/v1/articles/10.1002\%2Fcpe.3516},
	author = {Hartwig Anzt and Blake Haugen and Jakub Kurzak and Piotr Luszczek and Jack Dongarra}
}
@conference {913,
	title = {GPU-accelerated Co-design of Induced Dimension Reduction: Algorithmic Fusion and Kernel Overlap},
	booktitle = {2nd International Workshop on Hardware-Software Co-Design for High Performance Computing},
	year = {2015},
	month = {2015-11},
	publisher = {ACM},
	organization = {ACM},
	address = {Austin, TX},
	abstract = {In this paper we present an optimized GPU co-design of the Induced Dimension Reduction (IDR) algorithm for solving linear systems. Starting from a baseline implementation based on the generic BLAS routines from the MAGMA software library, we apply optimizations
that are based on kernel fusion and kernel overlap. Runtime experiments are used to investigate the benefit of the distinct optimization techniques for different variants of the IDR algorithm. A comparison to the reference implementation reveals that the interplay between them can succeed in cutting the overall runtime by up to about one third.},
	author = {Hartwig Anzt and Eduardo Ponce and Gregory D. Peterson and Jack Dongarra}
}
@article {881,
	title = {Implementation and Tuning of Batched Cholesky Factorization and Solve for NVIDIA GPUs},
	journal = {IEEE Transactions on Parallel and Distributed Systems},
	number = {1045-9219},
	year = {2015},
	month = {2015-11},
	author = {Jakub Kurzak and Hartwig Anzt and Mark Gates and Jack Dongarra}
}
@conference {878,
	title = {Iterative Sparse Triangular Solves for Preconditioning},
	booktitle = {EuroPar 2015},
	year = {2015},
	month = {2015-08},
	publisher = {Springer Berlin},
	organization = {Springer Berlin},
	address = {Vienna, Austria},
	abstract = {Sparse triangular solvers are typically parallelized using level scheduling techniques, but parallel eciency is poor on high-throughput architectures like GPUs. We propose using an iterative approach for solving sparse triangular systems when an approximation is suitable. This approach will not work for all problems, but can be successful for sparse triangular matrices arising from incomplete factorizations, where an approximate solution is acceptable. We demonstrate the performance gains that this approach can have on GPUs in the context of solving sparse linear systems with a preconditioned Krylov subspace method. We also illustrate the effect of using asynchronous iterations.},
	doi = {10.1007/978-3-662-48096-0_50},
	url = {http://dx.doi.org/10.1007/978-3-662-48096-0_50},
	author = {Hartwig Anzt and Edmond Chow and Jack Dongarra}
}
@article {1347,
	title = {MAGMA MIC: Optimizing Linear Algebra for Intel Xeon Phi},
	year = {2015},
	month = {2015-06},
	publisher = {ISC High Performance (ISC15), Intel Booth Presentation},
	address = {Frankfurt, Germany},
	author = {Hartwig Anzt and Jack Dongarra and Mark Gates and Azzam Haidar and Khairul Kabir and Piotr Luszczek and Stanimire Tomov and Ichitaro Yamazaki}
}
@article {936,
	title = {Parallel Programming Models for Dense Linear Algebra on Heterogeneous Systems},
	journal = {Supercomputing Frontiers and Innovations},
	volume = {2},
	number = {4},
	year = {2015},
	month = {2015-10},
	abstract = {We present a review of the current best practices in parallel programming models for dense linear algebra (DLA) on heterogeneous architectures. We consider multicore CPUs, stand alone manycore coprocessors, GPUs, and combinations of these. Of interest is the evolution of the programming models for DLA libraries {\textendash} in particular, the evolution from the popular LAPACK and ScaLAPACK libraries to their modernized counterparts PLASMA (for multicore CPUs) and MAGMA (for heterogeneous architectures), as well as other programming models and libraries. Besides providing insights into the programming techniques of the libraries considered, we outline our view of the current strengths and weaknesses of their programming models {\textendash} especially in regards to hardware trends and ease of programming high-performance numerical software that current applications need {\textendash} in order to motivate work and future directions for the next generation of parallel programming models for high-performance linear algebra libraries on heterogeneous systems.},
	keywords = {dense linear algebra, gpu, HPC, Multicore, plasma, Programming models, runtime},
	doi = {10.14529/jsfi1504},
	author = {Maksims Abalenkovs and Ahmad Abdelfattah and Jack Dongarra and Mark Gates and Azzam Haidar and Jakub Kurzak and Piotr Luszczek and Stanimire Tomov and Ichitaro Yamazaki and Asim YarKhan}
}
@conference {893,
	title = {Random-Order Alternating Schwarz for Sparse Triangular Solves},
	booktitle = {2015 SIAM Conference on Applied Linear Algebra (SIAM LA)},
	year = {2015},
	month = {2015-10},
	publisher = {SIAM},
	organization = {SIAM},
	address = {Atlanta, GA},
	abstract = {Block-asynchronous Jacobi is an iteration method where a locally synchronous iteration is embedded in an asynchronous global iteration. The unknowns are partitioned into small subsets, and while the components within the same subset are iterated in Jacobi fashion, no update order in-between the subsets is enforced. The values of the non-local entries remain constant during the local iterations, which can result in slow inter-subset information propagation and slow convergence. Interpreting of the subsets as subdomains allows to transfer the concept of domain overlap typically enhancing the information propagation to block-asynchronous solvers. In this talk we explore the impact of overlapping domains to convergence and performance of block-asynchronous Jacobi iterations, and present results obtained by running this solver class on state-of-the-art HPC systems.},
	author = {Hartwig Anzt and Edmond Chow and Daniel Szyld and Jack Dongarra}
}
@techreport {874,
	title = {Scheduling for fault-tolerance: an introduction},
	journal = {Innovative Computing Laboratory Technical Report},
	number = {ICL-UT-15-02},
	year = {2015},
	month = {2015-01},
	publisher = {University of Tennessee},
	author = {Guillaume Aupy and Yves Robert}
}
@conference {911,
	title = {Tuning Stationary Iterative Solvers for Fault Resilience},
	booktitle = {6th Workshop on Latest Advances in Scalable Algorithms for Large-Scale Systems (ScalA15)},
	year = {2015},
	month = {2015-11},
	publisher = {ACM},
	organization = {ACM},
	address = {Austin, TX},
	abstract = {As the transistor{\textquoteright}s feature size decreases following Moore{\textquoteright}s Law, hardware will become more prone to permanent, intermittent, and transient errors, increasing the number of failures experienced by applications, and diminishing the confidence of users. As a result,
resilience is considered the most difficult under addressed issue faced by the High Performance Computing community. In this paper, we address the design of error resilient iterative solvers for sparse linear systems. Contrary to most previous approaches, based on Krylov subspace methods, for this purpose we analyze stationary component-wise relaxation. Concretely, starting from a plain implementation of the Jacobi iteration, we design a low-cost component-wise technique that elegantly handles bit-flips, turning the initial synchronized solver into an asynchronous iteration. Our experimental study employs sparse incomplete factorizations from several practical applications to expose the convergence delay incurred by the fault-tolerant implementation.},
	author = {Hartwig Anzt and Jack Dongarra and Enrique S. Quintana-Orti}
}
@techreport {837,
	title = {Accelerating the LOBPCG method on GPUs using a blocked Sparse Matrix Vector Product},
	journal = {University of Tennessee Computer Science Technical Report},
	number = {UT-EECS-14-731},
	year = {2014},
	month = {2014-10},
	publisher = {University of Tennessee},
	abstract = {This paper presents a heterogeneous CPU-GPU algorithm design and optimized implementation for an entire sparse iterative eigensolver {\textendash} the Locally Optimal Block Preconditioned Conjugate Gradient (LOBPCG) {\textendash} starting from low-level GPU data structures and kernels to the higher-level algorithmic choices and overall heterogeneous design. Most notably, the eigensolver leverages the high-performance of a new GPU kernel developed for the simultaneous multiplication of a sparse matrix and a set of vectors (SpMM). This is a building block that serves as a backbone for not only block-Krylov, but also for other methods relying on blocking for acceleration in general. The heterogeneous LOBPCG developed here reveals the potential of this type of eigensolver by highly optimizing all of its components, and can be viewed as a benchmark for other SpMM-dependent applications. Compared to non-blocked algorithms, we show that the performance speedup factor of SpMM vs. SpMV-based algorithms is up to six on GPUs like NVIDIA{\textquoteright}s K40. In particular, a typical SpMV performance range in double precision is 20 to 25 GFlop/s, while the SpMM is in the range of 100 to 120 GFlop/s. Compared to highly-optimized CPU implementations, e.g., the SpMM from MKL on two eight-core Intel Xeon E5-2690s, our kernel is 3 to 5x. faster on a K40 GPU. For comparison to other computational loads, the same GPU to CPU performance acceleration is observed for the SpMV product, as well as dense linear algebra, e.g., matrix-matrix multiplication and factorizations like LU, QR, and Cholesky. Thus, the modeled GPU (vs. CPU) acceleration for the entire solver is also 3 to 5x. In practice though, currently available CPU implementations are much slower due to missed optimization opportunities, as we show.},
	author = {Hartwig Anzt and Stanimire Tomov and Jack Dongarra}
}
@conference {812,
	title = {Hybrid Multi-Elimination ILU Preconditioners on GPUs},
	booktitle = {International Heterogeneity in Computing Workshop (HCW), IPDPS 2014},
	year = {2014},
	month = {2014-05},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Phoenix, AZ},
	abstract = {Abstract{\textemdash}Iterative solvers for sparse linear systems often benefit from using preconditioners. While there are implementations for many iterative methods that leverage the computing power of accelerators, porting the latest developments in preconditioners to accelerators has been challenging. In this paper we develop a selfadaptive multi-elimination preconditioner for graphics processing units (GPUs). The preconditioner is based on a multi-level incomplete LU factorization and uses a direct dense solver for the bottom-level system. For test matrices from the University of Florida matrix collection, we investigate the influence of handling the triangular solvers in the distinct iteration steps in either single or double precision arithmetic. Integrated into a Conjugate Gradient method, we show that our multi-elimination algorithm is highly competitive against popular preconditioners, including multi-colored symmetric Gauss-Seidel relaxation preconditioners, and (multi-colored symmetric) ILU for numerous problems.},
	author = {Dimitar Lukarski and Hartwig Anzt and Stanimire Tomov and Jack Dongarra}
}
@techreport {838,
	title = {Implementing a Sparse Matrix Vector Product for the SELL-C/SELL-C-σ formats on NVIDIA GPUs},
	journal = {University of Tennessee Computer Science Technical Report},
	number = {UT-EECS-14-727},
	year = {2014},
	month = {2014-04},
	publisher = {University of Tennessee},
	abstract = {Numerical methods in sparse linear algebra typically rely on a fast and efficient matrix vector product, as this usually is the backbone of iterative algorithms for solving eigenvalue problems or linear systems. Against the background of a large diversity in the characteristics of high performance computer architectures, it is a challenge to derive a cross-platform efficient storage format along with fast matrix vector kernels. Recently, attention focused on the SELL-C- format, a sliced ELLPACK format enhanced by row-sorting to reduce the fill in when padding rows with zeros. In this paper we propose an additional modification resulting in the padded sliced ELLPACK (SELLP) format, for which we develop a sparse matrix vector CUDA kernel that is able to efficiently exploit the computing power of NVIDIA GPUs. We show that the kernel we developed outperforms straight-forward implementations for the widespread CSR and ELLPACK formats, and is highly competitive to the implementations in the highly optimized CUSPARSE library.},
	author = {Hartwig Anzt and Stanimire Tomov and Jack Dongarra}
}
@article {825,
	title = {Improving the Energy Efficiency of Sparse Linear System Solvers on Multicore and Manycore Systems},
	journal = {Philosophical Transactions of the Royal Society A -- Mathematical, Physical and Engineering Sciences},
	volume = {372},
	year = {2014},
	month = {2014-07},
	abstract = {While most recent breakthroughs in scientific research rely on complex simulations carried out in large-scale supercomputers, the power draft and energy spent for this purpose is increasingly becoming a limiting factor to this trend. In this paper, we provide an overview of the current status in energy-efficient scientific computing by reviewing different technologies used to monitor power draft as well as power- and energy-saving mechanisms available in commodity hardware. For the particular domain of sparse linear algebra, we analyze the energy efficiency of a broad collection of hardware architectures and investigate how algorithmic and implementation modifications can improve the energy performance of sparse linear system solvers, without negatively impacting their performance.},
	keywords = {energy efficiency, graphics processing units, High Performance Computing, iterative solvers, multicore processors, sparse linear systems},
	doi = {10.1098/rsta.2013.0279},
	author = {Hartwig Anzt and Enrique S. Quintana-Orti}
}
@conference {807,
	title = {Improving the performance of CA-GMRES on multicores with multiple GPUs},
	booktitle = {IPDPS 2014},
	year = {2014},
	month = {2014-05},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Phoenix, AZ},
	abstract = {Abstract{\textemdash}The Generalized Minimum Residual (GMRES) method is one of the most widely-used iterative methods for solving nonsymmetric linear systems of equations. In recent years, techniques to avoid communication in GMRES have gained attention because in comparison to floating-point operations, communication is becoming increasingly expensive on modern computers. Since graphics processing units (GPUs) are now becoming crucial component in computing, we investigate the effectiveness of these techniques on multicore CPUs with multiple GPUs. While we present the detailed performance studies of a matrix powers kernel on multiple GPUs, we particularly focus on orthogonalization strategies that have a great impact on both the numerical stability and performance of GMRES, especially as the matrix becomes sparser or ill-conditioned. We present the experimental results on two eight-core Intel Sandy Bridge CPUs with three NDIVIA Fermi GPUs and demonstrate that significant speedups can be obtained by avoiding communication, either on a GPU or between the GPUs. As part of our study, we investigate several optimization techniques for the GPU kernels that can also be used in other iterative solvers besides GMRES. Hence, our studies not only emphasize the importance of avoiding communication on GPUs, but they also provide insight about the effects of these optimization techniques on the performance of the sparse solvers, and may have greater impact beyond GMRES.},
	author = {Ichitaro Yamazaki and Hartwig Anzt and Stanimire Tomov and Mark Hoemmen and Jack Dongarra}
}
@conference {833,
	title = {Optimizing Krylov Subspace Solvers on Graphics Processing Units},
	booktitle = {Fourth International Workshop on Accelerators and Hybrid Exascale Systems (AsHES), IPDPS 2014},
	year = {2014},
	month = {2014-05},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Phoenix, AZ},
	abstract = {Krylov subspace solvers are often the method of choice when solving sparse linear systems iteratively. At the same time, hardware accelerators such as graphics processing units (GPUs) continue to offer significant floating point performance gains for matrix and vector computations through easy-to-use libraries of computational kernels. However, as these libraries are usually composed of a well optimized but limited set of linear algebra operations, applications that use them often fail to leverage the full potential of the accelerator. In this paper we target the acceleration of the BiCGSTAB solver for GPUs, showing that significant improvement can be achieved by reformulating the method and developing application-specific kernels instead of using the generic CUBLAS library provided by NVIDIA. We propose an implementation that benefits from a significantly reduced number of kernel launches and GPUhost communication events, by means of increased data locality and a simultaneous reduction of multiple scalar products. Using experimental data, we show that, depending on the dominance of the untouched sparse matrix vector products, significant performance improvements can be achieved compared to a reference implementation based on the CUBLAS library. We feel that such optimizations are crucial for the subsequent development of highlevel sparse linear algebra libraries.},
	author = {Stanimire Tomov and Piotr Luszczek and Ichitaro Yamazaki and Jack Dongarra and Hartwig Anzt and William Sawyer}
}
@conference {714,
	title = {Self-Adaptive Multiprecision Preconditioners on Multicore and Manycore Architectures},
	booktitle = {VECPAR 2014},
	year = {2014},
	month = {2014-06},
	address = {Eugene, OR},
	abstract = {Based on the premise that preconditioners needed for scientific computing are not only required to be robust in the numerical sense, but also scalable for up to thousands of light-weight cores, we argue that this two-fold goal is achieved for the recently developed self-adaptive multi-elimination preconditioner. For this purpose, we revise the underlying idea and analyze the performance of implementations realized in the PARALUTION and MAGMA open-source software libraries on GPU architectures (using either CUDA or OpenCL), Intel{\textquoteright}s Many Integrated Core Architecture, and Intel{\textquoteright}s Sandy Bridge processor. The comparison with other well-established preconditioners like multi-coloured Gauss-Seidel, ILU(0) and multi-colored ILU(0), shows that the twofold goal of a numerically stable cross-platform performant algorithm is achieved.},
	author = {Hartwig Anzt and Dimitar Lukarski and Stanimire Tomov and Jack Dongarra}
}
@conference {863,
	title = {Task-Based Programming for Seismic Imaging: Preliminary Results},
	booktitle = {2014 IEEE International Conference on High Performance Computing and Communications (HPCC)},
	year = {2014},
	month = {2014-08},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Paris, France},
	abstract = {The level of hardware complexity of current supercomputers is forcing the High Performance Computing (HPC) community to reconsider parallel programming paradigms and standards. The high-level of hardware abstraction provided by task-based paradigms make them excellent candidates for writing portable codes that can consistently deliver high performance across a wide range of platforms. While this paradigm has proved efficient for achieving such goals for dense and sparse linear solvers, it is yet to be demonstrated that industrial parallel codes{\textemdash}relying on the classical Message Passing Interface (MPI) standard and that accumulate dozens of years of expertise (and countless lines of code){\textemdash}may be revisited to turn them into efficient task-based programs. In this paper, we study the applicability of task-based programming in the case of a Reverse Time Migration (RTM) application for Seismic Imaging. The initial MPI-based application is turned into a task-based code executed on top of the PaRSEC runtime system. Preliminary results show that the approach is competitive with (and even potentially superior to) the original MPI code on a homogeneous multicore node, and can more efficiently exploit complex hardware such as a cache coherent Non Uniform Memory Access (ccNUMA) node or an Intel Xeon Phi accelerator.},
	keywords = {plasma},
	author = {Lionel Boillot and George Bosilca and Emmanuel Agullo and Henri Calandra}
}
@article {826,
	title = {Unveiling the Performance-energy Trade-off in Iterative Linear System Solvers for Multithreaded Processors},
	journal = {Concurrency and Computation: Practice and Experience},
	volume = {27},
	year = {2014},
	month = {2014-09},
	pages = {885-904},
	chapter = {885},
	abstract = {In this paper, we analyze the interactions occurring in the triangle performance-power-energy for the execution of a pivotal numerical algorithm, the iterative conjugate gradient (CG) method, on a diverse collection of parallel multithreaded architectures. This analysis is especially timely in a decade where the power wall has arisen as a major obstacle to build faster processors. Moreover, the CG method has recently been proposed as a complement to the LINPACK benchmark, as this iterative method is argued to be more archetypical of the performance of today{\textquoteright}s scientific and engineering applications. To gain insights about the benefits of hands-on optimizations we include runtime and energy efficiency results for both out-of-the-box usage relying exclusively on compiler optimizations, and implementations manually optimized for target architectures, that range from general-purpose and digital signal multicore processors to manycore graphics processing units, all representative of current multithreaded systems.},
	keywords = {CG, CPUs, energy efficiency, GPUs, low-power architectures},
	doi = {10.1002/cpe.3341},
	url = {http://dx.doi.org/10.1002/cpe.3341},
	author = {Jos{\'e} I. Aliaga and Hartwig Anzt and Maribel Castillo and Juan C. Fern{\'a}ndez and Germ{\'a}n Le{\'o}n and Joaqu{\'\i}n P{\'e}rez and Enrique S. Quintana-Orti}
}
@article {icl:719,
	title = {A Block-Asynchronous Relaxation Method for Graphics Processing Units},
	journal = {Journal of Parallel and Distributed Computing},
	volume = {73},
	year = {2013},
	month = {2013-12},
	pages = {1613{\textendash}1626},
	abstract = {In this paper, we analyze the potential of asynchronous relaxation methods on Graphics Processing Units (GPUs). We develop asynchronous iteration algorithms in CUDA and compare them with parallel implementations of synchronous relaxation methods on CPU- or GPU-based systems. For a set of test matrices from UFMC we investigate convergence behavior, performance and tolerance to hardware failure. We observe that even for our most basic asynchronous relaxation scheme, the method can efficiently leverage the GPUs computing power and is, despite its lower convergence rate compared to the Gauss{\textendash}Seidel relaxation, still able to provide solution approximations of certain accuracy in considerably shorter time than Gauss{\textendash}Seidel running on CPUs- or GPU-based Jacobi. Hence, it overcompensates for the slower convergence by exploiting the scalability and the good fit of the asynchronous schemes for the highly parallel GPU architectures. Further, enhancing the most basic asynchronous approach with hybrid schemes{\textendash}using multiple iterations within the {\textquoteleft}{\textquoteleft}subdomain{\textquoteright}{\textquoteright} handled by a GPU thread block{\textendash}we manage to not only recover the loss of global convergence but often accelerate convergence of up to two times, while keeping the execution time of a global iteration practically the same. The combination with the advantageous properties of asynchronous iteration methods with respect to hardware failure identifies the high potential of the asynchronous methods for Exascale computing.},
	doi = {http://dx.doi.org/10.1016/j.jpdc.2013.05.008},
	author = {Hartwig Anzt and Stanimire Tomov and Jack Dongarra and Vincent Heuveline}
}
@techreport {684,
	title = {On the Combination of Silent Error Detection and Checkpointing},
	journal = {UT-CS-13-710},
	year = {2013},
	month = {2013-06},
	publisher = {University of Tennessee Computer Science Technical Report},
	abstract = {In this paper, we revisit traditional checkpointing and rollback recovery strategies, with a focus on silent data corruption errors. Contrarily to fail-stop failures, such latent errors cannot be detected immediately, and a mechanism to detect them must be provided. We consider two models: (i) errors are detected after some delays following a probability distribution (typically, an Exponential distribution); (ii) errors are detected through some verification mechanism. In both cases, we compute the optimal period in order to minimize the waste, i.e., the fraction of time where nodes do not perform useful computations. In practice, only a fixed number of checkpoints can be kept in memory, and the first model may lead to an irrecoverable failure. In this case, we compute the minimum period required for an acceptable risk. For the second model, there is no risk of irrecoverable failure, owing to the verification mechanism, but the corresponding overhead is included in the waste. Finally, both models are instantiated using realistic scenarios and application/architecture parameters.},
	keywords = {checkpointing, error recovery, High-performance computing, silent data corruption, verification},
	url = {http://www.netlib.org/lapack/lawnspdf/lawn278.pdf},
	author = {Guillaume Aupy and Anne Benoit and Thomas Herault and Yves Robert and Frederic Vivien and Dounia Zaidouni}
}
@techreport {682,
	title = {Implementing a systolic algorithm for QR factorization on multicore clusters with PaRSEC},
	journal = {Lawn 277},
	number = {UT-CS-13-709},
	year = {2013},
	month = {2013-05},
	abstract = {This article introduces a new systolic algorithm for QR factorization, and its implementation on a supercomputing cluster of multicore nodes. The algorithm targets a virtual 3D-array and requires only local communications. The implementation of the algorithm uses threads at the node level, and MPI for inter-node communications. The complexity of the implementation is addressed with the PaRSEC software, which takes as input a parametrized dependence graph, which is derived from the algorithm, and only requires the user to decide, at the high-level, the allocation of tasks to nodes. We show that the new algorithm exhibits competitive performance with state-of-the-art QR routines on a supercomputer called Kraken, which shows that high-level programming environments, such as PaRSEC, provide a viable alternative to enhance the production of quality software on complex and hierarchical architectures},
	author = {Guillaume Aupy and Mathieu Faverge and Yves Robert and Jakub Kurzak and Piotr Luszczek and Jack Dongarra}
}
@article {icl:704,
	title = {Multithreading in the PLASMA Library},
	journal = {Multi and Many-Core Processing: Architecture, Programming, Algorithms, \& Applications},
	year = {2013},
	month = {2013-00},
	publisher = {Taylor \& Francis},
	keywords = {plasma},
	author = {Jakub Kurzak and Piotr Luszczek and Asim YarKhan and Mathieu Faverge and Julien Langou and Henricus Bouwmeester and Jack Dongarra},
	editor = {Mohamed Ahmed and Reda Ammar and Sanguthevar Rajasekaran}
}
@techreport {702,
	title = {Optimal Checkpointing Period: Time vs. Energy},
	journal = {University of Tennessee Computer Science Technical Report (also LAWN 281)},
	number = {ut-eecs-13-718},
	year = {2013},
	month = {2013-10},
	publisher = {University of Tennessee},
	author = {Guillaume Aupy and Anne Benoit and Thomas Herault and Yves Robert and Jack Dongarra}
}
@article {icl:697,
	title = {Block-asynchronous Multigrid Smoothers for GPU-accelerated Systems},
	journal = {ICCS 2012},
	year = {2012},
	month = {2012-06},
	address = {Omaha, NE},
	author = {Hartwig Anzt and Stanimire Tomov and Mark Gates and Jack Dongarra and Vincent Heuveline}
}
@article {icl:723,
	title = {GPU-Accelerated Asynchronous Error Correction for Mixed Precision Iterative Refinement},
	journal = {EuroPar 2012 (also LAWN 260)},
	year = {2012},
	month = {2012-08},
	address = {Rhodes Island, Greece},
	author = {Hartwig Anzt and Piotr Luszczek and Jack Dongarra and Vincent Heuveline}
}
@article {icl:730,
	title = {Matrices Over Runtime Systems at Exascale},
	journal = {Supercomputing {\textquoteright}12 (poster)},
	year = {2012},
	month = {2012-11},
	address = {Salt Lake City, Utah},
	author = {Emmanuel Agullo and George Bosilca and Cedric Castagn{\`e}de and Jack Dongarra and Hatem Ltaeif and Stanimire Tomov}
}
@article {icl:709,
	title = {Optimizing Memory-Bound Numerical Kernels on GPU Hardware Accelerators},
	journal = {VECPAR 2012},
	year = {2012},
	month = {2012-07},
	address = {Kobe, Japan},
	author = {Ahmad Abdelfattah and Jack Dongarra and David Keyes and Hatem Ltaeif}
}
@inproceedings {icl:736,
	title = {User Level Failure Mitigation in MPI},
	journal = {Euro-Par 2012: Parallel Processing Workshops},
	volume = {7640},
	year = {2012},
	month = {2012-08},
	pages = {499-504},
	publisher = {Springer Berlin Heidelberg},
	address = {Rhodes Island, Greece},
	keywords = {ftmpi},
	author = {Wesley Bland},
	editor = {Ioannis Caragiannis and Michael Alexander and Rosa M. Badia and Mario Cannataro and Alexandru Costan and Marco Danelutto and Frederic Desprez and Bettina Krammer and Sahuquillo, J. and Stephen L. Scott and J. Weidendorfer}
}
@inproceedings {icl:713,
	title = {Weighted Block-Asynchronous Iteration on GPU-Accelerated Systems},
	journal = {Tenth International Workshop on Algorithms, Models and Tools for Parallel Computing on Heterogeneous Platforms (Best Paper)},
	year = {2012},
	month = {2012-08},
	address = {Rhodes Island, Greece},
	author = {Hartwig Anzt and Stanimire Tomov and Jack Dongarra and Vincent Heuveline}
}
@article {icl:701,
	title = {Weighted Block-Asynchronous Relaxation for GPU-Accelerated Systems},
	journal = {SIAM Journal on Computing (submitted)},
	year = {2012},
	month = {2012-03},
	author = {Hartwig Anzt and Jack Dongarra and Vincent Heuveline}
}
@inproceedings {icl:605,
	title = {Algebraic Schwarz Preconditioning for the Schur Complement: Application to the Time-Harmonic Maxwell Equations Discretized by a Discontinuous Galerkin Method.},
	journal = {The Twentieth International Conference on Domain Decomposition Methods},
	year = {2011},
	month = {2011-02},
	address = {La Jolla, California},
	url = {http://hal.inria.fr/inria-00577639},
	author = {Emmanuel Agullo and Luc Giraud and Amina Guermouche and Azzam Haidar and Stephane Lanteri and Jean Roman}
}
@article {icl:661,
	title = {Block-asynchronous Multigrid Smoothers for GPU-accelerated Systems},
	number = {UT-CS-11-689},
	year = {2011},
	month = {2011-12},
	keywords = {magma},
	author = {Hartwig Anzt and Stanimire Tomov and Mark Gates and Jack Dongarra and Vincent Heuveline}
}
@techreport {icl:656,
	title = {A Block-Asynchronous Relaxation Method for Graphics Processing Units},
	journal = {University of Tennessee Computer Science Technical Report},
	number = {UT-CS-11-687 / LAWN 258},
	year = {2011},
	month = {2011-11},
	keywords = {magma},
	author = {Hartwig Anzt and Stanimire Tomov and Jack Dongarra and Vincent Heuveline}
}
@techreport {icl:662,
	title = {GPU-Accelerated Asynchronous Error Correction for Mixed Precision Iterative Refinement},
	journal = {University of Tennessee Computer Science Technical Report UT-CS-11-690 (also Lawn 260)},
	year = {2011},
	month = {2011-12},
	keywords = {magma},
	author = {Hartwig Anzt and Piotr Luszczek and Jack Dongarra and Vincent Heuveline}
}
@article {icl:653,
	title = {A Hybridization Methodology for High-Performance Linear Algebra Software for GPUs},
	journal = {in GPU Computing Gems, Jade Edition},
	volume = {2},
	year = {2011},
	month = {2011-00},
	pages = {473-484},
	publisher = {Elsevier},
	keywords = {magma, morse},
	author = {Emmanuel Agullo and Cedric Augonnet and Jack Dongarra and Hatem Ltaeif and Raymond Namyst and Samuel Thibault and Stanimire Tomov},
	editor = {Wen-mei W. Hwu}
}
@article {icl:643,
	title = {The International Exascale Software Project Roadmap},
	journal = {International Journal of High Performance Computing},
	volume = {25},
	number = {1},
	year = {2011},
	month = {2011-01},
	pages = {3-60},
	abstract = {Over the last 20 years, the open-source community has provided more and more software on which the world{\textquoteright}s high-performance computing systems depend for performance and productivity. The community has invested millions of dollars and years of effort to build key components. However, although the investments in these separate software elements have been tremendously valuable, a great deal of productivity has also been lost because of the lack of planning, coordination, and key integration of technologies necessary to make them work together smoothly and efficiently, both within individual petascale systems and between different systems. It seems clear that this completely uncoordinated development model will not provide the software needed to support the unprecedented parallelism required for peta/ exascale computation on millions of cores, or the flexibility required to exploit new hardware models and features, such as transactional memory, speculative execution, and graphics processing units. This report describes the work of the community to prepare for the challenges of exascale computing, ultimately combing their efforts in a coordinated International Exascale Software Project.},
	doi = {https://doi.org/10.1177/1094342010391989},
	author = {Jack Dongarra and Pete Beckman and Terry Moore and Patrick Aerts and Giovanni Aloisio and Jean-Claude Andre and David Barkai and Jean-Yves Berthou and Taisuke Boku and Bertrand Braunschweig and Franck Cappello and Barbara Chapman and Xuebin Chi and Alok Choudhary and Sudip Dosanjh and Thom Dunning and Sandro Fiore and Al Geist and Bill Gropp and Robert Harrison and Mark Hereld and Michael Heroux and Adolfy Hoisie and Koh Hotta and Zhong Jin and Yutaka Ishikawa and Fred Johnson and Sanjay Kale and Richard Kenway and David Keyes and Bill Kramer and Jesus Labarta and Alain Lichnewsky and Thomas Lippert and Bob Lucas and Barney MacCabe and Satoshi Matsuoka and Paul Messina and Peter Michielse and Bernd Mohr and Matthias S. Mueller and Wolfgang E. Nagel and Hiroshi Nakashima and Michael E. Papka and Dan Reed and Mitsuhisa Sato and Ed Seidel and John Shalf and David Skinner and Marc Snir and Thomas Sterling and Rick Stevens and Fred Streitz and Bob Sugar and Shinji Sumimoto and William Tang and John Taylor and Rajeev Thakur and Anne Trefethen and Mateo Valero and Aad van der Steen and Jeffrey Vetter and Peg Williams and Robert Wisniewski and Kathy Yelick}
}
@article {icl:599,
	title = {LU Factorization for Accelerator-Based Systems},
	journal = {IEEE/ACS AICCSA 2011},
	year = {2011},
	month = {2011-12},
	address = {Sharm-El-Sheikh, Egypt},
	keywords = {magma, morse},
	author = {Emmanuel Agullo and Cedric Augonnet and Jack Dongarra and Mathieu Faverge and Julien Langou and Hatem Ltaeif and Stanimire Tomov}
}
@article {icl:606,
	title = {Parallel algebraic domain decomposition solver for the solution of augmented systems.},
	journal = {Parallel, Distributed, Grid and Cloud Computing for Engineering, Ajaccio, Corsica, France, 12-15 April},
	year = {2011},
	month = {2011-00},
	author = {Emmanuel Agullo and Luc Giraud and Amina Guermouche and Azzam Haidar and Jean Roman}
}
@article {icl:677,
	title = {QCG-OMPI: MPI Applications on Grids.},
	journal = {Future Generation Computer Systems},
	volume = {27},
	number = {4},
	year = {2011},
	month = {2011-01},
	pages = {435-369},
	author = {Emmanuel Agullo and Camille Coti and Thomas Herault and Julien Langou and Sylvain Peyronnet and A. Rezmerita and Franck Cappello and Jack Dongarra}
}
@article {1363,
	title = {Accelerating Linear Algebra on Heterogeneous Architectures of Multicore and GPUs using MAGMA and DPLASMA and StarPU Schedulers},
	year = {2010},
	month = {2010-07},
	publisher = {2010 Symposium on Application Accelerators in. High-Performance Computing (SAAHPC{\textquoteright}10), Tutorial},
	author = {Stanimire Tomov and George Bosilca and Cedric Augonnet}
}
@article {1364,
	title = {Autotuning Dense Linear Algebra Libraries on GPUs},
	year = {2010},
	month = {2010-06},
	publisher = {Sixth International Workshop on Parallel Matrix Algorithms and Applications (PMAA 2010)},
	address = {Basel, Switzerland},
	author = {Rajib Nath and Stanimire Tomov and Emmanuel Agullo and Jack Dongarra}
}
@techreport {icl:585,
	title = {Faster, Cheaper, Better - A Hybridization Methodology to Develop Linear Algebra Software for GPUs},
	journal = {LAPACK Working Note},
	number = {230},
	year = {2010},
	month = {2010-00},
	keywords = {magma, morse},
	author = {Emmanuel Agullo and Cedric Augonnet and Jack Dongarra and Hatem Ltaeif and Raymond Namyst and Samuel Thibault and Stanimire Tomov}
}
@article {icl:553,
	title = {MaPHyS or the Development of a Parallel Algebraic Domain Decomposition Solver in the Course of the Solstice Project},
	journal = {Sparse Days 2010 Meeting at CERFACS},
	year = {2010},
	month = {2010-06},
	address = {Toulouse, France},
	author = {Emmanuel Agullo and Luc Giraud and Amina Guermouche and Azzam Haidar and Jean Roman and Yohan Lee-Tin-Yien}
}
@article {icl:579,
	title = {Proceedings of the International Conference on Computational Science},
	journal = {ICCS 2010},
	year = {2010},
	month = {2010-05},
	publisher = {Elsevier},
	address = {Amsterdam},
	editor = {Peter M. Sloot and Geert Dick van Albada and Jack Dongarra}
}
@article {icl:574,
	title = {QCG-OMPI: MPI Applications on Grids},
	journal = {Future Generation Computer Systems},
	volume = {27},
	number = {4},
	year = {2010},
	month = {2010-03},
	pages = {357-369},
	author = {Emmanuel Agullo and Camille Coti and Thomas Herault and Julien Langou and Sylvain Peyronnet and A. Rezmerita and Franck Cappello and Jack Dongarra}
}
@inproceedings {icl:532,
	title = {QR Factorization of Tall and Skinny Matrices in a Grid Computing Environment},
	journal = {24th IEEE International Parallel and Distributed Processing Symposium (also LAWN 224)},
	year = {2010},
	month = {2010-04},
	address = {Atlanta, GA},
	author = {Emmanuel Agullo and Camille Coti and Jack Dongarra and Thomas Herault and Julien Langou}
}
@inproceedings {icl:577,
	title = {QR Factorization on a Multicore Node Enhanced with Multiple GPU Accelerators},
	journal = {Proceedings of IPDPS 2011},
	number = {ICL-UT-10-04},
	year = {2010},
	month = {2010-10},
	address = {Anchorage, AK},
	keywords = {magma, morse, plasma},
	author = {Emmanuel Agullo and Cedric Augonnet and Jack Dongarra and Mathieu Faverge and Hatem Ltaeif and Samuel Thibault and Stanimire Tomov}
}
@article {1362,
	title = {Scheduling Cholesky Factorization on Multicore Architectures with GPU Accelerators},
	year = {2010},
	month = {2010-07},
	publisher = {2010 Symposium on Application Accelerators in High-Performance Computing (SAAHPC{\textquoteright}10), Poster},
	address = {Knoxville, TN},
	author = {Emmanuel Agullo and Cedric Augonnet and Jack Dongarra and Hatem Ltaeif and Raymond Namyst and Rajib Nath and Jean Roman and Samuel Thibault and Stanimire Tomov}
}
@article {icl:567,
	title = {Self-Healing Network for Scalable Fault-Tolerant Runtime Environments},
	journal = {Future Generation Computer Systems},
	volume = {26},
	number = {3},
	year = {2010},
	month = {2010-03},
	pages = {479-485},
	author = {Thara Angskun and Graham Fagg and George Bosilca and Jelena Pjesivac{\textendash}Grbovic and Jack Dongarra}
}
@inproceedings {icl:588,
	title = {Tile QR Factorization with Parallel Panel Processing for Multicore Architectures},
	journal = {24th IEEE International Parallel and Distributed Processing Symposium (submitted)},
	year = {2010},
	month = {2010-00},
	author = {Bilel Hadri and Emmanuel Agullo and Jack Dongarra}
}
@article {icl:552,
	title = {Towards a Complexity Analysis of Sparse Hybrid Linear Solvers},
	journal = {PARA 2010},
	year = {2010},
	month = {2010-06},
	address = {Reykjavik, Iceland},
	author = {Emmanuel Agullo and Luc Giraud and Amina Guermouche and Azzam Haidar and Jean Roman}
}
@inproceedings {icl:488,
	title = {Comparative Study of One-Sided Factorizations with Multiple Software Packages on Multi-Core Hardware},
	journal = {2009 International Conference for High Performance Computing, Networking, Storage, and Analysis (SC {\textquoteright}09) (to appear)},
	year = {2009},
	month = {2009-00},
	author = {Emmanuel Agullo and Bilel Hadri and Hatem Ltaeif and Jack Dongarra}
}
@article {icl:507,
	title = {Computational Science {\textendash} ICCS 2009, Proceedings of the 9th International Conference},
	journal = {Lecture Notes in Computer Science: Theoretical Computer Science and General Issues},
	volume = {-},
	number = {5544-5545},
	year = {2009},
	month = {2009-05},
	address = {Baton Rouge, LA},
	editor = {Gabrielle Allen and Jaros{\l}aw Nabrzyski and E. Seidel and Geert Dick van Albada and Jack Dongarra and Peter M. Sloot}
}
@article {icl:524,
	title = {Enhancing Parallelism of Tile QR Factorization for Multicore Architectures},
	journal = {Submitted to Transaction on Parallel and Distributed Systems},
	year = {2009},
	month = {2009-12},
	keywords = {plasma},
	author = {Bilel Hadri and Hatem Ltaeif and Emmanuel Agullo and Jack Dongarra}
}
@inproceedings {icl:474,
	title = {A Holistic Approach for Performance Measurement and Analysis for Petascale Applications},
	journal = {ICCS 2009 Joint Workshop: Tools for Program Development and Analysis in Computational Science and Software Engineering for Large-Scale Computing},
	volume = {2009},
	year = {2009},
	month = {2009-05},
	pages = {686-695},
	publisher = {Springer-Verlag Berlin Heidelberg 2009},
	address = {Baton Rouge, Louisiana},
	keywords = {point, test},
	author = {Heike Jagode and Jack Dongarra and Sadaf Alam and Jeffrey Vetter and W. Spear and Allen D. Malony},
	editor = {Gabrielle Allen}
}
@article {icl:480,
	title = {Impact of Quad-core Cray XT4 System and Software Stack on Scientific Computation},
	journal = {Euro-Par 2009, Lecture Notes in Computer Science},
	volume = {5704/2009},
	year = {2009},
	month = {2009-08},
	pages = {334-344},
	publisher = {Springer Berlin / Heidelberg},
	address = {Delft, The Netherlands},
	keywords = {test},
	author = {Sadaf Alam and Richard F. Barrett and Heike Jagode and J. A. Kuehn and Steve W. Poole and R. Sankaran}
}
@article {icl:481,
	title = {The International Exascale Software Project: A Call to Cooperative Action by the Global High Performance Community},
	journal = {International Journal of High Performance Computing Applications (to appear)},
	year = {2009},
	month = {2009-07},
	author = {Jack Dongarra and Pete Beckman and Patrick Aerts and Franck Cappello and Thomas Lippert and Satoshi Matsuoka and Paul Messina and Terry Moore and Rick Stevens and Anne Trefethen and Mateo Valero}
}
@inproceedings {icl:602,
	title = {Modeling the Office of Science Ten Year Facilities Plan: The PERI Architecture Tiger Team},
	journal = {SciDAC 2009, Journal of Physics: Conference Series},
	volume = {180(2009)012039},
	year = {2009},
	month = {2009-07},
	publisher = {IOP Publishing},
	address = {San Diego, California},
	keywords = {test},
	author = {Bronis R. de Supinski and Sadaf Alam and David Bailey and Laura Carrington and Chris Daley and Anshu Dubey and Todd Gamblin and Dan Gunter and Paul D. Hovland and Heike Jagode and Karen Karavanic and Gabriel Marin and John Mellor-Crummey and Shirley Moore and Boyana Norris and Leonid Oliker and Catherine Olschanowsky and Philip C. Roth and Martin Schulz and Sameer Shende and Allan Snavely}
}
@inproceedings {icl:512,
	title = {A Note on Auto-tuning GEMM for GPUs},
	journal = {9th International Conference on Computational Science (ICCS 2009)},
	number = {5544-5545},
	year = {2009},
	month = {2009-05},
	pages = {884-892},
	address = {Baton Rouge, LA},
	doi = {10.1007/978-3-642-01970-8_89},
	author = {Yinan Li and Jack Dongarra and Stanimire Tomov},
	editor = {Gabrielle Allen and Jaros{\l}aw Nabrzyski and E. Seidel and Geert Dick van Albada and Jack Dongarra and Peter M. Sloot}
}
@article {1352,
	title = {Numerical Linear Algebra on Emerging Architectures: The PLASMA and MAGMA Projects},
	year = {2009},
	month = {2009-11},
	publisher = {The International Conference for High Performance Computing, Networking, Storage, and Analysis (SC09)},
	address = {Portland, OR},
	author = {Emmanuel Agullo and James Demmel and Jack Dongarra and Bilel Hadri and Jakub Kurzak and Julien Langou and Hatem Ltaeif and Piotr Luszczek and Rajib Nath and Stanimire Tomov and Asim YarKhan and Vasily Volkov}
}
@inproceedings {icl:486,
	title = {Numerical Linear Algebra on Emerging Architectures: The PLASMA and MAGMA Projects},
	journal = {Journal of Physics: Conference Series},
	volume = {180},
	year = {2009},
	month = {2009-00},
	keywords = {magma, plasma},
	author = {Emmanuel Agullo and James Demmel and Jack Dongarra and Bilel Hadri and Jakub Kurzak and Julien Langou and Hatem Ltaeif and Piotr Luszczek and Stanimire Tomov}
}
@article {icl:492,
	title = {Optimizing Matrix Multiplication for a Short-Vector SIMD Architecture - CELL Processor},
	journal = {Parallel Computing},
	volume = {35},
	year = {2009},
	month = {2009-00},
	pages = {138-150},
	author = {Wesley Alvaro and Jakub Kurzak and Jack Dongarra}
}
@techreport {icl:487,
	title = {Tall and Skinny QR Matrix Factorization Using Tile Algorithms on Multicore Architectures},
	journal = {Innovative Computing Laboratory Technical Report (also LAPACK Working Note 222 and CS Tech Report UT-CS-09-645)},
	number = {ICL-UT-09-03},
	year = {2009},
	month = {2009-09},
	keywords = {plasma},
	author = {Bilel Hadri and Hatem Ltaeif and Emmanuel Agullo and Jack Dongarra}
}
@inproceedings {icl:522,
	title = {Tile QR Factorization with Parallel Panel Processing for Multicore Architectures},
	journal = {accepted in 24th IEEE International Parallel and Distributed Processing Symposium (IPDPS 2010)},
	year = {2009},
	month = {2009-12},
	address = {Atlanta, GA},
	keywords = {plasma},
	author = {Bilel Hadri and Hatem Ltaeif and Emmanuel Agullo and Jack Dongarra}
}
@article {icl:515,
	title = {Transparent Cross-Platform Access to Software Services using GridSolve and GridRPC},
	journal = {in Cloud Computing and Software Services: Theory and Techniques (to appear)},
	year = {2009},
	month = {2009-00},
	publisher = {CRC Press},
	keywords = {netsolve},
	author = {Keith Seymour and Asim YarKhan and Jack Dongarra},
	editor = {Syed Ahson and Mohammad Ilyas}
}
@inproceedings {icl:453,
	journal = {8th International Conference on Computational Science (ICCS), Proceedings Parts I, II, and III, Lecture Notes in Computer Science},
	volume = {5101},
	year = {2008},
	month = {2008-01},
	publisher = {Springer Berlin},
	address = {Krakow, Poland},
	editor = {Marian Bubak and Geert Dick van Albada and Jack Dongarra and Peter M. Sloot}
}
@article {icl:451,
	title = {DARPA{\textquoteright}s HPCS Program: History, Models, Tools, Languages},
	journal = {in Advances in Computers},
	volume = {72},
	year = {2008},
	month = {2008-01},
	publisher = {Elsevier},
	author = {Jack Dongarra and Robert Graybill and William Harrod and Robert Lucas and Ewing Lusk and Piotr Luszczek and Janice McMahon and Allan Snavely and Jeffrey Vetter and Katherine Yelick and Sadaf Alam and Roy Campbell and Laura Carrington and Tzu-Yi Chen and Omid Khalili and Jeremy Meredith and Mustafa Tikir},
	editor = {M. Zelkowitz}
}
@inproceedings {icl:440,
	title = {Exploring New Architectures in Accelerating CFD for Air Force Applications},
	journal = {Proceedings of the DoD HPCMP User Group Conference},
	year = {2008},
	month = {2008-01},
	address = {Seattle, Washington},
	keywords = {magma},
	author = {Jack Dongarra and Shirley Moore and Gregory D. Peterson and Stanimire Tomov and Jeff Allred and Vincent Natoli and David Richie}
}
@techreport {icl:405,
	title = {Fast and Small Short Vector SIMD Matrix Multiplication Kernels for the CELL Processor},
	journal = {University of Tennessee Computer Science Technical Report},
	number = {UT-CS-08-609, (also LAPACK Working Note 189)},
	year = {2008},
	month = {2008-01},
	keywords = {plasma},
	author = {Wesley Alvaro and Jakub Kurzak and Jack Dongarra}
}
@article {icl:409,
	title = {High Performance GridRPC Middleware},
	journal = {Recent developments in Grid Technology and Applications},
	year = {2008},
	month = {2008-00},
	publisher = {Nova Science Publishers},
	keywords = {netsolve},
	author = {Yves Caniou and Eddy Caron and Frederic Desprez and Hidemoto Nakada and Yoshio Tanaka and Keith Seymour},
	editor = {George A. Gravvanis and John P. Morrison and Hamid R. Arabnia and D. A. Power}
}
@inproceedings {icl:412,
	title = {Usage of the Scalasca Toolset for Scalable Performance Analysis of Large-scale Parallel Applications},
	journal = {Proceedings of the 2nd International Workshop on Tools for High Performance Computing},
	year = {2008},
	month = {2008-01},
	pages = {157-167},
	publisher = {Springer},
	address = {Stuttgart, Germany},
	keywords = {point},
	author = {Felix Wolf and Brian Wylie and Erika Abraham and Wolfgang Frings and Karl F{\"u}rlinger and Markus Geimer and Marc-Andre Hermanns and Bernd Mohr and Shirley Moore and Matthias Pfeifer},
	editor = {Michael Resch and Rainer Keller and Valentin Himmler and Bettina Krammer and A Schulz}
}
@inproceedings {icl:355,
	title = {Binomial Graph: A Scalable and Fault- Tolerant Logical Network Topology},
	journal = {Proceedings of The Fifth International Symposium on Parallel and Distributed Processing and Applications (ISPA07)},
	year = {2007},
	month = {2007-08},
	publisher = {Springer},
	address = {Niagara Falls, Canada},
	keywords = {ftmpi},
	author = {Thara Angskun and George Bosilca and Jack Dongarra}
}
@article {icl:357,
	title = {Decision Trees and MPI Collective Algorithm Selection Problem},
	journal = {Euro-Par 2007},
	year = {2007},
	month = {2007-08},
	pages = {105{\textendash}115},
	publisher = {Springer},
	address = {Rennes, France},
	keywords = {ftmpi},
	author = {Jelena Pjesivac{\textendash}Grbovic and George Bosilca and Graham Fagg and Thara Angskun and Jack Dongarra}
}
@article {icl:356,
	title = {MPI Collective Algorithm Selection and Quadtree Encoding},
	journal = {Parallel Computing (Special Edition: EuroPVM/MPI 2006)},
	year = {2007},
	month = {2007-00},
	publisher = {Elsevier},
	keywords = {ftmpi},
	author = {Jelena Pjesivac{\textendash}Grbovic and George Bosilca and Graham Fagg and Thara Angskun and Jack Dongarra}
}
@inproceedings {icl:374,
	title = {Optimal Routing in Binomial Graph Networks},
	journal = {The International Conference on Parallel and Distributed Computing, applications and Technologies (PDCAT)},
	year = {2007},
	month = {2007-12},
	publisher = {IEEE Computer Society},
	address = {Adelaide, Australia},
	keywords = {ftmpi},
	author = {Thara Angskun and George Bosilca and Brad Vander Zanden and Jack Dongarra}
}
@article {icl:358,
	title = {Performance Analysis of MPI Collective Operations},
	journal = {Cluster computing},
	volume = {10},
	number = {2},
	year = {2007},
	month = {2007-06},
	pages = {127-143},
	publisher = {Springer Netherlands},
	keywords = {ftmpi},
	author = {Jelena Pjesivac{\textendash}Grbovic and Thara Angskun and George Bosilca and Graham Fagg and Edgar Gabriel and Jack Dongarra}
}
@inproceedings {icl:354,
	title = {Reliability Analysis of Self-Healing Network using Discrete-Event Simulation},
	journal = {Proceedings of Seventh IEEE International Symposium on Cluster Computing and the Grid (CCGrid {\textquoteright}07)},
	year = {2007},
	month = {2007-05},
	pages = {437-444},
	publisher = {IEEE Computer Society},
	keywords = {ftmpi},
	author = {Thara Angskun and George Bosilca and Graham Fagg and Jelena Pjesivac{\textendash}Grbovic and Jack Dongarra}
}
@inproceedings {icl:384,
	title = {Scalability Analysis of the SPEC OpenMP Benchmarks on Large-Scale Shared Memory Multiprocessors},
	journal = {Proceedings of the 2007 International Conference on Computational Science (ICCS 2007)},
	volume = {4487-4490},
	year = {2007},
	pages = {815-822},
	publisher = {Springer LNCS},
	address = {Beijing, China},
	keywords = {kojak},
	doi = {10.1007/978-3-540-72586-2_115},
	author = {Karl F{\"u}rlinger and Michael Gerndt and Jack Dongarra},
	editor = {Yong Shi and Jack Dongarra and Geert Dick van Albada and Peter M. Sloot}
}
@inproceedings {icl:380,
	title = {Self-Healing in Binomial Graph Networks},
	journal = {2nd International Workshop On Reliability in Decentralized Distributed Systems (RDDS 2007)},
	year = {2007},
	month = {2007-11},
	address = {Vilamoura, Algarve, Portugal},
	author = {Thara Angskun and George Bosilca and Jack Dongarra}
}
@article {icl:315,
	title = {Flexible collective communication tuning architecture applied to Open MPI},
	journal = {2006 Euro PVM/MPI (submitted)},
	year = {2006},
	month = {2006-01},
	address = {Bonn, Germany},
	keywords = {ftmpi},
	author = {Graham Fagg and Jelena Pjesivac{\textendash}Grbovic and George Bosilca and Thara Angskun and Jack Dongarra}
}
@techreport {icl:314,
	title = {MPI Collective Algorithm Selection and Quadtree Encoding},
	journal = {ICL Technical Report},
	number = {ICL-UT-06-11},
	year = {2006},
	month = {2006-00},
	keywords = {ftmpi},
	author = {Jelena Pjesivac{\textendash}Grbovic and Graham Fagg and Thara Angskun and George Bosilca and Jack Dongarra}
}
@article {icl:323,
	title = {MPI Collective Algorithm Selection and Quadtree Encoding},
	journal = {Lecture Notes in Computer Science},
	volume = {4192},
	number = {ICL-UT-06-13},
	year = {2006},
	month = {2006-09},
	pages = {40-48},
	publisher = {Springer Berlin / Heidelberg},
	keywords = {ftmpi},
	author = {Jelena Pjesivac{\textendash}Grbovic and Graham Fagg and Thara Angskun and George Bosilca and Jack Dongarra}
}
@article {icl:316,
	title = {Scalable Fault Tolerant Protocol for Parallel Runtime Environments},
	journal = {2006 Euro PVM/MPI},
	number = {ICL-UT-06-12},
	year = {2006},
	month = {2006-00},
	address = {Bonn, Germany},
	keywords = {ftmpi},
	author = {Thara Angskun and Graham Fagg and George Bosilca and Jelena Pjesivac{\textendash}Grbovic and Jack Dongarra}
}
@inproceedings {icl:330,
	title = {Self-Healing Network for Scalable Fault Tolerant Runtime Environments},
	journal = {DAPSYS 2006, 6th Austrian-Hungarian Workshop on Distributed and Parallel Systems},
	year = {2006},
	month = {2006-01},
	address = {Innsbruck, Austria},
	author = {Thara Angskun and Graham Fagg and George Bosilca and Jelena Pjesivac{\textendash}Grbovic and Jack Dongarra}
}
@inproceedings {icl:307,
	title = {A Systematic Multi-step Methodology for Performance Analysis of Communication Traces of Distributed Applications based on Hierarchical Clustering},
	journal = {Proc. of the 5th International Workshop on Performance Modeling, Evaluation, and Organization of Parallel and Distributed Systems (PMEO-PDS 2006)},
	number = {ICL-UT-05-06},
	year = {2006},
	month = {2006-04},
	publisher = {IEEE Computer Society},
	address = {Rhodes Island, Greece},
	keywords = {kojak},
	author = {Gabriela Aguilera and Patricia J. Teller and Michela Taufer and Felix Wolf}
}
@inproceedings {icl:296,
	title = {Analysis and Optimization of Yee_Bench using Hardware Performance Counters},
	journal = {Proceedings of Parallel Computing 2005 (ParCo)},
	year = {2005},
	month = {2005-01},
	address = {Malaga, Spain},
	abstract = {In this paper, we report on our analysis and optimization of a serial Fortran 90 benchmark called Yee bench. This benchmark has been run on a variety of architectures and its performance is reasonably well understood. However, on AMD Opteron based machines, we found unexpected dips in the delivered MFLOPS of the code for a seemingly random set of problem sizes. Through the use of the Opteron{\textquoteright}s on-chip hardware performance counters andPapiEx, aPAPI based tool, we discovered that these drops were directly related to high L1 cache miss rates for these problem sizes. The high miss rates could be attributed to the fact that in the two core regions of the code we have references to three dynamically allocated arrays which compete for the same set in the Opteron{\textquoteright}s 2-way set associative cache. We validated this conclusion by accurately predicting those problem sizes that exhibit this problem. We were able to alleviate these performance anomalies using variable intra-array padding to effectively accomplish inter-array padding. We conclude with some comments on the general applicability of this method as well how one might improving the implementation of the Fortran 90ALLOCATE intrinsic to handle this case. 1.},
	keywords = {papi},
	author = {Ulf Andersson and Phil Mucci}
}
@inproceedings {icl:284,
	title = {Comparison of Nonlinear Conjugate-Gradient methods for computing the Electronic Properties of Nanostructure Architectures},
	journal = {Proceedings of 5th International Conference on Computational Science (ICCS)},
	year = {2005},
	month = {2005-01},
	pages = {317-325},
	publisher = {Springer{\textquoteright}s Lecture Notes in Computer Science},
	address = {Atlanta, GA, USA},
	keywords = {doe-nano},
	author = {Stanimire Tomov and Julien Langou and Andrew Canning and Lin-Wang Wang and Jack Dongarra},
	editor = {V. S. Sunderman and Geert Dick van Albada and Peter M. Sloot and Jack Dongarra}
}
@inproceedings {icl:265,
	title = {Fault Tolerant High Performance Computing by a Coding Approach},
	journal = {Proceedings of ACM SIGPLAN Symposium on Principles and Practice of Parallel Programming (to appear)},
	year = {2005},
	month = {2005-01},
	address = {Chicago, Illinois},
	keywords = {ftmpi, grads, lacsi, sans},
	author = {Zizhong Chen and Graham Fagg and Edgar Gabriel and Julien Langou and Thara Angskun and George Bosilca and Jack Dongarra}
}
@article {icl:276,
	title = {NetSolve: Grid Enabling Scientific Computing Environments},
	journal = {Grid Computing and New Frontiers of High Performance Processing},
	number = {14},
	year = {2005},
	month = {2005-00},
	publisher = {Elsevier},
	keywords = {netsolve},
	author = {Keith Seymour and Asim YarKhan and Sudesh Agrawal and Jack Dongarra},
	editor = {Lucio Grandinetti}
}
@conference {icl:297,
	title = {PerfMiner: Cluster-Wide Collection, Storage and Presentation of Application Level Hardware Performance Data},
	booktitle = {European Conference on Parallel Processing (Euro-Par 2005)},
	year = {2005},
	month = {2005-09},
	publisher = {Springer},
	organization = {Springer},
	address = {Monte de Caparica, Portugal},
	abstract = {We present PerfMiner, a system for the transparent collection, storage and presentation of thread-level hardware performance data across an entire cluster. Every sub-process/thread spawned by the user through the batch system is measured with near zero overhead and no dilation of run-time. Performance metrics are collected at the thread level using tool built on top of the Performance Application Programming Interface (PAPI). As the hardware counters are virtualized by the OS, the resulting counts are largely unaffected by other kernel or user processes. PerfMiner correlates this performance data with metadata from the batch system and places it in a database. Through a command line and web interface, the user can make queries to the database to report information on everything from overall workload characterization and system utilization to the performance of a single thread in a specific application. This is in contrast to other monitoring systems that report aggregate system-wide metrics sampled over a period of time. In this paper, we describe our implementation of PerfMiner as well as present some results from the test deployment of PerfMiner across three different clusters at the Center for Parallel Computers at The Royal Institute of Technology in Stockholm, Sweden.},
	keywords = {papi},
	doi = {https://doi.org/10.1007/11549468_1},
	author = {Phil Mucci and Daniel Ahlin and Johan Danielsson and Per Ekman and Lars Malinowski}
}
@inproceedings {icl:249,
	title = {Performance Analysis of MPI Collective Operations},
	journal = {4th International Workshop on Performance Modeling, Evaluation, and Optmization of Parallel and Distributed Systems (PMEO-PDS {\textquoteright}05)},
	year = {2005},
	month = {2005-04},
	address = {Denver, Colorado},
	keywords = {ftmpi},
	author = {Jelena Pjesivac{\textendash}Grbovic and Thara Angskun and George Bosilca and Graham Fagg and Edgar Gabriel and Jack Dongarra}
}
@article {icl:306,
	title = {Performance Analysis of MPI Collective Operations},
	journal = {Cluster Computing Journal (to appear)},
	year = {2005},
	month = {2005-01},
	keywords = {ftmpi},
	author = {Jelena Pjesivac{\textendash}Grbovic and Thara Angskun and George Bosilca and Graham Fagg and Edgar Gabriel and Jack Dongarra}
}
@conference {icl:298,
	title = {Performance Profiling and Analysis of DoD Applications using PAPI and TAU},
	booktitle = {Proceedings of DoD HPCMP UGC 2005},
	year = {2005},
	month = {2005-06},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Nashville, TN},
	keywords = {papi},
	author = {Shirley Moore and David Cronk and Felix Wolf and Avi Purkayastha and Patricia J. Teller and Robert Araiza and Gabriela Aguilera and Jamie Nava}
}
@inproceedings {icl:279,
	title = {Scalable Fault Tolerant MPI: Extending the Recovery Algorithm},
	journal = {Proceedings of 12th European Parallel Virtual Machine and Message Passing Interface Conference - Euro PVM/MPI},
	volume = {3666},
	year = {2005},
	month = {2005-09},
	pages = {67},
	publisher = {Springer-Verlag Berlin},
	address = {Sorrento (Naples) , Italy},
	keywords = {ftmpi},
	author = {Graham Fagg and Thara Angskun and George Bosilca and Jelena Pjesivac{\textendash}Grbovic and Jack Dongarra},
	editor = {Beniamino Di Martino}
}
@article {icl:236,
	title = {Cray X1 Evaluation Status Report},
	journal = {Oak Ridge National Laboratory Report},
	volume = {/-2004/13},
	year = {2004},
	month = {2004-01},
	author = {Pratul Agarwal and R. A. Alexander and E. Apra and Satish Balay and Arthur S. Bland and James Colgan and Eduardo D{\textquoteright}Azevedo and Jack Dongarra and Tom Dunigan and Mark Fahey and Al Geist and M. Gordon and Robert Harrison and Dinesh Kaushik and M. Krishnakumar and Piotr Luszczek and Tony Mezzacapa and Jeff Nichols and Jarek Nieplocha and Leonid Oliker and T. Packwood and M. Pindzola and Thomas C. Schulthess and Jeffrey Vetter and James B White and T. Windus and Patrick H. Worley and Thomas Zacharia}
}
@inproceedings {icl:201,
	title = {Design of an Interactive Environment for Numerically Intensive Parallel Linear Algebra Calculations},
	journal = {International Conference on Computational Science},
	year = {2004},
	month = {2004-06},
	publisher = {Springer Verlag},
	address = {Poland},
	keywords = {lacsi, lfc},
	doi = {10.1007/978-3-540-25944-2_35},
	author = {Piotr Luszczek and Jack Dongarra},
	editor = {Marian Bubak and Geert Dick van Albada and Peter M. Sloot and Jack Dongarra}
}
@inproceedings {icl:230,
	title = {Extending the MPI Specification for Process Fault Tolerance on High Performance Computing Systems},
	journal = {Proceedings of ISC2004 (to appear)},
	year = {2004},
	month = {2004-06},
	address = {Heidelberg, Germany},
	keywords = {ftmpi, lacsi},
	author = {Graham Fagg and Edgar Gabriel and George Bosilca and Thara Angskun and Zizhong Chen and Jelena Pjesivac{\textendash}Grbovic and Kevin London and Jack Dongarra}
}
@article {icl:240,
	title = {Process Fault-Tolerance: Semantics, Design and Applications for High Performance Computing},
	journal = {International Journal for High Performance Applications and Supercomputing (to appear)},
	year = {2004},
	month = {2004-04},
	keywords = {ftmpi, lacsi},
	author = {Graham Fagg and Edgar Gabriel and Zizhong Chen and Thara Angskun and George Bosilca and Jelena Pjesivac{\textendash}Grbovic and Jack Dongarra}
}
@article {icl:169,
	title = {Computational Science {\textemdash} ICCS 2003},
	journal = {Lecture Notes in Computer Science},
	volume = {2657-2660},
	year = {2003},
	month = {2003-06},
	publisher = {Springer-Verlag, Berlin},
	address = {ICCS 2003, International Conference. Melbourne, Australia},
	issn = {978-3-540-40194-0},
	author = {Peter M. Sloot and David Abramson and Alexander V. Bogdanov and Jack Dongarra and Albert Zomaya and Yuriy Gorbachev}
}
@article {icl:131,
	title = {Energy Minimization of Protein Tertiary Structure by Parallel Simulated Annealing using Genetic Crossover},
	journal = {Special Issue on Biological Applications of Genetic and Evolutionary Computation (submitted)},
	year = {2003},
	month = {2003-03},
	author = {Tomoyuki Hiroyasu and Mitsunori Miki and Shinya Ogura and Keiko Aoi and Takeshi Yoshida and Yuko Okamoto and Jack Dongarra}
}
@inproceedings {icl:153,
	title = {Fault Tolerant Communication Library and Applications for High Performance Computing},
	journal = {Los Alamos Computer Science Institute (LACSI) Symposium 2003 (presented)},
	year = {2003},
	month = {2003-10},
	address = {Santa Fe, NM},
	keywords = {ftmpi, lacsi},
	author = {Graham Fagg and Edgar Gabriel and Zizhong Chen and Thara Angskun and George Bosilca and Antonin Bukovsky and Jack Dongarra}
}
@inproceedings {icl:144,
	title = {A Fault-Tolerant Communication Library for Grid Environments},
	journal = {17th Annual ACM International Conference on Supercomputing (ICS{\textquoteright}03) International Workshop on Grid Computing and e-Science},
	year = {2003},
	month = {2003-06},
	address = {San Francisco},
	keywords = {ftmpi, lacsi},
	author = {Edgar Gabriel and Graham Fagg and Antonin Bukovsky and Thara Angskun and Jack Dongarra}
}
@article {icl:168,
	title = {High Performance Computing for Computational Science},
	journal = {Lecture Notes in Computer Science},
	volume = {2565},
	year = {2003},
	month = {2003-01},
	publisher = {Springer-Verlag, Berlin},
	address = {VECPAR 2002, 5th International Conference June 26-28, 2002},
	author = {Jose Palma and Jack Dongarra and Vicente Hern{\'a}ndez},
	editor = {Antonio Augusto Sousa}
}
@article {icl:143,
	title = {NetSolve: Past, Present, and Future - A Look at a Grid Enabled Server},
	journal = {Making the Global Infrastructure a Reality},
	year = {2003},
	month = {2003-00},
	publisher = {Wiley Publishing},
	keywords = {netsolve},
	author = {Sudesh Agrawal and Jack Dongarra and Keith Seymour and Sathish Vadhiyar},
	editor = {Francine Berman and Geoffrey Fox and Anthony Hey}
}
@article {icl:138,
	title = {Scheduling in the Grid Application Development Software Project},
	journal = {Resource Management in the Grid},
	year = {2003},
	month = {2003-03},
	publisher = {Kluwer Publishers},
	keywords = {grads},
	author = {Holly Dail and Otto Sievert and Francine Berman and Henri Casanova and Asim YarKhan and Sathish Vadhiyar and Jack Dongarra and Chuang Liu and Lingyun Yang and Dave Angulo and Ian Foster}
}
@article {icl:113,
	title = {A Comparison of Parallel Solvers for General Narrow Banded Linear Systems},
	journal = {Parallel and Distributed Computing Practices},
	volume = {2},
	year = {2002},
	month = {2002-10},
	pages = {385-400},
	author = {Peter Arbenz and Andrew Cleary and Jack Dongarra and Markus Hegland}
}
@techreport {icl:83,
	title = {Hardware Software Server in NetSolve},
	journal = {ICL Technical Report},
	number = {ICL-UT-02-02},
	year = {2002},
	month = {2002-01},
	keywords = {netsolve},
	author = {Sudesh Agrawal}
}
@article {icl:207,
	title = {Innovations of the NetSolve Grid Computing System},
	journal = {Concurrency: Practice and Experience},
	volume = {14},
	number = {13-15},
	year = {2002},
	month = {2002-01},
	pages = {1457-1479},
	keywords = {netsolve},
	author = {Dorian Arnold and Henri Casanova and Jack Dongarra}
}
@article {icl:101,
	title = {Middleware for the Use of Storage in Communication},
	journal = {Parallel Computing},
	volume = {28},
	number = {12},
	year = {2002},
	month = {2002-08},
	pages = {1773-1788},
	keywords = {netsolve},
	author = {Micah Beck and Dorian Arnold and Alessandro Bassi and Francine Berman and Henri Casanova and Jack Dongarra and Terry Moore and Graziano Obertelli and James Plank and Martin Swany and Sathish Vadhiyar and Rich Wolski}
}
@inproceedings {icl:79,
	title = {Toward a Framework for Preparing and Executing Adaptive Grid Programs},
	journal = {International Parallel and Distributed Processing Symposium: IPDPS 2002 Workshops},
	year = {2002},
	month = {2002-04},
	pages = {0171},
	address = {Fort Lauderdale, FL},
	keywords = {grads},
	author = {Ken Kennedy and John Mellor-Crummey and Keith Cooper and Linda Torczon and Francine Berman and Andrew Chien and Dave Angulo and Ian Foster and Dennis Gannon and Lennart Johnsson and Carl Kesselman and Jack Dongarra and Sathish Vadhiyar}
}
@techreport {icl:96,
	title = {Users{\textquoteright} Guide to NetSolve v1.4.1},
	journal = {ICL Technical Report},
	number = {ICL-UT-02-05},
	year = {2002},
	month = {2002-06},
	keywords = {netsolve},
	author = {Sudesh Agrawal and Dorian Arnold and Susan Blackford and Jack Dongarra and Michelle Miller and Kiran Sagi and Zhiao Shi and Keith Seymour and Sathish Vadhiyar}
}
@article {icl:87,
	title = {On the Convergence of Computational and Data Grids},
	journal = {Parallel Processing Letters},
	volume = {11},
	number = {2-3},
	year = {2001},
	month = {2001-01},
	pages = {187-202},
	keywords = {netsolve},
	author = {Dorian Arnold and Sathish Vadhiyar and Jack Dongarra}
}
@inproceedings {icl:5,
	title = {Enabling Full Service Surrogates Using the Portable Channel Representation},
	journal = {Tenth International World Wide Web Conference Proceedings (to appear),},
	year = {2001},
	month = {2001-05},
	address = {Hong Kong},
	author = {Micah Beck and Terry Moore and Leif Abrahamsson and Chistophe Achouiantz and Patrik Johansson}
}
@article {icl:4,
	title = {Logistical Computing and Internetworking: Middleware for the Use of Storage in Communication},
	journal = {submitted to SC2001},
	year = {2001},
	month = {2001-11},
	address = {Denver, Colorado},
	keywords = {netsolve},
	author = {Micah Beck and Dorian Arnold and Alessandro Bassi and Francine Berman and Henri Casanova and Jack Dongarra and Terry Moore and Graziano Obertelli and James Plank and Martin Swany and Sathish Vadhiyar and Rich Wolski}
}
@inproceedings {icl:19,
	title = {Metacomputing Support for the SARA3D Structural Acoustics Application},
	journal = {Department of Defense Users{\textquoteright} Group Conference (to appear)},
	year = {2001},
	month = {2001-06},
	address = {Biloxi, Mississippi},
	keywords = {netsolve},
	author = {Shirley Moore and Dorian Arnold and David Cronk}
}
@article {icl:23,
	title = {Overview of High Performance Computers},
	journal = {Handbook of Massive Data Sets},
	year = {2001},
	month = {2001-01},
	pages = {791-852},
	publisher = {Kluwer Academic Publishers},
	author = {Aad J. van der Steen and Jack Dongarra},
	editor = {James Abello and Panos Pardalos and Mauricio Resende}
}
@inproceedings {icl:27,
	title = {Developing an Architecture to Support the Implementation and Development of Scientific Computing Applications},
	journal = {to appear in Proceedings of Working Conference 8: Software Architecture for Scientific Computing Applications},
	year = {2000},
	month = {2000-10},
	address = {Ottawa, Canada},
	keywords = {netsolve},
	author = {Dorian Arnold and Jack Dongarra}
}
@inproceedings {icl:28,
	title = {The NetSolve Environment: Progressing Towards the Seamless Grid},
	journal = {2000 International Conference on Parallel Processing (ICPP-2000)},
	year = {2000},
	month = {2000-08},
	address = {Toronto, Canada},
	keywords = {netsolve},
	author = {Dorian Arnold and Jack Dongarra}
}
@article {icl:227,
	title = {Providing Infrastructure and Interface to High Performance Applications in a Distributed Setting},
	journal = {ASTC-HPC 2000},
	year = {2000},
	month = {2000-04},
	address = {Washington, DC},
	author = {Dorian Arnold and Wonsuck Lee and Jack Dongarra and Mary Wheeler}
}
@inproceedings {icl:24,
	title = {Request Sequencing: Optimizing Communication for the Grid},
	journal = {Lecture Notes in Computer Science: Proceedings of 6th International Euro-Par Conference 2000, Parallel Processing},
	year = {2000},
	month = {2000-01},
	pages = {V1900,1213-1222},
	address = {(Germany: Springer Verlag 2000)},
	keywords = {netsolve},
	author = {Dorian Arnold and Dieter Bachmann and Jack Dongarra}
}
@inproceedings {icl:25,
	title = {Seamless Access to Adaptive Solver Algorithms},
	journal = {Proceedings of 16th IMACS World Congress 2000 on Scientific Computing, Applications Mathematics and Simulation},
	year = {2000},
	month = {2000-08},
	address = {Lausanne, Switzerland},
	keywords = {netsolve},
	author = {Dorian Arnold and Susan Blackford and Jack Dongarra and Victor Eijkhout and Tinghua Xu}
}
@techreport {icl:228,
	title = {Secure Remote Access to Numerical Software and Computation Hardware},
	journal = {University of Tennessee Computer Science Technical Report, UT-CS-00-446},
	year = {2000},
	month = {2000-07},
	author = {Dorian Arnold and Shirley Browne and Jack Dongarra and Graham Fagg and Keith Moore}
}
@inproceedings {icl:26,
	title = {Secure Remote Access to Numerical Software and Computational Hardware},
	journal = {Proceedings of the DoD HPC Users Group Conference (HPCUG) 2000},
	year = {2000},
	month = {2000-06},
	address = {Albuquerque, NM},
	keywords = {netsolve},
	author = {Dorian Arnold and Shirley Browne and Jack Dongarra and Graham Fagg and Keith Moore}
}
@techreport {icl:51,
	title = {A Comparison of Parallel Solvers for Diagonally Dominant and General Narrow Banded Linear Systems II (LAPACK Working Note 143)},
	journal = {University of Tennessee Computer Science Department Technical Report},
	number = {UT-CS-99-415},
	year = {1999},
	month = {1999-01},
	author = {Peter Arbenz and Andrew Cleary and Jack Dongarra and Markus Hegland}
}
@techreport {icl:52,
	title = {A Comparison of Parallel Solvers for General Narrow Banded Linear Systems (LAPACK Working Note 142)},
	journal = {University of Tennessee Computer Science Technical Report},
	number = {UT-CS-99-414},
	year = {1999},
	month = {1999-01},
	author = {Peter Arbenz and Andrew Cleary and Jack Dongarra and Markus Hegland}
}
@article {icl:50,
	title = {LAPACK Users{\textquoteright} Guide, 3rd ed.},
	journal = {Philadelphia: Society for Industrial and Applied Mathematics},
	year = {1999},
	month = {1999-01},
	author = {Ed Anderson and Zhaojun Bai and Christian Bischof and Susan Blackford and James Demmel and Jack Dongarra and Jeremy Du Croz and Anne Greenbaum and Sven Hammarling and Alan McKenney and Danny Sorensen}
}
