@inproceedings{aee5765f97064e33a5548153e61c18d0,
title = "On MapReduce Scheduling in Hadoop Yarn on Heterogeneous Clusters",
abstract = "Hadoop is a distributed computing system widely used for big data processing in various domains. As the data volume continues to increase rapidly, Hadoop systems have become a critical contributor to the success of many big data applications. The MapReduce scheduler is a key component that determines the overall performance of a Hadoop cluster. In this paper, we formulate and investigate a task scheduling problem in a heterogeneous Hadoop cluster to minimize the completion time of a batch of MapReduce jobs. We first design a prediction model to predict the end time of a task, which is used for placing the corresponding data block on a node in advance to reduce the data transmission time and the overall job completion time. Based on this prediction model, we propose a task matching-based scheduling algorithm, referred to as TMSA, to schedule the tasks in the task queue in Hadoop, by taking into account the real-time performance of each node in the cluster and the matching degree between nodes and tasks. Experimental results show that the prediction model achieves high accuracy and TMSA significantly reduces the completion time of a batch of MapReduce jobs compared to existing schedulers.",
keywords = "Hadoop, MapReduce, YARN, distributed computing, task scheduler",
author = "Meng Wang and Wu, {Chase Q.} and Huiyan Cao and Yang Liu and Yongqiang Wang and Aiqin Hou",
note = "Publisher Copyright: {\textcopyright} 2018 IEEE.; 17th IEEE International Conference on Trust, Security and Privacy in Computing and Communications and 12th IEEE International Conference on Big Data Science and Engineering, Trustcom/BigDataSE 2018 ; Conference date: 31-07-2018 Through 03-08-2018",
year = "2018",
month = sep,
day = "5",
doi = "10.1109/TrustCom/BigDataSE.2018.00264",
language = "English (US)",
isbn = "9781538643877",
series = "Proceedings - 17th IEEE International Conference on Trust, Security and Privacy in Computing and Communications and 12th IEEE International Conference on Big Data Science and Engineering, Trustcom/BigDataSE 2018",
publisher = "Institute of Electrical and Electronics Engineers Inc.",
pages = "1747--1754",
booktitle = "Proceedings - 17th IEEE International Conference on Trust, Security and Privacy in Computing and Communications and 12th IEEE International Conference on Big Data Science and Engineering, Trustcom/BigDataSE 2018",
address = "United States",
}