{"id":3359,"date":"2021-05-31T18:19:48","date_gmt":"2021-05-31T10:19:48","guid":{"rendered":"https:\/\/wyxxt.org.cn\/?p=3359"},"modified":"2023-12-04T14:59:17","modified_gmt":"2023-12-04T06:59:17","slug":"mapreduce-yarn%e9%9b%86%e7%be%a4%e6%90%ad%e5%bb%ba","status":"publish","type":"post","link":"https:\/\/wyxxt.org.cn\/?p=3359","title":{"rendered":"MapReduce-Yarn\u96c6\u7fa4\u642d\u5efa"},"content":{"rendered":"

\u67b6\u6784<\/h1>\n

HDFS\u548cYARN\u662f\u4e24\u4e2a\u6982\u5ff5\uff0c\u642d\u5efaYARN\u548cHDFS\u65e0\u51b2\u7a81\u3002<\/p>\n\n\n\n\n\n\n\n\n
HOST<\/th>\nNN<\/th>\nNN<\/th>\nJNN<\/th>\nDN<\/th>\nZKFC<\/th>\nZK<\/th>\nRM<\/th>\nNM<\/th>\n<\/tr>\n<\/thead>\n
node01<\/td>\n*<\/td>\n<\/td>\n*<\/td>\n<\/td>\n*<\/td>\n<\/td>\n<\/td>\n<\/td>\n<\/tr>\n
node02<\/td>\n<\/td>\n*<\/td>\n*<\/td>\n*<\/td>\n*<\/td>\n*<\/td>\n<\/td>\n*<\/td>\n<\/tr>\n
node03<\/td>\n<\/td>\n<\/td>\n*<\/td>\n*<\/td>\n<\/td>\n*<\/td>\n*<\/td>\n*<\/td>\n<\/tr>\n
node04<\/td>\n<\/td>\n<\/td>\n<\/td>\n*<\/td>\n<\/td>\n*<\/td>\n*<\/td>\n*<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n

\u53d1\u5c55\u8fdb\u7a0b<\/h1>\n\n\n\n\n\n\n
hdoop<\/th>\n1.x<\/th>\n2.x<\/th>\n3.x<\/th>\n<\/tr>\n<\/thead>\n
hdfs<\/td>\nno ha<\/td>\nha(\u5411\u524d\u517c\u5bb9\uff0c\u6ca1\u6709\u8fc7\u591a\u7684\u6539NN\uff0c\u800c\u662f\u901a\u8fc7\u65b0\u589e\u89d2\u8272zkfc)<\/td>\n<\/td>\n<\/tr>\n
yarn<\/td>\nno yarn<\/td>\nyarn\uff08\u5df2\u7ecf\u6709\u4e86ha\u7684\u6982\u5ff5\uff0c\u4e0d\u662f\u65b0\u589e\u89d2\u8272\uff0c\u800c\u662f\u76f4\u63a5\u5728RM\u8fdb\u7a0b\u4e2d\u589e\u52a0\u4e86HA\u7684\u6a21\u5757\uff09<\/td>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n

\u901a\u8fc7\u5b98\u7f51<\/h1>\n

https:\/\/hadoop.apache.org\/docs\/r2.10.1\/hadoop-yarn\/hadoop-yarn-site\/ResourceManagerHA.html<\/p>\n

\u4f2a\u5206\u5e03\u5f0f<\/h2>\n

etc\/hadoop\/mapred-site.xml<\/p>\n

<configuration>\n    <property>\n        <name>mapreduce.framework.name<\/name>\n        <value>yarn<\/value>\n    <\/property>\n<\/configuration>\n<\/code><\/pre>\n
etc\/hadoop\/yarn-site.xml<\/p>\n
<configuration>\n    <property>\n        <name>yarn.nodemanager.aux-services<\/name>\n        <value>mapreduce_shuffle<\/value>\n    <\/property>\n<\/configuration>\n<\/code><\/pre>\n
HA<\/h2>\n
etc\/hadoop\/yarn-site.xml<\/p>\n
<property>\n  <name>yarn.resourcemanager.ha.enabled<\/name>\n  <value>true<\/value>\n<\/property>\n<property>\n  <name>yarn.resourcemanager.zk-address<\/name>\n  <value>node01:2181,node02:2181,node03:2181<\/value>\n<\/property>\n<property>\n  <name>yarn.resourcemanager.cluster-id<\/name>\n  <value>mashibing<\/value>\n<\/property>\n<property>\n  <name>yarn.resourcemanager.ha.rm-ids<\/name>\n  <value>rm1,rm2<\/value>\n<\/property>\n<property>\n  <name>yarn.resourcemanager.hostname.rm1<\/name>\n  <value>node03<\/value>\n<\/property>\n<property>\n  <name>yarn.resourcemanager.hostname.rm2<\/name>\n  <value>node04<\/value>\n<\/property>\n<property>\n  <name>yarn.resourcemanager.webapp.address.rm1<\/name>\n  <value>node03:8088<\/value>\n<\/property>\n<property>\n  <name>yarn.resourcemanager.webapp.address.rm2<\/name>\n  <value>node04:8088<\/value>\n<\/property>\n<\/code><\/pre>\n
\u914d\u7f6e<\/h2>\n
\u5206\u53d1<\/h3>\n
scp mapred-site.xml yarn-site.xml node02:`pwd`\nscp mapred-site.xml yarn-site.xml node03:`pwd`\nscp mapred-site.xml yarn-site.xml node04:`pwd`\n<\/code><\/pre>\n
vi slaves \/\/\u53ef\u4ee5\u4e0d\u7528\u7ba1\uff0c\u642d\u5efahdfs\u65f6\u5019\u5df2\u7ecf\u6539\u8fc7\u4e86<\/p>\n
\u542f\u52a8<\/h3>\n
# \u542f\u52a8 NodeManager\nstart-yarn.sh\n# \u542f\u52a8 ResourceManager\nyarn-daemon.sh start resourcemanager\n<\/code><\/pre>\n
\u8bbf\u95ee\u9a8c\u8bc1<\/h3>\n
http:\/\/node03:8088
\nhttp:\/\/node04:8088<\/p>\n
\"\"<\/p>\n
\u4f7f\u7528<\/h2>\n
hdfs dfs -mkdir -p \/data\/wc\/input\n\nhdfs dfs -D dfs.blocksize=1048576 -put data.txt \/data\/wc\/input\n<\/code><\/pre>\n
\"\"<\/p>\n
\u8ba1\u7b97<\/h3>\n
\"\"<\/p>\n
hadoop jar hadoop-mapreduce-examples-2.10.1.jar wordcount \/data\/wc\/input \/data\/wc\/output\n<\/code><\/pre>\n
\"\"<\/p>\n
\"\"<\/p>\n
\u7ed3\u679c<\/h3>\n
\"\"<\/p>\n
\u4ee3\u7801<\/h2>\n
Hadoop<\/a><\/p>\n
maven\u6253\u5305<\/h3>\n
    \n
  1. clean<\/li>\n
  2. package<\/li>\n<\/ol>\n
    \"\"<\/p>\n
    \u62f7\u8d1d\u5230\u670d\u52a1\u5668\u6267\u884c<\/h3>\n
    \"\"<\/p>\n
    \u7ed3\u679c<\/h3>\n
    \"\"<\/p>\n","protected":false},"excerpt":{"rendered":"
    \u67b6\u6784 HDFS\u548cYARN\u662f\u4e24\u4e2a\u6982\u5ff5\uff0c\u642d\u5efaYARN\u548cHDFS\u65e0\u51b2\u7a81\u3002 HOST NN NN JNN DN ZKF […]<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"closed","ping_status":"closed","sticky":false,"template":"","format":"standard","meta":{"site-sidebar-layout":"default","site-content-layout":"","ast-site-content-layout":"default","site-content-style":"default","site-sidebar-style":"default","ast-global-header-display":"","ast-banner-title-visibility":"","ast-main-header-display":"","ast-hfb-above-header-display":"","ast-hfb-below-header-display":"","ast-hfb-mobile-header-display":"","site-post-title":"","ast-breadcrumbs-content":"","ast-featured-img":"","footer-sml-layout":"","ast-disable-related-posts":"","theme-transparent-header-meta":"","adv-header-id-meta":"","stick-header-meta":"","header-above-stick-meta":"","header-main-stick-meta":"","header-below-stick-meta":"","astra-migrate-meta-layouts":"default","ast-page-background-enabled":"default","ast-page-background-meta":{"desktop":{"background-color":"var(--ast-global-color-4)","background-image":"","background-repeat":"repeat","background-position":"center center","background-size":"auto","background-attachment":"scroll","background-type":"","background-media":"","overlay-type":"","overlay-color":"","overlay-opacity":"","overlay-gradient":""},"tablet":{"background-color":"","background-image":"","background-repeat":"repeat","background-position":"center center","background-size":"auto","background-attachment":"scroll","background-type":"","background-media":"","overlay-type":"","overlay-color":"","overlay-opacity":"","overlay-gradient":""},"mobile":{"background-color":"","background-image":"","background-repeat":"repeat","background-position":"center center","background-size":"auto","background-attachment":"scroll","background-type":"","background-media":"","overlay-type":"","overlay-color":"","overlay-opacity":"","overlay-gradient":""}},"ast-content-background-meta":{"desktop":{"background-color":"var(--ast-global-color-5)","background-image":"","background-repeat":"repeat","background-position":"center center","background-size":"auto","background-attachment":"scroll","background-type":"","background-media":"","overlay-type":"","overlay-color":"","overlay-opacity":"","overlay-gradient":""},"tablet":{"background-color":"var(--ast-global-color-5)","background-image":"","background-repeat":"repeat","background-position":"center center","background-size":"auto","background-attachment":"scroll","background-type":"","background-media":"","overlay-type":"","overlay-color":"","overlay-opacity":"","overlay-gradient":""},"mobile":{"background-color":"var(--ast-global-color-5)","background-image":"","background-repeat":"repeat","background-position":"center center","background-size":"auto","background-attachment":"scroll","background-type":"","background-media":"","overlay-type":"","overlay-color":"","overlay-opacity":"","overlay-gradient":""}},"footnotes":""},"categories":[15],"tags":[389],"class_list":["post-3359","post","type-post","status-publish","format-standard","hentry","category-15","tag-389"],"_links":{"self":[{"href":"https:\/\/wyxxt.org.cn\/index.php?rest_route=\/wp\/v2\/posts\/3359","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/wyxxt.org.cn\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/wyxxt.org.cn\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/wyxxt.org.cn\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/wyxxt.org.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=3359"}],"version-history":[{"count":21,"href":"https:\/\/wyxxt.org.cn\/index.php?rest_route=\/wp\/v2\/posts\/3359\/revisions"}],"predecessor-version":[{"id":3380,"href":"https:\/\/wyxxt.org.cn\/index.php?rest_route=\/wp\/v2\/posts\/3359\/revisions\/3380"}],"wp:attachment":[{"href":"https:\/\/wyxxt.org.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=3359"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/wyxxt.org.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=3359"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/wyxxt.org.cn\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=3359"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}