map_t.py
import sys
import re
p = re.compile(r'\w+')
for line in sys.stdin:
ss = line.strip().split(' ')
for s in ss:
if len(p.findall(s)) < 1:
continue
s_low = p.findall(s)[0].lower()
print(s_low + ',' + '1')
reduce_t.py
import sys
cur_word = None
s = 0
for line in sys.stdin:
word, val = line.strip().split(',')
if cur_word is None:
cur_word = word
if cur_word != word:
print('%s\t%s' % (cur_word, s))
cur_word = word
s = 0
s += int(val)
print('%s\t%s' % (cur_word, sum))
HADOOP_CMD="/usr/local/src/hadoop-2.6.1/bin/hadoop"
STREAM_JAR_PATH="/usr/local/src/hadoop-2.6.1/share/hadoop/tools/lib/hadoop-streaming-2.6.1.jar"
INPUT_FILE_PATH="/data/words.txt"
OUTPUT_PATH="/output/wc"
$HADOOP_CMD fs -rmr -skipTrash $OUTPUT_PATH
# Step 1.
$HADOOP_CMD jar $STREAM_JAR_PATH \
-input $INPUT_FILE_PATH \
-output $OUTPUT_PATH \
-mapper "python map_t.py" \
-reducer "python red_t.py" \
-file ./map_t.py \
-file ./red_t.py
本文内容由网友自发贡献，版权归原作者所有，本站不承担相应法律责任。如您发现有涉嫌抄袭侵权的内容，请联系:hwhale#tublm.com(使用前将#替换为@)
