Open presentation, training material. Presented at CSIRO Big Data 2.0 workshop in September 2013, North Ryde, Australia. Animated by hands-on examples.

2. ●
●
●

3. ●

4. ●
●
●
●
●
●
●
●
●
●

5. ●
●
●
●
●
●
●

6. ●
●
●
●
●

7. ●
●
●

8. ●
●

9. ●
●
●
●
> hadoop fs

10. hadoop fs
●
●
●
●
●
●
●

11. ●
●
$ hadoop fs
ls
●
$ hadoop fs –help ls

12. ●
$ hadoop fs –ls <path>
$ hadoop fs –ls /
●
$ hadoop fs -ls
$ hadoop fs –ls /user/cloudera
●
●

13. ●
$ hadoop fs -mkdir data
$ hadoop fs -ls
●
$ cd ~/bigdata/Exercises/hadoop/data
$ ls -l
$ hadoop fs –put mammograms.zip data

14. ●
●
●
http://localhost:50070
fsck: an HDFS utility
$ hadoop fsck /user/cloudera/data/mammograms.zip
-blocks -locations -files
●
$ head -n 100 ato_centenary.txt
| hadoop fs –put - data/ato100.txt

15. ●
$ head -n 1000 ato_centenary.txt
| hadoop fs –put - data/ato100.txt
●
put: ‘data/ato100.txt': File exists
●
$ hadoop fs -rm data/ato100.txt
$ head -n 1000 ato_centenary.txt
| hadoop fs –put - data/ato100.txt

16. ●
$ hadoop fs -cat data/ato100.txt | less
●
$ hadoop fs -get data/ato100.txt ato100.txt
●
-mv, -cp, -rmdir, -stat ...

17. ●
●
●
●
●
●
●
●
●
●

18. ●
●
●
○
■

19. ●
○
●
○
●
○
●
○
○
○

21. ●
●
●
●
●

22. ●
●
●
●
●
$ javac –classpath
`hadoop classpath` *.java
●
$ jar cvf csiro.jar *.class
●
$ hadoop jar csiro.jar Csiro input_dir output_dir

23. ●
○
●
●
map(in_key, in_value) ->
(inter_key, inter_value) list

24. ●
○
■
■
■
●

25. ●
let map(key, value) =
emit(key.toUpper(), value.
toUpper())
(‘csiro’, ‘cci’) -> (‘CSIRO’, ‘CCI’)
(‘csiro’, ‘cesre’) -> (‘CSIRO’, ‘CESRE’)
(‘csiro’, ‘cmse’) -> (‘CSIRO’, ‘CMSE’)
(‘toyota’, ‘yaris’) -> (‘TOYOTA’,
‘YARIS’)

26. ●
let map(key, value) =
foreach char c in value:
emit(key, c)
(‘cci’, ‘csiro’) -> (‘cci’, ‘c’), (‘cci’, ’s’),
(‘cci’, ‘i’), (‘cci’, ‘r’),
(‘cci’, ‘o’)
(‘open’, ‘nasa’) -> (‘open’, ‘n’), (‘open’, ’a’),
(‘open’, ‘s’), (‘open’, ‘a’)

27. ●
let map(key, value) =
emit(value.length(), value)
(‘csiro’, ‘cci’) -> (‘3’, ‘cci’)
(‘csiro’, ‘cesre’) -> (‘5’, ‘cesre’)
(‘csiro’, ‘cmse’) -> (‘4’, ‘cmse’)
(‘toyota’, ‘yaris’) -> (‘5’, ‘yaris’)

28. ●
●
○
○
○
●
○

29. ●
map(String input_key, String input_value)
foreach word w in input_value:
emit(w, 1)
reduce(String output_key,
Iterator<int> intermediate_values)
set count = 0
foreach v in intermediate_values:
count += v
emit(output_key, count)

30. ●
Wordcount
$ cd ~/bigdata/Exercises/hadoop/wordcount; ls
WordCount.java
WordMapper.java
SumReducer.java
●
$ javac –classpath
`hadoop classpath` *.java
●
$ jar cvf wc.jar *.class

31. ●
$ hadoop jar wc.jar WordCount data/ato100.txt ato_wc
●
$ hadoop fs ls ato_wc
$ hadoop fs -cat ato_wc/part-r-00000 | less
$ hadoop fs -cat ato_wc/* | grep ‘ATO|CSIRO’
●
$ hadoop fs -rm -r ato_wc

32. ●
Average max temperature
●

33. ●
$ cd ~/bigdata/Exercises/hadoop/data
$ less nsw_temp.csv
$ less bom_data_Note.txt

34. ●
map(String input_key, String input_value):
emit(input_value[3], input_value[5])
(‘IDCJAC0010,061087,1965,01,02,32.2,1,Y’)->(‘01’, 32.2)
(‘IDCJAC0010,066062,1890,04,27,20.2,1,Y’)->(‘04’, 20.2)
(‘IDCJAC0010,066062,2012,02,03,21.0,1,Y’)->(‘02’, 21.1)

35. ●
reduce(String month, Iterator<double> values)
set count = 0
set sum = 0
foreach v in values:
sum += v
count++
set mean = sum/count
emit(month, mean)

36. ●
$ cd ../averagetemp
$ gedit *.java&
AverageTemp.java
AverageTempMapper.java
AverageReducer.java
●
$ cd ../wordcount
$ gedit *.java&

37. ●
●
$ hadoop fs -put ../data/nsw_temp.csv data
$ javac –classpath `hadoop classpath` *.java
$ jar cvf avt.jar *.class
$ hadoop jar avt.jar AverageTemp data/nsw_temp.csv avt

38. ●
$ hadoop fs -cat avt/part-1-00000
~/bigdata/Exercises/hadoop/averagetemp/sample_solution

40. ●
○
○
●
●
●
○

41. ●
●
●
●

42. ●
●
●
●

44. ●
●
○
○
●
●
○

45. ●
●
●
●
●
●

47. ●
●
●

51. ●
○
○
○
●
○
●
○
○
○

52. ●
●
●
○
○
○
○
○
○

54. https://github.com/tomaszbednarz/pig-abc-toilets
●
●
●
We have list of local ABC Radio
stations in Australia
We have list of all Public Toilets
across Australia
We want to find a closest toilet to
a Radio Station
Demonstration of:
●
●
●
Data Schemas
Use of external libraries
Google Maps API

55. ●
●
●
●
●
●
●

56. ●
●
●
●
●
●
●
●