### Cluster Mode (not Local Mode)
C:\hdp\hadoop-2.4.0.2.1.3.0-1981> hadoop fs -mkdir -p /pig/input
# Data Set: Movies
C:\hdp\hadoop-2.4.0.2.1.3.0-1981> hadoop fs -copyFromLocal C:\GitRepos\gudiseva\Hadoop\MayuraSamples\Pig\Input\movies_data.txt /pig/input/movies_data.txt
C:\hdp\hadoop-2.4.0.2.1.3.0-1981> pig
grunt> movies = LOAD '/pig/input/movies_data.txt' USING PigStorage (',') AS (id:int, name:chararray, year:int, rating:float, duration:int);
grunt> DUMP movies;
grunt> clear;
grunt> DESCRIBE movies;
movies: {id: int,name: chararray,year: int,rating: float,duration: int}
grunt> ILLUSTRATE movies;
------------------------------------------------------------------------------------------------
| movies | id:int | name:chararray | year:int | rating:float | duration:int |
------------------------------------------------------------------------------------------------
| | 260 | My Teacher's Wife | 1999 | 3.3 | 5359 |
------------------------------------------------------------------------------------------------
grunt> ILLUSTRATE movies_greater_than_four;
------------------------------------------------------------------------------------------------------------------------------------------
| movies | id:int | name:chararray | year:int | rating:float | duration:int |
------------------------------------------------------------------------------------------------------------------------------------------
| | 1289 | National Geographic: Search for the Battleship Bismarck | 1989 | 3.8 | 3147 |
| | 955 | The Boondock Saints | 1999 | 4.1 | 6507 |
------------------------------------------------------------------------------------------------------------------------------------------
------------------------------------------------------------------------------------------------------------------------
| movies_greater_than_four | id:int | name:chararray | year:int | rating:float | duration:int |
------------------------------------------------------------------------------------------------------------------------
| | 955 | The Boondock Saints | 1999 | 4.1 | 6507 |
------------------------------------------------------------------------------------------------------------------------
grunt> movies_greater_than_four = FILTER movies BY (float) rating > 4.0; // Typecasting required only for default conversion)
grunt> movies_greater_than_four = FILTER movies BY rating > 4.0;
grunt> DUMP movies_greater_than_four;
grunt> STORE movies_greater_than_four INTO '/pig/output/movies_greater_than_four';
grunt> movies_between_50_60 = FILTER movies BY year > 1950 AND year < 1960;
grunt> STORE movies_between_50_60 INTO '/pig/output/movies_between_50_60';
grunt> movies_starting_with_A = FILTER movies BY name matches 'A.*';
grunt> STORE movies_starting_with_A INTO '/pig/output/movies_starting_with_A';
grunt> movies_duration_gt_2 = FILTER movies BY duration > 7200;
grunt> STORE movies_duration_gt_2 INTO '/pig/output/movies_duration_gt_2';
grunt> movies_rating_3_4 = FILTER movies BY rating > 3.0 AND rating < 4.0;
grunt> STORE movies_rating_3_4 INTO '/pig/output/movies_rating_3_4';
grunt> movies_duration = FOREACH movies GENERATE name, (double) (duration/60);
grunt> DUMP movies_duration;
grunt> movies_group_by_year = GROUP movies BY year;
grunt> DUMP movies_group_by_year;
grunt> movies_count_by_year = FOREACH movies_group_by_year GENERATE group, COUNT(movies); // Note: group here should be in lower case
grunt> DUMP movies_count_by_year;
grunt> movies_group_all = GROUP movies_count_by_year ALL;
grunt> movies_sum_all = FOREACH movies_group_all GENERATE SUM(movies_count_by_year.$1);
grunt> DUMP movies_sum_all;
(49590)
grunt> movies_asc = ORDER movies BY year ASC;
grunt> DUMP movies_asc;
grunt> movies_desc = ORDER movies BY year DESC;
grunt> DUMP movies_desc;
C:\hdp\hadoop-2.4.0.2.1.3.0-1981> hadoop fs -copyFromLocal C:\GitRepos\gudiseva\Hadoop\MayuraSamples\Pig\Input\movies_with_duplicates.txt /pig/input/movies_with_duplicates.txt
grunt> movies_with_dups = LOAD '/pig/input/movies_with_duplicates.txt' USING PigStorage (',') AS (id:int, name:chararray, year:int, rating:float, duration:int);
grunt> DUMP movies_with_dups;
grunt> movies_no_dups = DISTINCT movies_with_dups;
grunt> DUMP movies_no_dups;
grunt> movies_top_10 = LIMIT movies 10;
grunt> DUMP movies_top_10;
grunt> movies_sample = SAMPLE movies 0.1;
grunt> DUMP movies_sample;
grunt> movies_sample_group_all = GROUP movies_sample ALL;
grunt> movies_sample_count = FOREACH movies_sample_group_all GENERATE COUNT(movies_sample.$0);
grunt> DUMP movies_sample_count;
(4875)
# Data Set: Employee
C:\hdp\hadoop-2.4.0.2.1.3.0-1981> hadoop fs -copyFromLocal C:\GitRepos\gudiseva\Hadoop\MayuraSamples\Pig\Input\empsample.txt /pig/input/empsample.txt
grunt> employee = load '/pig/input/empsample.txt' USING PigStorage (',') AS (id:int, name:chararray, sal:int, loc:chararray);
grunt> DUMP employee;
grunt> ILLUSTRATE employee;
--------------------------------------------------------------------------------
| employee | id:int | name:chararray | sal:int | loc:chararray |
--------------------------------------------------------------------------------
| | 107 | name4 | 22222 | chennai |
--------------------------------------------------------------------------------
grunt> emp_name1 = FOREACH employee GENERATE name, sal;
grunt> DUMP emp_name1;
grunt> STORE emp_name1 INTO '/pig/output/emp_name1';
grunt> emp_name2 = FOREACH employee GENERATE $0, $3;
grunt> DUMP emp_name2;
grunt> emp_sal = FILTER employee BY sal < 2000;
grunt> DUMP emp_sal;
Table: emp (employee.txt)
------------------------
| eno,ename,esal,dno |
| 1,arvind,1000,1 |
| 2,ramesh,2000,2 |
| 3,haritha,4000,3 |
| 4,sailu,5000,4 |
------------------------
C:\hdp\hadoop-2.4.0.2.1.3.0-1981>hadoop fs -copyFromLocal C:\GitRepos\gudiseva\Hadoop\MayuraSamples\Pig\Input\employee.txt /pig/input/employee.txt
grunt> emp = load '/pig/input/employee.txt' USING PigStorage (',') AS (eno:int, ename:chararray, esal:float, dno:int);
2015-05-17 12:34:15,164 [main] WARN org.apache.pig.PigServer - Encountered Warning IMPLICIT_CAST_TO_DOUBLE 1 time(s).
Table: dept (department.txt)
------------------------
| dno,dname,location |
| 1,it,hyd |
| 2,hr,chn |
| 3,mr,blr |
| 4,sa,mum |
------------------------
C:\hdp\hadoop-2.4.0.2.1.3.0-1981>hadoop fs -copyFromLocal C:\GitRepos\gudiseva\Hadoop\MayuraSamples\Pig\Input\department.txt /pig/input/department.txt
grunt> dept = load '/pig/input/department.txt' USING PigStorage (',') AS (dno:int, dname:chararray, location:chararray);
2015-05-17 12:34:36,706 [main] WARN org.apache.pig.PigServer - Encountered Warning IMPLICIT_CAST_TO_DOUBLE 1 time(s).
grunt> emp_info = JOIN emp BY dno LEFT OUTER, dept BY dno;
2015-05-17 12:35:06,470 [main] WARN org.apache.pig.PigServer - Encountered Warning IMPLICIT_CAST_TO_DOUBLE 1 time(s).
grunt> dump emp_info;
grunt> quit;
---
### Local Mode
C:\hdp\hadoop-2.4.0.2.1.3.0-1981> pig -x local
grunt> employee = LOAD '/C:/GitRepos/gudiseva/Hadoop/MayuraSamples/Pig/Input/empsample.txt' USING PigStorage(',') AS (id:int, name:chararray, sal:int, loc:chararray);
grunt> emp_sal_group = GROUP employee BY sal;
grunt> emp_count = FOREACH emp_sal_group GENERATE group, COUNT (employee);
grunt> DUMP emp_count;
(10000,2)
(11111,1)
(22222,1)
References:
1. Apache Pig Tutorial – Part 1 (http://www.rohitmenon.com/index.php/apache-pig-tutorial-part-1/)
2. Apache Pig Tutorial – Part 2 (http://www.rohitmenon.com/index.php/apache-pig-tutorial-part-2/)
3. Code Samples and Dataset are available in GitHub (https://github.com/gudiseva/Hadoop.git)
---
No comments:
Post a Comment