Differences

This shows you the differences between two versions of the page.

Link to this comparison view

Both sides previous revision Previous revision
kb:hive [2018/11/13 12:32]
yehuda
kb:hive [2018/11/13 12:37] (current)
yehuda
Line 13: Line 13:
 -- source system. After the first processing run, it will house the on-going, most up-to-date set of records from the source system: -- source system. After the first processing run, it will house the on-going, most up-to-date set of records from the source system:
 -- --
- 
- 
 CREATE TABLE base_table ( CREATE TABLE base_table (
 id string, id string,
-field1 string, +field1 string, field2 string, field3 string, field4 string, field5 string,
-field2 string, +
-field3 string, +
-field4 string, +
-field5 string,+
 modified_date string modified_date string
 ) )
Line 32: Line 26:
 -- any delimited files with incremental change records, located in HDFS under the ‘/​user/​hive/​incremental_append’ ​ -- any delimited files with incremental change records, located in HDFS under the ‘/​user/​hive/​incremental_append’ ​
 -- directory: -- directory:
- 
- 
- 
 CREATE EXTERNAL TABLE incremental_table ( CREATE EXTERNAL TABLE incremental_table (
 id string, id string,
-field1 string, +field1 string, field2 string, field3 string, field4 string, field5 string,
-field2 string, +
-field3 string, +
-field4 string, +
-field5 string,+
 modified_date string modified_date string
 ) )
Line 52: Line 39:
 -- Change (incremental_table) tables and is reduced only to the most recent records for each unique “id”. ​ -- Change (incremental_table) tables and is reduced only to the most recent records for each unique “id”. ​
 -- It is defined as follows: -- It is defined as follows:
- 
- 
 CREATE VIEW reconcile_view AS CREATE VIEW reconcile_view AS
 SELECT t1.* FROM SELECT t1.* FROM
Line 63: Line 48:
  
 JOIN JOIN
 +
 ( (
    ​SELECT id, max(modified_date) max_modified ​    ​SELECT id, max(modified_date) max_modified ​
kb/hive.txt · Last modified: 2018/11/13 12:37 by yehuda
Back to top
Driven by DokuWiki Recent changes RSS feed Valid CSS Valid XHTML 1.0