Scribd
Upload
11 views

DBT Cheat Document

Uploaded by

Kalyan Bezawada
Copyright
© © All Rights Reserved
Available Formats
Download as DOCX, PDF, TXT or read online on Scribd
11 views

DBT Cheat Document

Uploaded by

Kalyan Bezawada
Copyright
© © All Rights Reserved
Available Formats
Download as DOCX, PDF, TXT or read online on Scribd
You are on page 1/ 15

MODELS:

==================================================
=============================
Models are nothing but select statements, below are the different types of
materializations (type of object that should be created in the database)
Example:
VIEWS
TABLES
EPHMERAL
INCREMENTAL
By Default its view , if its table its Transient below is the config change needed in
the model file
==================================================
=============================
{{
config(
materialized='table',
transient=false,
secure=true,
query_tag='dbt'
)
}}
SELECT 1 ID,'p' nm
dbt run =>it will execute all the models
dbt run -m abc
dbt run -s abc
dbt run --select abc
dbt run --model abc
dbt run --exclude m1 m2 ( Runs except m1 m2 models)
==================================================
=============================
Types of Test
Singular or Generic Tests
DATA TESTING :

2 TYPES OF TEST??
SINGULAR TEST :
APPLICABLE FR ONLY 1 MODEL

GENERIC TEST : PREBUILT AND CUSTOM

UNIQUE
NOT_NULL
ACCEPTED_VALUES
RELATIONSHIP
dbt source:
Document can be generated using dbt docs generate
dbt seeds:
if there is no change to the exisitng seed you can run the seed with the model name
Example : dbt seed -m modelname
if there is change to the file strucuture you need to do afull refresh
Example : dbt seed -m filename --full-refresh
if you want refer a table or view or any other object in dbt you need to use
ref{'object'} predifened macro
==================================================
============================
How a dbt project is operated ?
dbt_project.yml (configuration yml) =>profiles.yml => credentials

dbt variables:
select * from {{ source('pr', 'COWIN_VACC_REG_TABLE') }}
sources:
- name: src31_db
database: AADHAR_DEV_DB
schema: AADHAR_SC
tables:
- name: AADHAAR_REG_T
- name: pr
database: JULY_DEV_DB
schema: JULY_SC
tables:
- name: COWIN_VACC_REG_TABLE
{{
config(
materialized='table',
transient=false,
alias='praveen_tbl',
query_tag='DBT_MODELS'
)
}}
Using it in the code:
{% set country_name= 'India' %}
SELECT * FROM {{ source('uat', 'sales_details') }}
--where country= 'India'
--where country = '{{country_name}}'
where country = '{{ var('country_name') }}
select *,'{{ var('src_country_name') }}' from {{ ref('country_stg') }}
{{
config(
materialized='table',
alias='TYE',
query_tag='TTT',
transient=false
)
}}
with cte
as (
select 1 id,'pravee' name from dual )
select * from cte;
{% macro clone_database(db_name,schema_name,tables) %}
{% for table in tables %}
{%set clone_sqlquery %}
CREATE OR REPLACE TABLE {{ db_name }}.{{ schema_name }}.
{{ table }}_cloned
CLONE {{ db_name }}.{{ schema_name }}.{{ table }};
{% endset %}
{% do run_query(clone_sqlquery) %}
{% endfor %}

{% endmacro %}

db_name,schema_name,tables
dev_db.dev_sc,t1,t2,t3;

CREATE OR REPLACE TABLE dev_db.dev_sc.t1_cloned


CLONE t1;

dbt run-operation clone_database --args '{"db_name": "DBT_DB",


"schema_name": "DBT_SC", "tables": ["table1","table2",
"table3","table4"]}'
chatgpt
gemini
perpezixlty
{% macro clone_mco_ddb(args) %}
{% set clone_query %}
create or replace database clond_db clone old_dab;
{% endset %}
{% do run_query(clone_query) %}
{% endmacro %}
dbt run-operation clone_database --args '{"src_db": "DBT_DB", "src_sc":
"DBT_SC","trg_db": "DBT_TRG_DB", "trg_sc": "DBT_TRG_SC",
"tables": ["table1","table2", "table3","table4"]}'

{% macro clone_db_tables_mco(src_db,src_sc,trg_db,trg_sc,table_list) %}
{% set create_db %}
CREATE OR REPLACE DATABASE {{trg_db}};
{% endset%}
{% do run_query(create_db) %}

{% set create_sc %}
CREATE OR REPLACE SCHEMA {{trg_db}}.{{trg_sc}};
{% endset%}
{% do run_query(create_sc) %}

{% for tbl in table_list %}


{% set source_table= src_db~'.'~src_sc~'.'~tbl %}
{% set target_table= trg_db~'.'~trg_sc~'.'~tbl %}
{% set clone_table %}
CREATE OR REPLACE TABLE {{target_table}} CLONE
{{source_table}};
{% endset%}
{% do run_query(clone_table) %}

{% endfor %}
{% endmacro %}
dbt run-operation clone_db_tables_mco --args
"{'src_db':'DEV_DB','src_sc':'RAW_SC','trg_db':'UAT_DB','trg_sc':'RAW_SC','ta
ble_list':['T1','T2','T3']}"

dbt run-operation clone_db_tables_mco --args


"{'src_db':'DEV_DB','src_sc':'RAW_SC','trg_db':'UAT_DB','trg_sc':'RAW_SC','ta
ble_list':['T1','T2','T3']}"

src_db,src_sc,trg_db,trg_sc,table_list

se warehouse wh;

hooks : prehooks => before executing


posthooks=> after executing

car_sales model ??
execute
use warehouse comute_wh;

after executing transform wh


vars:
src_country_name: 'Australia'
des_country_name: 'India'
country_name : '''India'''
vwh: 'COMPUTE_WH'
default_wh: 'TRANSFORM_WH'
{% macro set_warehouse(warehouse_name) %}
use warehouse {{warehouse_name}};
{% endmacro %}

{% macro default_warehouse(warehouse_name) %}
use warehouse {{warehouse_name}};
{% endmacro %}

--SELECT * FROM DEV_DB.STG_SCHEMA.ABC_SALES_DETAILS

{{
config(
materialized='table',
transient=false,
query_tag='DBT_MODELS',
pre_hook=["{{set_warehouse(var('vwh'))}}"],
post_hook=["{{default_warehouse(var('default_wh'))}}"]
)
}}

SELECT * FROM {{ source('stg_prv', 'ABC_SALES_DETAILS') }}


WHERE country= {{ var('country_name') }}

Snapshots:
SCD 1 : OVERWRITE
SCD 2 : HISTORICAL DATA
ID ,NAME ,CITY
1,PRAVEEN,HYD --CURRENT DATA

AFTER 1 YR =>BNG
1,PRAVEEN,BNG --SCD1

PRAVEEN TRAINEE 2020

---
AFTER 2 YEARS 2022 -DVELOPER

START_DATE END_DATE
PRAVEEN TRAINEE 01-JAN-2020 01-JAN-2022
PRAVEEN associa 01-JAN-2022 null
scd2

------------------
in dbt
scd 2 ???
snapshot

2 startegies in snaphost
timestamp
check_cols
emp.csv
id,name,emp_status
1,praveen,active
22,kumar,active

{% snapshot emp_dtls_snp %}
{{
config(
target_database= 'trg_db',
target_schema= 'trg_sc',
unique_key= 'id',
strategy='check',
check_cols=['emp_status',],
invalidate_hard_deletes=true
)
}}

--Snapshots must be configured with a 'strategy', 'unique_key', and


'target_schema'. praveen
select * from {{ ref('emp_details') }}

{% endsnapshot %}

dbt snapshot -m emp_dtls_snp

You might also like