dbt artifacts parser

DBT-ARTIFACTS-PARSER

Este é um artefato DBT analisado em Python. Isso nos permite lidar com catalog.json , manifest.json , run-results.json e sources.json como objetos python.

Este pacote foi projetado principalmente para o núcleo DBT, permitindo a interação perfeita com artefatos DBT como objetos Python. Enquanto o DBT Cloud fornece tipos de artefatos adicionais além dos do DBT-Core, este pacote não oferece suporte abrangente para eles.

Versões suportadas e compatibilidade

️ Nota importante:

• O Pydantic V1 não será suportado para DBT 1.9 ou posterior.
• Para analisar o DBT 1.9 ou posterior, migre seu código para o Pydantic V2.
• Reavaliaremos a compatibilidade da versão após o lançamento do Pydantic V3.

Versão	Versão DBT suportada	Versão Pydantic suportada
0,8	DBT 1,5 a 1.9	pydantic v2
0,7	DBT 1,5 a 1.8	pydantic v2
0,6	DBT 1,5 a 1.8	Pydantic v1
0,5	DBT 1.5 a 1.7	Pydantic v1

Instalação

pip install -U dbt-artifacts-parser

Classes Python

Essas são as classes para analisar artefatos DBT.

Catálogo

• Catalogv1 para catalog.json v1

Manifesto

• Manifestv1 para manifest.json v1
• Manifestv2 para manifest.json v2
• Manifestv3 para manifest.json v3
• Manifestv4 para manifest.json v4
• Manifestv5 para manifest.json v5
• Manifestv6 para manifest.json v6
• Manifestv7 para manifest.json v7
• Manifestv8 para manifest.json v8
• Manifestv9 para manifest.json v9
• Manifestv10 para manifest.json v10
• Manifestv11 para manifest.json v11
• Manifestv12 para manifest.json v12

Execute resultados

• RunResultsv1 para run_results.json v1
• RunResultsv2 para run_results.json v2
• RunResultsv3 para run_results.json v3
• RunResultsv4 para run_results.json v4
• RunResultsv5 para run_results.json v5
• RunResultsv6 para run_results.json v6

Fontes

• Fontesv1 para fontes.json v1
• Fontesv2 para fontes.json v2
• Fontesv3 para fontes.json v3

Exemplos

Parse catalog.json

 import json

# parse any version of catalog.json
from dbt_artifacts_parser . parser import parse_catalog

with open ( "path/to/catalog.json" , "r" ) as fp :
    catalog_dict = json . load ( fp )
    catalog_obj = parse_catalog ( catalog = catalog_dict )

# parse catalog.json v1
from dbt_artifacts_parser . parser import parse_catalog_v1

with open ( "path/to/catalog.json" , "r" ) as fp :
    catalog_dict = json . load ( fp )
    catalog_obj = parse_catalog_v1 ( catalog = catalog_dict )

Parse manifest.json

 import json

# parse any version of manifest.json
from dbt_artifacts_parser . parser import parse_manifest

with open ( "path/to/manifest.json" , "r" ) as fp :
    manifest_dict = json . load ( fp )
    manifest_obj = parse_manifest ( manifest = manifest_dict )

# parse manifest.json v1
from dbt_artifacts_parser . parser import parse_manifest_v1

with open ( "path/to/manifest.json" , "r" ) as fp :
    manifest_dict = json . load ( fp )
    manifest_obj = parse_manifest_v1 ( manifest = manifest_dict )

# parse manifest.json v2
from dbt_artifacts_parser . parser import parse_manifest_v2

with open ( "path/to/manifest.json" , "r" ) as fp :
    manifest_dict = json . load ( fp )
    manifest_obj = parse_manifest_v2 ( manifest = manifest_dict )

# parse manifest.json v3
from dbt_artifacts_parser . parser import parse_manifest_v3

with open ( "path/to/manifest.json" , "r" ) as fp :
    manifest_dict = json . load ( fp )
    manifest_obj = parse_manifest_v3 ( manifest = manifest_dict )

# parse manifest.json v4
from dbt_artifacts_parser . parser import parse_manifest_v4

with open ( "path/to/manifest.json" , "r" ) as fp :
    manifest_dict = json . load ( fp )
    manifest_obj = parse_manifest_v4 ( manifest = manifest_dict )

# parse manifest.json v5
from dbt_artifacts_parser . parser import parse_manifest_v5

with open ( "path/to/manifest.json" , "r" ) as fp :
    manifest_dict = json . load ( fp )
    manifest_obj = parse_manifest_v5 ( manifest = manifest_dict )

# parse manifest.json v6
from dbt_artifacts_parser . parser import parse_manifest_v6

with open ( "path/to/manifest.json" , "r" ) as fp :
    manifest_dict = json . load ( fp )
    manifest_obj = parse_manifest_v6 ( manifest = manifest_dict )

# parse manifest.json v7
from dbt_artifacts_parser . parser import parse_manifest_v7

with open ( "path/to/manifest.json" , "r" ) as fp :
    manifest_dict = json . load ( fp )
    manifest_obj = parse_manifest_v7 ( manifest = manifest_dict )

# parse manifest.json v8
from dbt_artifacts_parser . parser import parse_manifest_v8

with open ( "path/to/manifest.json" , "r" ) as fp :
    manifest_dict = json . load ( fp )
    manifest_obj = parse_manifest_v8 ( manifest = manifest_dict )

# parse manifest.json v9
from dbt_artifacts_parser . parser import parse_manifest_v9

with open ( "path/to/manifest.json" , "r" ) as fp :
    manifest_dict = json . load ( fp )
    manifest_obj = parse_manifest_v9 ( manifest = manifest_dict )

# parse manifest.json v10
from dbt_artifacts_parser . parser import parse_manifest_v10

with open ( "path/to/manifest.json" , "r" ) as fp :
    manifest_dict = json . load ( fp )
    manifest_obj = parse_manifest_v10 ( manifest = manifest_dict )

# parse manifest.json v11
from dbt_artifacts_parser . parser import parse_manifest_v11

with open ( "path/to/manifest.json" , "r" ) as fp :
    manifest_dict = json . load ( fp )
    manifest_obj = parse_manifest_v11 ( manifest = manifest_dict )

# parse manifest.json v12
from dbt_artifacts_parser . parser import parse_manifest_v12

with open ( "path/to/manifest.json" , "r" ) as fp :
    manifest_dict = json . load ( fp )
    manifest_obj = parse_manifest_v12 ( manifest = manifest_dict )

Parse correndo-results.json

 import json

# parse any version of run-results.json
from dbt_artifacts_parser . parser import parse_run_results

with open ( "path/to/run-resultsjson" , "r" ) as fp :
    run_results_dict = json . load ( fp )
    run_results_obj = parse_run_results ( run_results = run_results_dict )

# parse run-results.json v1
from dbt_artifacts_parser . parser import parse_run_results_v1

with open ( "path/to/run-results.json" , "r" ) as fp :
    run_results_dict = json . load ( fp )
    run_results_obj = parse_run_results_v1 ( run_results = run_results_dict )

# parse run-results.json v2
from dbt_artifacts_parser . parser import parse_run_results_v2

with open ( "path/to/run-results.json" , "r" ) as fp :
    run_results_dict = json . load ( fp )
    run_results_obj = parse_run_results_v2 ( run_results = run_results_dict )

# parse run-results.json v3
from dbt_artifacts_parser . parser import parse_run_results_v3

with open ( "path/to/run-results.json" , "r" ) as fp :
    run_results_dict = json . load ( fp )
    run_results_obj = parse_run_results_v3 ( run_results = run_results_dict )

# parse run-results.json v4
from dbt_artifacts_parser . parser import parse_run_results_v4

with open ( "path/to/run-results.json" , "r" ) as fp :
    run_results_dict = json . load ( fp )
    run_results_obj = parse_run_results_v4 ( run_results = run_results_dict )

# parse run-results.json v5
from dbt_artifacts_parser . parser import parse_run_results_v5

with open ( "path/to/run-results.json" , "r" ) as fp :
    run_results_dict = json . load ( fp )
    run_results_obj = parse_run_results_v5 ( run_results = run_results_dict )

# parse run-results.json v6
from dbt_artifacts_parser . parser import parse_run_results_v6

with open ( "path/to/run-results.json" , "r" ) as fp :
    run_results_dict = json . load ( fp )
    run_results_obj = parse_run_results_v6 ( run_results = run_results_dict )

Parse fontes.json

 import json

# parse any version of sources.json
from dbt_artifacts_parser . parser import parse_sources

with open ( "path/to/sources.json" , "r" ) as fp :
    sources_dict = json . load ( fp )
    sources_obj = parse_sources ( sources = sources_dict )

# parse sources.json v1
from dbt_artifacts_parser . parser import parse_sources_v1

with open ( "path/to/sources.json" , "r" ) as fp :
    sources_dict = json . load ( fp )
    sources_obj = parse_sources_v1 ( sources = sources_dict )

# parse sources.json v2
from dbt_artifacts_parser . parser import parse_sources_v2

with open ( "path/to/sources.json" , "r" ) as fp :
    sources_dict = json . load ( fp )
    sources_obj = parse_sources_v2 ( sources = sources_dict )

# parse sources.json v3
from dbt_artifacts_parser . parser import parse_sources_v3

with open ( "path/to/sources.json" , "r" ) as fp :
    sources_dict = json . load ( fp )
    sources_obj = parse_sources_v3 ( sources = sources_dict )

Colaboradores

Obrigado por suas contribuições!

Se você estiver interessado em contribuir para este pacote, consulte o contribuinte.md.

Yu Ishikawa	Nulo	Brent Brewington	Judah Rand	Mayur Singal	Mohamed Nabil Mahmoud Hafez
Onkar Ravgan	Nulo