import hashlib import json import os import random import time from datetime import datetime from unittest.mock import ANY, patch from test.integration.base import DBTIntegrationTest, use_profile, AnyFloat, \ AnyString, AnyStringWith, normalize, Normalized from dbt.exceptions import CompilationException def _read_file(path): with open(path, 'r') as fp: return fp.read().replace('\r', '').replace('\\r', '') class LineIndifferent: def __init__(self, expected): self.expected = expected.replace('\r', '') def __eq__(self, other): return self.expected == other.replace('\r', '') def __repr__(self): return 'LineIndifferent({!r})'.format(self.expected) def __str__(self): return self.__repr__() class OneOf: def __init__(self, *options): self.options = options def __eq__(self, other): return any(o == other for o in self.options) def __repr__(self): return 'OneOf(*{!r})'.format(self.options) def __str__(self): return self.__repr__() def _read_json(path): # read json generated by dbt. with open(path) as fp: return json.load(fp) def walk_files(path): for root, dirs, files in os.walk(path): for basename in files: yield os.path.join(root, basename) class TestDocsGenerateEscapes(DBTIntegrationTest): prefix = "pgtest{}{:04}".format(int(time.time()), random.randint(0, 9999)) @property def schema(self): return 'docs_generate_029' @staticmethod def dir(path): return normalize(path) @property def models(self): return self.dir("trivial_models") def run_and_generate(self): self.assertEqual(len(self.run_dbt(['run'])), 1) os.remove(normalize('target/manifest.json')) os.remove(normalize('target/run_results.json')) self.run_dbt(['docs', 'generate']) @use_profile('postgres') def test_postgres_include_schema(self): self.run_and_generate() manifest = _read_json('./target/manifest.json') self.assertIn('nodes', manifest) self.assertEqual(len(manifest['nodes']), 1) self.assertIn('model.test.model', manifest['nodes']) self.assertIn('schema', manifest['nodes']['model.test.model']) self.assertEqual('pg', manifest['nodes']['model.test.model']['schema'][:2]) class TestDocsGenerate(DBTIntegrationTest): setup_alternate_db = True def setUp(self): super().setUp() self.maxDiff = None @property def schema(self): return 'docs_generate_029' @staticmethod def dir(path): return normalize(path) @property def models(self): return self.dir("models") @property def packages_config(self): return { 'packages': [ { 'git': 'https://github.com/fishtown-analytics/dbt-integration-project', 'revision': 'dbt/0.17.0', }, ], } @property def project_config(self): return { 'config-version': 2, 'quoting': { 'identifier': False } } def run_and_generate(self, extra=None, seed_count=1, model_count=1, alternate_db=None, args=None): if alternate_db is None: alternate_db = self.alternative_database project = { "data-paths": [self.dir("seed")], 'macro-paths': [self.dir('macros')], 'vars': { 'alternate_db': alternate_db, }, 'seeds': { 'quote_columns': True, }, } if extra: project.update(extra) self.use_default_project(project) vars_arg = '--vars={{test_schema: {}}}'.format(self.unique_schema()) self.assertEqual(len(self.run_dbt(["seed", vars_arg])), seed_count) self.assertEqual(len(self.run_dbt(['run', vars_arg])), model_count) os.remove(normalize('target/manifest.json')) os.remove(normalize('target/run_results.json')) self.generate_start_time = datetime.utcnow() base_args = ['docs', 'generate', vars_arg] if args: base_args.extend(args) self.run_dbt(base_args) def _no_stats(self): return { 'has_stats': { 'id': 'has_stats', 'label': 'Has Stats?', 'value': False, 'description': 'Indicates whether there are statistics for this table', 'include': False, }, } def _redshift_stats(self): return { "has_stats": { "id": "has_stats", "label": "Has Stats?", "value": True, "description": "Indicates whether there are statistics for this table", "include": False }, "encoded": { "id": "encoded", "label": "Encoded", "value": "Y", "description": "Indicates whether any column in the table has compression encoding defined.", "include": True }, "diststyle": { "id": "diststyle", "label": "Dist Style", "value": AnyStringWith(None), "description": "Distribution style or distribution key column, if key distribution is defined.", "include": True }, "max_varchar": { "id": "max_varchar", "label": "Max Varchar", "value": AnyFloat(), "description": "Size of the largest column that uses a VARCHAR data type.", "include": True }, "size": { "id": "size", "label": "Approximate Size", "value": AnyFloat(), "description": "Approximate size of the table, calculated from a count of 1MB blocks", "include": True }, "pct_used": { "id": "pct_used", "label": "Disk Utilization", "value": AnyFloat(), "description": "Percent of available space that is used by the table.", "include": True }, "stats_off": { "id": "stats_off", "label": "Stats Off", "value": AnyFloat(), "description": "Number that indicates how stale the table statistics are; 0 is current, 100 is out of date.", "include": True }, "rows": { "id": "rows", "label": "Approximate Row Count", "value": AnyFloat(), "description": "Approximate number of rows in the table. This value includes rows marked for deletion, but not yet vacuumed.", "include": True }, } def _snowflake_stats(self): return { 'has_stats': { 'id': 'has_stats', 'label': 'Has Stats?', 'value': True, 'description': 'Indicates whether there are statistics for this table', 'include': False, }, 'bytes': { 'id': 'bytes', 'label': 'Approximate Size', 'value': AnyFloat(), 'description': 'Approximate size of the table as reported by Snowflake', 'include': True, }, 'row_count': { 'id': 'row_count', 'label': 'Row Count', 'value': 1.0, 'description': 'An approximate count of rows in this table', 'include': True, }, } def _bigquery_stats(self, is_table, partition=None, cluster=None): stats = {} if is_table: stats.update({ 'num_bytes': { 'id': 'num_bytes', 'label': AnyString(), 'value': AnyFloat(), 'description': AnyString(), 'include': True, }, 'num_rows': { 'id': 'num_rows', 'label': AnyString(), 'value': AnyFloat(), 'description': AnyString(), 'include': True, } }) if partition is not None: stats.update({ 'partitioning_type': { 'id': 'partitioning_type', 'label': AnyString(), 'value': partition, 'description': AnyString(), 'include': True } }) if cluster is not None: stats.update({ 'clustering_fields': { 'id': 'clustering_fields', 'label': AnyString(), 'value': cluster, 'description': AnyString(), 'include': True } }) has_stats = { 'id': 'has_stats', 'label': 'Has Stats?', 'value': bool(stats), 'description': 'Indicates whether there are statistics for this table', 'include': False, } stats['has_stats'] = has_stats return stats def _expected_catalog(self, id_type, text_type, time_type, view_type, table_type, model_stats, seed_stats=None, case=None, case_columns=False, model_database=None): if case is None: case = lambda x: x col_case = case if case_columns else lambda x: x if seed_stats is None: seed_stats = model_stats if model_database is None: model_database = self.default_database my_schema_name = self.unique_schema() role = self.get_role() expected_cols = { col_case('id'): { 'name': col_case('id'), 'index': 1, 'type': id_type, 'comment': None, }, col_case('first_name'): { 'name': col_case('first_name'), 'index': 2, 'type': text_type, 'comment': None, }, col_case('email'): { 'name': col_case('email'), 'index': 3, 'type': text_type, 'comment': None, }, col_case('ip_address'): { 'name': col_case('ip_address'), 'index': 4, 'type': text_type, 'comment': None, }, col_case('updated_at'): { 'name': col_case('updated_at'), 'index': 5, 'type': time_type, 'comment': None, }, } return { 'nodes': { 'model.test.model': { 'unique_id': 'model.test.model', 'metadata': { 'schema': my_schema_name, 'database': model_database, 'name': case('model'), 'type': view_type, 'comment': None, 'owner': role, }, 'stats': model_stats, 'columns': expected_cols, }, 'seed.test.seed': { 'unique_id': 'seed.test.seed', 'metadata': { 'schema': my_schema_name, 'database': self.default_database, 'name': case('seed'), 'type': table_type, 'comment': None, 'owner': role, }, 'stats': seed_stats, 'columns': expected_cols, }, }, 'sources': {} } def expected_postgres_catalog(self): return self._expected_catalog( id_type='integer', text_type='text', time_type='timestamp without time zone', view_type='VIEW', table_type='BASE TABLE', model_stats=self._no_stats() ) def get_role(self): if self.adapter_type in {'postgres', 'redshift'}: profile = self.get_profile(self.adapter_type) target_name = profile['test']['target'] return profile['test']['outputs'][target_name]['user'] elif self.adapter_type == 'snowflake': return self.run_sql('select current_role()', fetch='one')[0] else: # bigquery, presto, other dbs that have no 'role' return None def expected_postgres_references_catalog(self): model_database = self.default_database my_schema_name = self.unique_schema() role = self.get_role() stats = self._no_stats() summary_columns = { 'first_name': { 'name': 'first_name', 'index': 1, 'type': 'text', 'comment': None, }, 'ct': { 'name': 'ct', 'index': 2, 'type': 'bigint', 'comment': None, }, } seed_columns = { 'id': { 'name': 'id', 'index': 1, 'type': 'integer', 'comment': None, }, 'first_name': { 'name': 'first_name', 'index': 2, 'type': 'text', 'comment': None, }, 'email': { 'name': 'email', 'index': 3, 'type': 'text', 'comment': None, }, 'ip_address': { 'name': 'ip_address', 'index': 4, 'type': 'text', 'comment': None, }, 'updated_at': { 'name': 'updated_at', 'index': 5, 'type': 'timestamp without time zone', 'comment': None, }, } return { 'nodes': { 'seed.test.seed': { 'unique_id': 'seed.test.seed', 'metadata': { 'schema': my_schema_name, 'database': self.default_database, 'name': 'seed', 'type': 'BASE TABLE', 'comment': None, 'owner': role, }, 'stats': stats, 'columns': seed_columns }, 'model.test.ephemeral_summary': { 'unique_id': 'model.test.ephemeral_summary', 'metadata': { 'schema': my_schema_name, 'database': model_database, 'name': 'ephemeral_summary', 'type': 'BASE TABLE', 'comment': None, 'owner': role, }, 'stats': stats, 'columns': summary_columns, }, 'model.test.view_summary': { 'unique_id': 'model.test.view_summary', 'metadata': { 'schema': my_schema_name, 'database': model_database, 'name': 'view_summary', 'type': 'VIEW', 'comment': None, 'owner': role, }, 'stats': stats, 'columns': summary_columns, }, }, 'sources': { "source.test.my_source.my_table": { "unique_id": "source.test.my_source.my_table", "metadata": { 'schema': my_schema_name, 'database': self.default_database, 'name': 'seed', 'type': 'BASE TABLE', 'comment': None, 'owner': role, }, "stats": stats, 'columns': seed_columns, }, }, } def expected_snowflake_catalog(self, case_columns=False): return self._expected_catalog( id_type='NUMBER', text_type='TEXT', time_type='TIMESTAMP_NTZ', view_type='VIEW', table_type='BASE TABLE', model_stats=self._no_stats(), seed_stats=self._snowflake_stats(), case=lambda x: x.upper(), model_database=self.alternative_database, case_columns=case_columns, ) def expected_bigquery_catalog(self): return self._expected_catalog( id_type='INT64', text_type='STRING', time_type='DATETIME', view_type='view', table_type='table', model_stats=self._bigquery_stats(False), seed_stats=self._bigquery_stats(True), model_database=self.alternative_database, ) def expected_presto_catalog(self): return self._expected_catalog( id_type='integer', text_type='varchar', time_type='timestamp', view_type='VIEW', table_type='BASE TABLE', model_stats=self._no_stats(), seed_stats=self._no_stats(), model_database=self.default_database, ) @staticmethod def _clustered_bigquery_columns(update_type): return { 'id': { 'comment': None, 'index': 1, 'name': 'id', 'type': 'INT64', }, 'first_name': { 'comment': None, 'index': 2, 'name': 'first_name', 'type': 'STRING', }, 'email': { 'comment': None, 'index': 3, 'name': 'email', 'type': 'STRING', }, 'ip_address': { 'comment': None, 'index': 4, 'name': 'ip_address', 'type': 'STRING', }, 'updated_at': { 'comment': None, 'index': 5, 'name': 'updated_at', 'type': update_type, }, } def expected_bigquery_complex_catalog(self): my_schema_name = self.unique_schema() role = self.get_role() table_stats = self._bigquery_stats(True) clustering_stats = self._bigquery_stats(True, partition='updated_at', cluster='first_name') multi_clustering_stats = self._bigquery_stats(True, partition='updated_at', cluster='first_name, email') nesting_columns = { 'field_1': { 'name': 'field_1', 'index': 1, 'type': 'INT64', 'comment': None }, 'field_2': { 'name': 'field_2', 'index': 2, 'type': 'INT64', 'comment': None }, 'field_3': { 'name': 'field_3', 'index': 3, 'type': 'INT64', 'comment': None }, 'nested_field.field_4': { 'name': 'nested_field.field_4', 'index': 4, 'type': 'INT64', 'comment': None }, 'nested_field.field_5': { 'name': 'nested_field.field_5', 'index': 5, 'type': 'INT64', 'comment': None } } return { 'nodes': { 'model.test.clustered': { 'unique_id': 'model.test.clustered', 'metadata': { 'comment': None, 'name': 'clustered', 'owner': None, 'schema': my_schema_name, 'database': self.default_database, 'type': 'table' }, 'stats': clustering_stats, 'columns': self._clustered_bigquery_columns('DATE'), }, 'model.test.multi_clustered': { 'unique_id': 'model.test.multi_clustered', 'metadata': { 'comment': None, 'name': 'multi_clustered', 'owner': None, 'schema': my_schema_name, 'database': self.default_database, 'type': 'table' }, 'stats': multi_clustering_stats, 'columns': self._clustered_bigquery_columns('DATE'), }, 'seed.test.seed': { 'unique_id': 'seed.test.seed', 'metadata': { 'comment': None, 'name': 'seed', 'owner': None, 'schema': my_schema_name, 'database': self.default_database, 'type': 'table', }, 'stats': table_stats, 'columns': self._clustered_bigquery_columns('DATETIME'), }, 'model.test.nested_view': { 'unique_id': 'model.test.nested_view', 'metadata': { 'schema': my_schema_name, 'database': self.default_database, 'name': 'nested_view', 'type': 'view', 'owner': role, 'comment': None }, 'stats': self._bigquery_stats(False), 'columns': nesting_columns, }, 'model.test.nested_table': { 'unique_id': 'model.test.nested_table', 'metadata': { 'schema': my_schema_name, 'database': self.default_database, 'name': 'nested_table', 'type': 'table', 'owner': role, 'comment': None }, 'stats': table_stats, 'columns': nesting_columns, }, }, 'sources': {}, } def expected_redshift_catalog(self): return self._expected_catalog( id_type='integer', text_type=AnyStringWith('character varying'), time_type='timestamp without time zone', view_type='VIEW', table_type='BASE TABLE', model_stats=self._no_stats(), seed_stats=self._redshift_stats(), ) def expected_redshift_incremental_catalog(self): my_schema_name = self.unique_schema() role = self.get_role() return { 'nodes': { 'model.test.model': { 'unique_id': 'model.test.model', 'metadata': { 'schema': my_schema_name, 'database': self.default_database, 'name': 'model', 'type': 'LATE BINDING VIEW', 'comment': None, 'owner': role, }, # incremental views have no stats 'stats': self._no_stats(), 'columns': { 'id': { 'name': 'id', 'index': 1, 'type': 'integer', 'comment': None, }, 'first_name': { 'name': 'first_name', 'index': 2, 'type': 'character varying(5)', 'comment': None, }, 'email': { 'name': 'email', 'index': 3, 'type': 'character varying(23)', 'comment': None, }, 'ip_address': { 'name': 'ip_address', 'index': 4, 'type': 'character varying(14)', 'comment': None, }, 'updated_at': { 'name': 'updated_at', 'index': 5, 'type': 'timestamp without time zone', 'comment': None, }, }, }, 'seed.test.seed': { 'unique_id': 'seed.test.seed', 'metadata': { 'schema': my_schema_name, 'database': self.default_database, 'name': 'seed', 'type': 'BASE TABLE', 'comment': None, 'owner': role, }, 'stats': self._redshift_stats(), 'columns': { 'id': { 'name': 'id', 'index': 1, 'type': 'integer', 'comment': None, }, 'first_name': { 'name': 'first_name', 'index': 2, 'type': 'character varying(5)', 'comment': None, }, 'email': { 'name': 'email', 'index': 3, 'type': 'character varying(23)', 'comment': None, }, 'ip_address': { 'name': 'ip_address', 'index': 4, 'type': 'character varying(14)', 'comment': None, }, 'updated_at': { 'name': 'updated_at', 'index': 5, 'type': 'timestamp without time zone', 'comment': None, }, }, }, }, 'sources': {}, } def verify_catalog(self, expected): self.assertTrue(os.path.exists('./target/catalog.json')) catalog = _read_json('./target/catalog.json') self.assertIn('generated_at', catalog) self.assertBetween( catalog.pop('generated_at'), start=self.generate_start_time, ) for key in 'nodes', 'sources': self.assertEqual(catalog[key], expected[key]) def verify_manifest_macros(self, manifest, expected=None): self.assertIn('macros', manifest) if expected is None: self._verify_generic_macro_structure(manifest) return for unique_id, expected_macro in expected.items(): self.assertIn(unique_id, manifest['macros']) actual_macro = manifest['macros'][unique_id] self.assertEqual(expected_macro, actual_macro) def _verify_generic_macro_structure(self, manifest): # just test a known global macro to avoid having to update this every # time they change. self.assertIn('macro.dbt.column_list', manifest['macros']) macro = manifest['macros']['macro.dbt.column_list'] self.assertEqual( set(macro), { 'path', 'original_file_path', 'package_name', 'root_path', 'name', 'unique_id', 'tags', 'resource_type', 'depends_on', 'meta', 'description', 'patch_path', 'arguments', 'macro_sql', 'docs' } ) # Don't compare the sql, just make sure it exists self.assertTrue(len(macro['macro_sql']) > 10) without_sql = { k: v for k, v in macro.items() if k not in {'macro_sql'} } # Windows means we can't hard-code these. helpers_path = Normalized('macros/materializations/helpers.sql') root_path = Normalized(os.path.join( self.dbt_core_install_root, 'include', 'global_project' )) self.assertEqual( { 'path': helpers_path, 'original_file_path': helpers_path, 'package_name': 'dbt', 'root_path': root_path, 'name': 'column_list', 'unique_id': 'macro.dbt.column_list', 'tags': [], 'resource_type': 'macro', 'depends_on': {'macros': []}, 'description': '', 'docs': {'show': True}, 'patch_path': None, 'meta': {}, 'arguments': [], }, without_sql, ) def expected_seeded_manifest(self, model_database=None): models_path = self.dir('models') model_sql_path = os.path.join(models_path, 'model.sql') model_schema_yml_path = os.path.join(models_path, 'schema.yml') seed_schema_yml_path = os.path.join(self.dir('seed'), 'schema.yml') my_schema_name = self.unique_schema() if model_database is None: model_database = self.alternative_database model_config = { 'database': model_database, 'enabled': True, 'materialized': 'view', 'pre-hook': [], 'post-hook': [], 'vars': {}, 'column_types': {}, 'quoting': {}, 'tags': [], 'persist_docs': {}, 'full_refresh': None, } return { 'nodes': { 'model.test.model': { 'build_path': Normalized('target/compiled/test/models/model.sql'), 'name': 'model', 'root_path': self.test_root_dir, 'resource_type': 'model', 'path': 'model.sql', 'original_file_path': model_sql_path, 'package_name': 'test', 'raw_sql': LineIndifferent(_read_file(model_sql_path).rstrip('\r\n')), 'refs': [['seed']], 'sources': [], 'depends_on': {'nodes': ['seed.test.seed'], 'macros': []}, 'unique_id': 'model.test.model', 'fqn': ['test', 'model'], 'tags': [], 'meta': {}, 'config': model_config, 'schema': my_schema_name, 'database': model_database, 'alias': 'model', 'description': 'The test model', 'columns': { 'id': { 'name': 'id', 'description': 'The user ID number', 'data_type': None, 'meta': {}, 'tags': [], }, 'first_name': { 'name': 'first_name', 'description': "The user's first name", 'data_type': None, 'meta': {}, 'tags': [], }, 'email': { 'name': 'email', 'description': "The user's email", 'data_type': None, 'meta': {}, 'tags': [], }, 'ip_address': { 'name': 'ip_address', 'description': "The user's IP address", 'data_type': None, 'meta': {}, 'tags': [], }, 'updated_at': { 'name': 'updated_at', 'description': "The last time this user's email was updated", 'data_type': None, 'meta': {}, 'tags': [], }, }, 'patch_path': model_schema_yml_path, 'docs': {'show': False}, 'compiled': True, 'compiled_sql': ANY, 'extra_ctes_injected': True, 'extra_ctes': [], 'injected_sql': ANY, }, 'seed.test.seed': { 'build_path': None, 'compiled': True, 'compiled_sql': '', 'config': { 'enabled': True, 'materialized': 'seed', 'persist_docs': {}, 'pre-hook': [], 'post-hook': [], 'vars': {}, 'column_types': {}, 'quoting': {}, 'tags': [], 'quote_columns': True, 'full_refresh': None, }, 'patch_path': seed_schema_yml_path, 'path': 'seed.csv', 'name': 'seed', 'root_path': self.test_root_dir, 'resource_type': 'seed', 'raw_sql': '', 'package_name': 'test', 'original_file_path': self.dir(os.path.join('seed', 'seed.csv')), 'refs': [], 'sources': [], 'depends_on': {'nodes': [], 'macros': []}, 'unique_id': 'seed.test.seed', 'fqn': ['test', 'seed'], 'tags': [], 'meta': {}, 'schema': my_schema_name, 'database': self.default_database, 'alias': 'seed', 'description': 'The test seed', 'columns': { 'id': { 'name': 'id', 'description': 'The user ID number', 'data_type': None, 'meta': {}, 'tags': [], }, 'first_name': { 'name': 'first_name', 'description': "The user's first name", 'data_type': None, 'meta': {}, 'tags': [], }, 'email': { 'name': 'email', 'description': "The user's email", 'data_type': None, 'meta': {}, 'tags': [], }, 'ip_address': { 'name': 'ip_address', 'description': "The user's IP address", 'data_type': None, 'meta': {}, 'tags': [], }, 'updated_at': { 'name': 'updated_at', 'description': "The last time this user's email was updated", 'data_type': None, 'meta': {}, 'tags': [], }, }, 'docs': {'show': True}, 'compiled': True, 'compiled_sql': '', 'extra_ctes_injected': True, 'extra_ctes': [], 'injected_sql': '', }, 'test.test.not_null_model_id': { 'alias': 'not_null_model_id', 'build_path': Normalized('target/compiled/test/models/schema.yml/schema_test/not_null_model_id.sql'), 'column_name': 'id', 'columns': {}, 'config': { 'column_types': {}, 'enabled': True, 'materialized': 'view', 'persist_docs': {}, 'post-hook': [], 'pre-hook': [], 'quoting': {}, 'vars': {}, 'tags': [], 'severity': 'ERROR', 'full_refresh': None, }, 'sources': [], 'depends_on': { 'macros': ['macro.dbt.test_not_null'], 'nodes': ['model.test.model'], }, 'description': '', 'fqn': ['test', 'schema_test', 'not_null_model_id'], 'name': 'not_null_model_id', 'original_file_path': model_schema_yml_path, 'package_name': 'test', 'patch_path': None, 'path': Normalized('schema_test/not_null_model_id.sql'), 'raw_sql': "{{ config(severity='ERROR') }}{{ test_not_null(**_dbt_schema_test_kwargs) }}", 'refs': [['model']], 'resource_type': 'test', 'root_path': self.test_root_dir, 'schema': my_schema_name, 'database': self.default_database, 'tags': ['schema'], 'meta': {}, 'unique_id': 'test.test.not_null_model_id', 'docs': {'show': True}, 'compiled': True, 'compiled_sql': AnyStringWith('count(*)'), 'extra_ctes_injected': True, 'extra_ctes': [], 'injected_sql': AnyStringWith('count(*)'), 'test_metadata': { 'namespace': None, 'name': 'not_null', 'kwargs': { 'column_name': 'id', 'model': "{{ ref('model') }}", }, }, }, 'test.test.test_nothing_model_': { 'alias': 'test_nothing_model_', 'build_path': Normalized('target/compiled/test/models/schema.yml/schema_test/test_nothing_model_.sql'), 'column_name': None, 'columns': {}, 'config': { 'column_types': {}, 'enabled': True, 'materialized': 'view', 'persist_docs': {}, 'post-hook': [], 'pre-hook': [], 'quoting': {}, 'vars': {}, 'tags': [], 'severity': 'ERROR', 'full_refresh': None, }, 'sources': [], 'depends_on': { 'macros': ['macro.test.test_nothing'], 'nodes': ['model.test.model'], }, 'description': '', 'fqn': ['test', 'schema_test', 'test_nothing_model_'], 'name': 'test_nothing_model_', 'original_file_path': model_schema_yml_path, 'package_name': 'test', 'patch_path': None, 'path': normalize('schema_test/test_nothing_model_.sql'), 'raw_sql': "{{ config(severity='ERROR') }}{{ test.test_nothing(**_dbt_schema_test_kwargs) }}", 'refs': [['model']], 'resource_type': 'test', 'root_path': self.test_root_dir, 'schema': my_schema_name, 'database': self.default_database, 'tags': ['schema'], 'meta': {}, 'unique_id': 'test.test.test_nothing_model_', 'docs': {'show': True}, 'compiled': True, 'compiled_sql': AnyStringWith('select 0'), 'extra_ctes_injected': True, 'extra_ctes': [], 'injected_sql': AnyStringWith('select 0'), 'test_metadata': { 'namespace': 'test', 'name': 'nothing', 'kwargs': { 'model': "{{ ref('model') }}", }, }, }, 'test.test.unique_model_id': { 'alias': 'unique_model_id', 'build_path': Normalized('target/compiled/test/models/schema.yml/schema_test/unique_model_id.sql'), 'column_name': 'id', 'columns': {}, 'config': { 'column_types': {}, 'enabled': True, 'materialized': 'view', 'persist_docs': {}, 'post-hook': [], 'pre-hook': [], 'quoting': {}, 'vars': {}, 'tags': [], 'severity': 'ERROR', 'full_refresh': None, }, 'sources': [], 'depends_on': { 'macros': ['macro.dbt.test_unique'], 'nodes': ['model.test.model'], }, 'description': '', 'fqn': ['test', 'schema_test', 'unique_model_id'], 'name': 'unique_model_id', 'original_file_path': model_schema_yml_path, 'package_name': 'test', 'patch_path': None, 'path': normalize('schema_test/unique_model_id.sql'), 'raw_sql': "{{ config(severity='ERROR') }}{{ test_unique(**_dbt_schema_test_kwargs) }}", 'refs': [['model']], 'resource_type': 'test', 'root_path': self.test_root_dir, 'schema': my_schema_name, 'database': self.default_database, 'tags': ['schema'], 'meta': {}, 'unique_id': 'test.test.unique_model_id', 'docs': {'show': True}, 'compiled': True, 'compiled_sql': AnyStringWith('count(*)'), 'extra_ctes_injected': True, 'extra_ctes': [], 'injected_sql': AnyStringWith('count(*)'), 'test_metadata': { 'namespace': None, 'name': 'unique', 'kwargs': { 'column_name': 'id', 'model': "{{ ref('model') }}", }, }, }, }, 'sources': {}, 'parent_map': { 'model.test.model': ['seed.test.seed'], 'seed.test.seed': [], 'test.test.not_null_model_id': ['model.test.model'], 'test.test.test_nothing_model_': ['model.test.model'], 'test.test.unique_model_id': ['model.test.model'], }, 'child_map': { 'model.test.model': [ 'test.test.not_null_model_id', 'test.test.test_nothing_model_', 'test.test.unique_model_id', ], 'seed.test.seed': ['model.test.model'], 'test.test.not_null_model_id': [], 'test.test.test_nothing_model_': [], 'test.test.unique_model_id': [], }, 'docs': { 'dbt.__overview__': ANY, 'test.macro_info': ANY, 'test.macro_arg_info': ANY, }, 'metadata': { 'project_id': '098f6bcd4621d373cade4e832627b4f6', 'send_anonymous_usage_stats': False, 'user_id': None, 'adapter_type': self.adapter_type, }, 'disabled': [], } def expected_postgres_references_manifest(self, model_database=None): if model_database is None: model_database = self.default_database my_schema_name = self.unique_schema() docs_path = self.dir('ref_models/docs.md') ephemeral_summary = LineIndifferent( '{% docs ephemeral_summary %}\nA summmary table of the ephemeral copy of the seed data\n{% enddocs %}' ) source_info = LineIndifferent('{% docs source_info %}\nMy source\n{% enddocs %}') summary_count = LineIndifferent( '{% docs summary_count %}\nThe number of instances of the first name\n{% enddocs %}' ) summary_first_name = LineIndifferent( '{% docs summary_first_name %}\nThe first name being summarized\n{% enddocs %}' ) table_info = LineIndifferent('{% docs table_info %}\nMy table\n{% enddocs %}') view_summary = LineIndifferent( '{% docs view_summary %}\nA view of the summary of the ephemeral copy of the seed data\n{% enddocs %}' ) column_info = LineIndifferent( '{% docs column_info %}\nAn ID field\n{% enddocs %}' ) macro_info = LineIndifferent( '{% docs macro_info %}\nMy custom test that I wrote that does nothing\n{% enddocs %}' ) macro_arg_info = LineIndifferent( '{% docs macro_arg_info %}\nThe model for my custom test\n{% enddocs %}' ) return { 'nodes': { 'model.test.ephemeral_copy': { 'alias': 'ephemeral_copy', 'build_path': Normalized('target/compiled/test/ref_models/ephemeral_copy.sql'), 'columns': {}, 'config': { 'column_types': {}, 'enabled': True, 'materialized': 'ephemeral', 'persist_docs': {}, 'post-hook': [], 'pre-hook': [], 'quoting': {}, 'vars': {}, 'tags': [], 'full_refresh': None, }, 'sources': [['my_source', 'my_table']], 'depends_on': { 'macros': [], 'nodes': ['source.test.my_source.my_table'] }, 'description': '', 'docs': {'show': True}, 'fqn': ['test', 'ephemeral_copy'], 'name': 'ephemeral_copy', 'original_file_path': self.dir('ref_models/ephemeral_copy.sql'), 'package_name': 'test', 'patch_path': None, 'path': 'ephemeral_copy.sql', 'raw_sql': LineIndifferent( '{{\n config(\n materialized = "ephemeral"\n )\n}}' '\n\nselect * from {{ source("my_source", "my_table") }}' ), 'refs': [], 'resource_type': 'model', 'root_path': self.test_root_dir, 'schema': my_schema_name, 'database': self.default_database, 'tags': [], 'meta': {}, 'unique_id': 'model.test.ephemeral_copy', 'compiled': True, 'compiled_sql': ANY, 'extra_ctes_injected': True, 'extra_ctes': [], 'injected_sql': ANY, }, 'model.test.ephemeral_summary': { 'alias': 'ephemeral_summary', 'build_path': Normalized('target/compiled/test/ref_models/ephemeral_summary.sql'), 'columns': { 'first_name': { 'description': 'The first name being summarized', 'name': 'first_name', 'data_type': None, 'meta': {}, 'tags': [], }, 'ct': { 'description': 'The number of instances of the first name', 'name': 'ct', 'data_type': None, 'meta': {}, 'tags': [], }, }, 'config': { 'column_types': {}, 'enabled': True, 'materialized': 'table', 'persist_docs': {}, 'post-hook': [], 'pre-hook': [], 'quoting': {}, 'vars': {}, 'tags': [], 'full_refresh': None, }, 'sources': [], 'depends_on': { 'macros': [], 'nodes': ['model.test.ephemeral_copy'] }, 'description': 'A summmary table of the ephemeral copy of the seed data', 'docs': {'show': True}, 'fqn': ['test', 'ephemeral_summary'], 'name': 'ephemeral_summary', 'original_file_path': self.dir('ref_models/ephemeral_summary.sql'), 'package_name': 'test', 'patch_path': self.dir('ref_models/schema.yml'), 'path': 'ephemeral_summary.sql', 'raw_sql': LineIndifferent( '{{\n config(\n materialized = "table"\n )\n}}\n\n' 'select first_name, count(*) as ct from ' "{{ref('ephemeral_copy')}}\ngroup by first_name\n" 'order by first_name asc' ), 'refs': [['ephemeral_copy']], 'resource_type': 'model', 'root_path': self.test_root_dir, 'schema': my_schema_name, 'database': self.default_database, 'tags': [], 'meta': {}, 'unique_id': 'model.test.ephemeral_summary', 'compiled': True, 'compiled_sql': ANY, 'extra_ctes_injected': True, 'extra_ctes': [ANY], 'injected_sql': ANY, }, 'model.test.view_summary': { 'alias': 'view_summary', 'build_path': Normalized('target/compiled/test/ref_models/view_summary.sql'), 'columns': { 'first_name': { 'description': 'The first name being summarized', 'name': 'first_name', 'data_type': None, 'meta': {}, 'tags': [], }, 'ct': { 'description': 'The number of instances of the first name', 'name': 'ct', 'data_type': None, 'meta': {}, 'tags': [], }, }, 'config': { 'column_types': {}, 'enabled': True, 'materialized': 'view', 'persist_docs': {}, 'post-hook': [], 'pre-hook': [], 'quoting': {}, 'vars': {}, 'tags': [], 'full_refresh': None, }, 'database': self.default_database, 'depends_on': { 'macros': [], 'nodes': ['model.test.ephemeral_summary'] }, 'description': 'A view of the summary of the ephemeral copy of the seed data', 'docs': {'show': True}, 'fqn': ['test', 'view_summary'], 'name': 'view_summary', 'original_file_path': self.dir('ref_models/view_summary.sql'), 'package_name': 'test', 'patch_path': self.dir('ref_models/schema.yml'), 'path': 'view_summary.sql', 'raw_sql': LineIndifferent( '{{\n config(\n materialized = "view"\n )\n}}\n\n' 'select first_name, ct from ' "{{ref('ephemeral_summary')}}\norder by ct asc" ), 'refs': [['ephemeral_summary']], 'resource_type': 'model', 'root_path': self.test_root_dir, 'schema': my_schema_name, 'sources': [], 'tags': [], 'meta': {}, 'unique_id': 'model.test.view_summary', 'compiled': True, 'compiled_sql': ANY, 'extra_ctes_injected': True, 'extra_ctes': [], 'injected_sql': ANY, }, 'seed.test.seed': { 'alias': 'seed', 'build_path': None, 'columns': { 'id': { 'name': 'id', 'description': 'The user ID number', 'data_type': None, 'meta': {}, 'tags': [], }, 'first_name': { 'name': 'first_name', 'description': "The user's first name", 'data_type': None, 'meta': {}, 'tags': [], }, 'email': { 'name': 'email', 'description': "The user's email", 'data_type': None, 'meta': {}, 'tags': [], }, 'ip_address': { 'name': 'ip_address', 'description': "The user's IP address", 'data_type': None, 'meta': {}, 'tags': [], }, 'updated_at': { 'name': 'updated_at', 'description': "The last time this user's email was updated", 'data_type': None, 'meta': {}, 'tags': [], }, }, 'config': { 'column_types': {}, 'enabled': True, 'materialized': 'seed', 'persist_docs': {}, 'post-hook': [], 'pre-hook': [], 'quoting': {}, 'vars': {}, 'tags': [], 'quote_columns': True, 'full_refresh': None, }, 'sources': [], 'depends_on': {'macros': [], 'nodes': []}, 'description': 'The test seed', 'docs': {'show': True}, 'fqn': ['test', 'seed'], 'name': 'seed', 'original_file_path': self.dir('seed/seed.csv'), 'package_name': 'test', 'patch_path': self.dir('seed/schema.yml'), 'path': 'seed.csv', 'raw_sql': '', 'refs': [], 'resource_type': 'seed', 'root_path': self.test_root_dir, 'schema': my_schema_name, 'database': self.default_database, 'tags': [], 'meta': {}, 'unique_id': 'seed.test.seed', 'compiled': True, 'compiled_sql': '', 'extra_ctes_injected': True, 'extra_ctes': [], 'injected_sql': '', }, }, 'sources': { 'source.test.my_source.my_table': { 'columns': { 'id': { 'description': 'An ID field', 'name': 'id', 'data_type': None, 'meta': {}, 'tags': [], } }, 'config': { 'enabled': True, }, 'quoting': { 'database': False, 'schema': None, 'identifier': True, 'column': None, }, 'database': self.default_database, 'description': 'My table', 'external': None, 'freshness': {'error_after': None, 'warn_after': None, 'filter': None}, 'identifier': 'seed', 'loaded_at_field': None, 'loader': 'a_loader', 'meta': {}, 'name': 'my_table', 'original_file_path': self.dir('ref_models/schema.yml'), 'package_name': 'test', 'path': self.dir('ref_models/schema.yml'), 'patch_path': None, 'resource_type': 'source', 'root_path': self.test_root_dir, 'schema': my_schema_name, 'source_description': 'My source', 'source_name': 'my_source', 'source_meta': {}, 'tags': [], 'unique_id': 'source.test.my_source.my_table', 'fqn': ['test', 'my_source', 'my_table'], }, }, 'docs': { 'dbt.__overview__': ANY, 'test.column_info': { 'block_contents': 'An ID field', 'name': 'column_info', 'original_file_path': docs_path, 'package_name': 'test', 'path': 'docs.md', 'root_path': self.test_root_dir, 'unique_id': 'test.column_info', }, 'test.ephemeral_summary': { 'block_contents': ( 'A summmary table of the ephemeral copy of the seed data' ), 'name': 'ephemeral_summary', 'original_file_path': docs_path, 'package_name': 'test', 'path': 'docs.md', 'root_path': self.test_root_dir, 'unique_id': 'test.ephemeral_summary', }, 'test.source_info': { 'block_contents': 'My source', 'name': 'source_info', 'original_file_path': docs_path, 'package_name': 'test', 'path': 'docs.md', 'root_path': self.test_root_dir, 'unique_id': 'test.source_info', }, 'test.summary_count': { 'block_contents': 'The number of instances of the first name', 'name': 'summary_count', 'original_file_path': docs_path, 'package_name': 'test', 'path': 'docs.md', 'root_path': self.test_root_dir, 'unique_id': 'test.summary_count', }, 'test.summary_first_name': { 'block_contents': 'The first name being summarized', 'name': 'summary_first_name', 'original_file_path': docs_path, 'package_name': 'test', 'path': 'docs.md', 'root_path': self.test_root_dir, 'unique_id': 'test.summary_first_name', }, 'test.table_info': { 'block_contents': 'My table', 'name': 'table_info', 'original_file_path': docs_path, 'package_name': 'test', 'path': 'docs.md', 'root_path': self.test_root_dir, 'unique_id': 'test.table_info', }, 'test.view_summary': { 'block_contents': ( 'A view of the summary of the ephemeral copy of the ' 'seed data' ), 'name': 'view_summary', 'original_file_path': docs_path, 'package_name': 'test', 'path': 'docs.md', 'root_path': self.test_root_dir, 'unique_id': 'test.view_summary', }, 'test.macro_info': { 'block_contents': 'My custom test that I wrote that does nothing', 'name': 'macro_info', 'original_file_path': self.dir('macros/macro.md'), 'package_name': 'test', 'path': 'macro.md', 'root_path': self.test_root_dir, 'unique_id': 'test.macro_info', }, 'test.macro_arg_info': { 'block_contents': 'The model for my custom test', 'name': 'macro_arg_info', 'original_file_path': self.dir('macros/macro.md'), 'package_name': 'test', 'path': 'macro.md', 'root_path': self.test_root_dir, 'unique_id': 'test.macro_arg_info', }, }, 'child_map': { 'model.test.ephemeral_copy': ['model.test.ephemeral_summary'], 'model.test.ephemeral_summary': ['model.test.view_summary'], 'model.test.view_summary': [], 'seed.test.seed': [], 'source.test.my_source.my_table': ['model.test.ephemeral_copy'], }, 'parent_map': { 'model.test.ephemeral_copy': ['source.test.my_source.my_table'], 'model.test.ephemeral_summary': ['model.test.ephemeral_copy'], 'model.test.view_summary': ['model.test.ephemeral_summary'], 'seed.test.seed': [], 'source.test.my_source.my_table': [], }, 'metadata': { 'project_id': '098f6bcd4621d373cade4e832627b4f6', 'send_anonymous_usage_stats': False, 'user_id': None, 'adapter_type': self.adapter_type, }, 'disabled': [], 'macros': { 'macro.test.test_nothing': { 'name': 'test_nothing', 'depends_on': {'macros': []}, 'description': 'My custom test that I wrote that does nothing', 'docs': {'show': True}, 'macro_sql': AnyStringWith('macro test_nothing'), 'original_file_path': self.dir('macros/dummy_test.sql'), 'path': self.dir('macros/dummy_test.sql'), 'package_name': 'test', 'meta': { 'some_key': 100, }, 'patch_path': self.dir('macros/schema.yml'), 'resource_type': 'macro', 'unique_id': 'macro.test.test_nothing', 'tags': [], 'root_path': self.test_root_dir, 'arguments': [ { 'name': 'model', 'type': 'Relation', 'description': 'The model for my custom test', }, ], } } } def expected_bigquery_complex_manifest(self): nested_view_sql_path = self.dir('bq_models/nested_view.sql') nested_table_sql_path = self.dir('bq_models/nested_table.sql') clustered_sql_path = self.dir('bq_models/clustered.sql') multi_clustered_sql_path = self.dir('bq_models/multi_clustered.sql') my_schema_name = self.unique_schema() return { 'nodes': { 'model.test.clustered': { 'alias': 'clustered', 'config': { 'cluster_by': ['first_name'], 'column_types': {}, 'enabled': True, 'materialized': 'table', 'partition_by': {'field': 'updated_at', 'data_type': 'date'}, 'persist_docs': {}, 'post-hook': [], 'pre-hook': [], 'quoting': {}, 'vars': {}, 'tags': [], 'full_refresh': None, }, 'sources': [], 'depends_on': {'macros': [], 'nodes': ['seed.test.seed']}, 'fqn': ['test', 'clustered'], 'build_path': Normalized('target/compiled/test/bq_models/clustered.sql'), 'name': 'clustered', 'original_file_path': clustered_sql_path, 'package_name': 'test', 'path': 'clustered.sql', 'raw_sql': LineIndifferent(_read_file(clustered_sql_path).rstrip('\r\n')), 'refs': [['seed']], 'resource_type': 'model', 'root_path': self.test_root_dir, 'schema': my_schema_name, 'database': self.default_database, 'tags': [], 'meta': {}, 'unique_id': 'model.test.clustered', 'columns': { 'email': { 'description': "The user's email", 'name': 'email', 'data_type': None, 'meta': {}, 'tags': [], }, 'first_name': { 'description': "The user's name", 'name': 'first_name', 'data_type': None, 'meta': {}, 'tags': [], }, 'id': { 'description': 'The user id', 'name': 'id', 'data_type': None, 'meta': {}, 'tags': [], }, 'ip_address': { 'description': "The user's IP address", 'name': 'ip_address', 'data_type': None, 'meta': {}, 'tags': [], }, 'updated_at': { 'description': 'When the user was updated', 'name': 'updated_at', 'data_type': None, 'meta': {}, 'tags': [], }, }, 'description': 'A clustered and partitioned copy of the test model', 'patch_path': self.dir('bq_models/schema.yml'), 'docs': {'show': True}, 'compiled': True, 'compiled_sql': ANY, 'extra_ctes_injected': True, 'extra_ctes': [], 'injected_sql': ANY, }, 'model.test.multi_clustered': { 'alias': 'multi_clustered', 'build_path': Normalized('target/compiled/test/bq_models/multi_clustered.sql'), 'config': { 'cluster_by': ['first_name', 'email'], 'column_types': {}, 'enabled': True, 'materialized': 'table', 'partition_by': {'field': 'updated_at', 'data_type': 'date'}, 'persist_docs': {}, 'post-hook': [], 'pre-hook': [], 'quoting': {}, 'tags': [], 'vars': {}, 'full_refresh': None, }, 'sources': [], 'depends_on': {'macros': [], 'nodes': ['seed.test.seed']}, 'fqn': ['test', 'multi_clustered'], 'name': 'multi_clustered', 'original_file_path': multi_clustered_sql_path, 'package_name': 'test', 'path': 'multi_clustered.sql', 'raw_sql': LineIndifferent(_read_file(multi_clustered_sql_path).rstrip('\r\n')), 'refs': [['seed']], 'resource_type': 'model', 'root_path': self.test_root_dir, 'schema': my_schema_name, 'database': self.default_database, 'tags': [], 'meta': {}, 'unique_id': 'model.test.multi_clustered', 'columns': { 'email': { 'description': "The user's email", 'name': 'email', 'data_type': None, 'meta': {}, 'tags': [], }, 'first_name': { 'description': "The user's name", 'name': 'first_name', 'data_type': None, 'meta': {}, 'tags': [], }, 'id': { 'description': 'The user id', 'name': 'id', 'data_type': None, 'meta': {}, 'tags': [], }, 'ip_address': { 'description': "The user's IP address", 'name': 'ip_address', 'data_type': None, 'meta': {}, 'tags': [], }, 'updated_at': { 'description': 'When the user was updated', 'name': 'updated_at', 'data_type': None, 'meta': {}, 'tags': [], }, }, 'description': 'A clustered and partitioned copy of the test model, clustered on multiple columns', 'patch_path': self.dir('bq_models/schema.yml'), 'docs': {'show': True}, 'compiled': True, 'compiled_sql': ANY, 'extra_ctes_injected': True, 'extra_ctes': [], 'injected_sql': ANY, }, 'model.test.nested_view': { 'alias': 'nested_view', 'build_path': Normalized('target/compiled/test/bq_models/nested_view.sql'), 'config': { 'column_types': {}, 'enabled': True, 'materialized': 'view', 'persist_docs': {}, 'post-hook': [], 'pre-hook': [], 'quoting': {}, 'vars': {}, 'tags': [], 'full_refresh': None, }, 'sources': [], 'depends_on': { 'macros': [], 'nodes': ['model.test.nested_table'] }, 'fqn': ['test', 'nested_view'], 'name': 'nested_view', 'original_file_path': nested_view_sql_path, 'package_name': 'test', 'path': 'nested_view.sql', 'raw_sql': LineIndifferent(_read_file(nested_view_sql_path).rstrip('\r\n')), 'refs': [['nested_table']], 'resource_type': 'model', 'root_path': self.test_root_dir, 'schema': my_schema_name, 'database': self.default_database, 'tags': [], 'meta': {}, 'unique_id': 'model.test.nested_view', 'columns': { 'field_1': { 'name': 'field_1', 'description': 'The first field', 'data_type': None, 'meta': {}, 'tags': [], }, 'field_2': { 'name': 'field_2', 'description': 'The second field', 'data_type': None, 'meta': {}, 'tags': [], }, 'field_3': { 'name': 'field_3', 'description': 'The third field', 'data_type': None, 'meta': {}, 'tags': [], }, 'nested_field.field_4': { 'name': 'nested_field.field_4', 'description': 'The first nested field', 'data_type': None, 'meta': {}, 'tags': [], }, 'nested_field.field_5': { 'name': 'nested_field.field_5', 'description': 'The second nested field', 'data_type': None, 'meta': {}, 'tags': [], }, }, 'description': 'The test model', 'patch_path': self.dir('bq_models/schema.yml'), 'docs': {'show': True}, 'compiled': True, 'compiled_sql': ANY, 'extra_ctes_injected': True, 'extra_ctes': [], 'injected_sql': ANY, }, 'model.test.nested_table': { 'alias': 'nested_table', 'build_path': Normalized('target/compiled/test/bq_models/nested_table.sql'), 'config': { 'column_types': {}, 'enabled': True, 'materialized': 'table', 'persist_docs': {}, 'post-hook': [], 'pre-hook': [], 'quoting': {}, 'vars': {}, 'tags': [], 'full_refresh': None, }, 'sources': [], 'depends_on': { 'macros': [], 'nodes': [] }, 'fqn': ['test', 'nested_table'], 'name': 'nested_table', 'original_file_path': nested_table_sql_path, 'package_name': 'test', 'patch_path': None, 'path': 'nested_table.sql', 'raw_sql': LineIndifferent(_read_file(nested_table_sql_path).rstrip('\r\n')), 'refs': [], 'resource_type': 'model', 'root_path': self.test_root_dir, 'schema': my_schema_name, 'database': self.default_database, 'tags': [], 'meta': {}, 'unique_id': 'model.test.nested_table', 'columns': {}, 'description': '', 'docs': {'show': True}, 'compiled': True, 'compiled_sql': ANY, 'extra_ctes_injected': True, 'extra_ctes': [], 'injected_sql': ANY, }, 'seed.test.seed': { 'build_path': None, 'patch_path': self.dir('seed/schema.yml'), 'path': 'seed.csv', 'name': 'seed', 'root_path': self.test_root_dir, 'resource_type': 'seed', 'raw_sql': '', 'package_name': 'test', 'original_file_path': self.dir('seed/seed.csv'), 'refs': [], 'sources': [], 'depends_on': { 'nodes': [], 'macros': [], }, 'unique_id': 'seed.test.seed', 'fqn': ['test', 'seed'], 'tags': [], 'meta': {}, 'config': { 'enabled': True, 'materialized': 'seed', 'persist_docs': {}, 'pre-hook': [], 'post-hook': [], 'vars': {}, 'column_types': {}, 'quoting': {}, 'tags': [], 'quote_columns': True, 'full_refresh': None, }, 'schema': my_schema_name, 'database': self.default_database, 'alias': 'seed', 'columns': { 'id': { 'name': 'id', 'description': 'The user ID number', 'data_type': None, 'meta': {}, 'tags': [], }, 'first_name': { 'name': 'first_name', 'description': "The user's first name", 'data_type': None, 'meta': {}, 'tags': [], }, 'email': { 'name': 'email', 'description': "The user's email", 'data_type': None, 'meta': {}, 'tags': [], }, 'ip_address': { 'name': 'ip_address', 'description': "The user's IP address", 'data_type': None, 'meta': {}, 'tags': [], }, 'updated_at': { 'name': 'updated_at', 'description': "The last time this user's email was updated", 'data_type': None, 'meta': {}, 'tags': [], }, }, 'description': 'The test seed', 'docs': {'show': True}, 'compiled': True, 'compiled_sql': '', 'extra_ctes_injected': True, 'extra_ctes': [], 'injected_sql': '', }, }, 'sources': {}, 'child_map': { 'model.test.clustered': [], 'model.test.multi_clustered': [], 'model.test.nested_table': ['model.test.nested_view'], 'model.test.nested_view': [], 'seed.test.seed': ['model.test.clustered', 'model.test.multi_clustered'] }, 'parent_map': { 'model.test.clustered': ['seed.test.seed'], 'model.test.multi_clustered': ['seed.test.seed'], 'seed.test.seed': [], 'model.test.nested_table': [], 'model.test.nested_view': ['model.test.nested_table'], }, 'docs': { 'dbt.__overview__': ANY, 'test.macro_info': ANY, 'test.macro_arg_info': ANY, }, 'metadata': { 'project_id': '098f6bcd4621d373cade4e832627b4f6', 'send_anonymous_usage_stats': False, 'user_id': None, 'adapter_type': self.adapter_type, }, 'disabled': [], } def _checksum_file(self, path): """windows has silly git behavior that adds newlines, and python does silly things if we just open(..., 'r').encode('utf-8'). """ with open(self.dir(path), 'rb') as fp: hashed = hashlib.sha256(fp.read()).hexdigest() return { 'name': 'sha256', 'checksum': hashed, } def _path_to(self, searched_path: str, relative_path: str): return { 'searched_path': normalize(searched_path), 'relative_path': normalize(relative_path), 'project_root': normalize(self.test_root_dir), } def _absolute_path_to(self, searched_path: str, relative_path: str): return os.path.join( normalize(self.test_root_dir), normalize(searched_path), normalize(relative_path) ) def expected_redshift_incremental_view_manifest(self): model_sql_path = self.dir('rs_models/model.sql') my_schema_name = self.unique_schema() return { 'nodes': { 'model.test.model': { 'build_path': Normalized('target/compiled/test/rs_models/model.sql'), 'name': 'model', 'root_path': self.test_root_dir, 'resource_type': 'model', 'path': 'model.sql', 'original_file_path': model_sql_path, 'package_name': 'test', 'raw_sql': LineIndifferent(_read_file(model_sql_path).rstrip('\r\n')), 'refs': [['seed']], 'sources': [], 'depends_on': { 'nodes': ['seed.test.seed'], 'macros': [], }, 'unique_id': 'model.test.model', 'fqn': ['test', 'model'], 'tags': [], 'meta': {}, 'config': { 'bind': False, 'column_types': {}, 'enabled': True, 'materialized': 'view', 'persist_docs': {}, 'post-hook': [], 'pre-hook': [], 'quoting': {}, 'tags': [], 'vars': {}, 'full_refresh': None, }, 'schema': my_schema_name, 'database': self.default_database, 'alias': 'model', 'description': 'The test model', 'columns': { 'id': { 'name': 'id', 'description': 'The user ID number', 'data_type': None, 'meta': {}, 'tags': [], }, 'first_name': { 'name': 'first_name', 'description': "The user's first name", 'data_type': None, 'meta': {}, 'tags': [], }, 'email': { 'name': 'email', 'description': "The user's email", 'data_type': None, 'meta': {}, 'tags': [], }, 'ip_address': { 'name': 'ip_address', 'description': "The user's IP address", 'data_type': None, 'meta': {}, 'tags': [], }, 'updated_at': { 'name': 'updated_at', 'description': "The last time this user's email was updated", 'data_type': None, 'meta': {}, 'tags': [], }, }, 'patch_path': self.dir('rs_models/schema.yml'), 'docs': {'show': True}, 'compiled': True, 'compiled_sql': ANY, 'extra_ctes_injected': True, 'extra_ctes': [], 'injected_sql': ANY, }, 'seed.test.seed': { 'build_path': None, 'patch_path': self.dir('seed/schema.yml'), 'path': 'seed.csv', 'name': 'seed', 'root_path': self.test_root_dir, 'resource_type': 'seed', 'raw_sql': '', 'package_name': 'test', 'original_file_path': self.dir('seed/seed.csv'), 'refs': [], 'sources': [], 'depends_on': { 'nodes': [], 'macros': [], }, 'unique_id': 'seed.test.seed', 'fqn': ['test', 'seed'], 'tags': [], 'meta': {}, 'config': { 'column_types': {}, 'enabled': True, 'materialized': 'seed', 'persist_docs': {}, 'post-hook': [], 'pre-hook': [], 'quoting': {}, 'tags': [], 'vars': {}, 'quote_columns': True, 'full_refresh': None, }, 'schema': my_schema_name, 'database': self.default_database, 'alias': 'seed', 'columns': { 'id': { 'name': 'id', 'description': 'The user ID number', 'data_type': None, 'meta': {}, 'tags': [], }, 'first_name': { 'name': 'first_name', 'description': "The user's first name", 'data_type': None, 'meta': {}, 'tags': [], }, 'email': { 'name': 'email', 'description': "The user's email", 'data_type': None, 'meta': {}, 'tags': [], }, 'ip_address': { 'name': 'ip_address', 'description': "The user's IP address", 'data_type': None, 'meta': {}, 'tags': [], }, 'updated_at': { 'name': 'updated_at', 'description': "The last time this user's email was updated", 'data_type': None, 'meta': {}, 'tags': [], }, }, 'description': 'The test seed', 'docs': {'show': True}, 'compiled': True, 'compiled_sql': ANY, 'extra_ctes_injected': True, 'extra_ctes': [], 'injected_sql': ANY, }, }, 'sources': {}, 'parent_map': { 'model.test.model': ['seed.test.seed'], 'seed.test.seed': [] }, 'child_map': { 'model.test.model': [], 'seed.test.seed': ['model.test.model'] }, 'docs': { 'dbt.__overview__': ANY, 'test.macro_info': ANY, 'test.macro_arg_info': ANY, }, 'metadata': { 'project_id': '098f6bcd4621d373cade4e832627b4f6', 'send_anonymous_usage_stats': False, 'user_id': None, 'adapter_type': self.adapter_type, }, 'disabled': [], } def verify_manifest(self, expected_manifest): self.assertTrue(os.path.exists('./target/manifest.json')) manifest = _read_json('./target/manifest.json') manifest_keys = frozenset({ 'nodes', 'sources', 'macros', 'parent_map', 'child_map', 'generated_at', 'docs', 'metadata', 'docs', 'disabled' }) self.assertEqual(frozenset(manifest), manifest_keys) for key in manifest_keys: if key == 'macros': self.verify_manifest_macros(manifest, expected_manifest.get('macros')) elif key == 'generated_at': self.assertBetween(manifest['generated_at'], start=self.generate_start_time) else: self.assertIn(key, expected_manifest) # sanity check self.assertEqual(manifest[key], expected_manifest[key]) def _quote(self, value): quote_char = '`' if self.adapter_type == 'bigquery' else '"' return '{0}{1}{0}'.format(quote_char, value) def expected_run_results(self, quote_schema=True, quote_model=False, model_database=None): """ The expected results of this run. """ models_path = self.dir('models') model_sql_path = os.path.join(models_path, 'model.sql') model_schema_yml_path = os.path.join(models_path, 'schema.yml') seed_schema_yml_path = os.path.join(self.dir('seed'), 'schema.yml') if model_database is None: model_database = self.alternative_database model_config = { 'database': model_database, 'enabled': True, 'materialized': 'view', 'persist_docs': {}, 'pre-hook': [], 'post-hook': [], 'vars': {}, 'column_types': {}, 'quoting': {}, 'tags': [], 'full_refresh': None, } schema = self.unique_schema() # we are selecting from the seed, which is always in the default db compiled_database = self.default_database if self.adapter_type != 'snowflake': compiled_database = self._quote(compiled_database) compiled_schema = self._quote(schema) if quote_schema else schema compiled_seed = self._quote('seed') if quote_model else 'seed' if self.adapter_type == 'bigquery': compiled_sql = '\n\nselect * from `{}`.`{}`.seed'.format( self.default_database, schema ) else: compiled_sql = '\n\nselect * from {}.{}.{}'.format( compiled_database, compiled_schema, compiled_seed ) return [ { 'error': None, 'execution_time': AnyFloat(), 'fail': None, 'warn': None, 'node': { 'alias': 'model', 'build_path': Normalized( 'target/compiled/test/models/model.sql' ), 'columns': { 'id': { 'description': 'The user ID number', 'name': 'id', 'data_type': None, 'meta': {}, 'tags': [], }, 'first_name': { 'description': "The user's first name", 'name': 'first_name', 'data_type': None, 'meta': {}, 'tags': [], }, 'email': { 'description': "The user's email", 'name': 'email', 'data_type': None, 'meta': {}, 'tags': [], }, 'ip_address': { 'description': "The user's IP address", 'name': 'ip_address', 'data_type': None, 'meta': {}, 'tags': [], }, 'updated_at': { 'description': "The last time this user's email was updated", 'name': 'updated_at', 'data_type': None, 'meta': {}, 'tags': [], } }, 'compiled': True, 'compiled_sql': compiled_sql, 'config': model_config, 'sources': [], 'depends_on': { 'macros': [], 'nodes': ['seed.test.seed'] }, 'description': 'The test model', 'docs': {'show': False}, 'extra_ctes': [], 'extra_ctes_injected': True, 'fqn': ['test', 'model'], 'injected_sql': compiled_sql, 'meta': {}, 'name': 'model', 'original_file_path': model_sql_path, 'package_name': 'test', 'patch_path': model_schema_yml_path, 'path': 'model.sql', 'raw_sql': LineIndifferent(_read_file(model_sql_path).rstrip('\r\n')), 'refs': [['seed']], 'resource_type': 'model', 'root_path': self.test_root_dir, 'schema': schema, 'database': model_database, 'tags': [], 'unique_id': 'model.test.model', }, 'thread_id': ANY, 'timing': [ANY, ANY], 'skip': False, 'status': None, }, { 'error': None, 'execution_time': AnyFloat(), 'fail': None, 'warn': None, 'node': { 'alias': 'seed', 'build_path': None, 'columns': { 'id': { 'description': 'The user ID number', 'name': 'id', 'data_type': None, 'meta': {}, 'tags': [], }, 'first_name': { 'description': "The user's first name", 'name': 'first_name', 'data_type': None, 'meta': {}, 'tags': [], }, 'email': { 'description': "The user's email", 'name': 'email', 'data_type': None, 'meta': {}, 'tags': [], }, 'ip_address': { 'description': "The user's IP address", 'name': 'ip_address', 'data_type': None, 'meta': {}, 'tags': [], }, 'updated_at': { 'description': "The last time this user's email was updated", 'name': 'updated_at', 'data_type': None, 'meta': {}, 'tags': [], } }, 'compiled': True, 'compiled_sql': '', 'config': { 'column_types': {}, 'enabled': True, 'materialized': 'seed', 'persist_docs': {}, 'post-hook': [], 'pre-hook': [], 'quoting': {}, 'vars': {}, 'tags': [], 'quote_columns': True, 'full_refresh': None, }, 'sources': [], 'depends_on': {'macros': [], 'nodes': []}, 'description': 'The test seed', 'docs': {'show': True}, 'extra_ctes': [], 'extra_ctes_injected': True, 'fqn': ['test', 'seed'], 'injected_sql': '', 'meta': {}, 'name': 'seed', 'original_file_path': self.dir('seed/seed.csv'), 'package_name': 'test', 'patch_path': seed_schema_yml_path, 'path': 'seed.csv', 'raw_sql': '', 'refs': [], 'resource_type': 'seed', 'root_path': self.test_root_dir, 'schema': schema, 'database': self.default_database, 'tags': [], 'unique_id': 'seed.test.seed', }, 'thread_id': ANY, 'timing': [ANY, ANY], 'skip': False, 'status': None, }, { 'error': None, 'execution_time': AnyFloat(), 'fail': None, 'warn': None, 'node': { 'alias': 'not_null_model_id', 'build_path': Normalized('target/compiled/test/models/schema.yml/schema_test/not_null_model_id.sql'), 'column_name': 'id', 'columns': {}, 'compiled': True, 'compiled_sql': AnyStringWith('id is null'), 'config': { 'column_types': {}, 'enabled': True, 'materialized': 'view', 'persist_docs': {}, 'post-hook': [], 'pre-hook': [], 'quoting': {}, 'vars': {}, 'tags': [], 'severity': 'ERROR', 'full_refresh': None, }, 'sources': [], 'depends_on': { 'macros': ['macro.dbt.test_not_null'], 'nodes': ['model.test.model'], }, 'description': '', 'docs': {'show': True}, 'extra_ctes': [], 'extra_ctes_injected': True, 'fqn': ['test', 'schema_test', 'not_null_model_id'], 'injected_sql': AnyStringWith('id is null'), 'meta': {}, 'name': 'not_null_model_id', 'original_file_path': model_schema_yml_path, 'package_name': 'test', 'patch_path': None, 'path': Normalized('schema_test/not_null_model_id.sql'), 'raw_sql': "{{ config(severity='ERROR') }}{{ test_not_null(**_dbt_schema_test_kwargs) }}", 'refs': [['model']], 'resource_type': 'test', 'root_path': self.test_root_dir, 'schema': schema, 'database': self.default_database, 'tags': ['schema'], 'unique_id': 'test.test.not_null_model_id', 'test_metadata': { 'namespace': None, 'name': 'not_null', 'kwargs': { 'column_name': 'id', 'model': "{{ ref('model') }}", }, }, }, 'thread_id': ANY, 'timing': [ANY, ANY], 'skip': False, 'status': None, }, { 'error': None, 'execution_time': AnyFloat(), 'fail': None, 'warn': None, 'node': { 'alias': 'test_nothing_model_', 'build_path': Normalized('target/compiled/test/models/schema.yml/schema_test/test_nothing_model_.sql'), 'column_name': None, 'columns': {}, 'compiled': True, 'compiled_sql': AnyStringWith('select 0'), 'config': { 'column_types': {}, 'enabled': True, 'materialized': 'view', 'persist_docs': {}, 'post-hook': [], 'pre-hook': [], 'quoting': {}, 'vars': {}, 'tags': [], 'severity': 'ERROR', 'full_refresh': None, }, 'database': self.default_database, 'depends_on': { 'macros': ['macro.test.test_nothing'], 'nodes': ['model.test.model'], }, 'description': '', 'docs': {'show': True}, 'extra_ctes': [], 'extra_ctes_injected': True, 'fqn': ['test', 'schema_test', 'test_nothing_model_'], 'injected_sql': AnyStringWith('select 0'), 'meta': {}, 'name': 'test_nothing_model_', 'original_file_path': model_schema_yml_path, 'package_name': 'test', 'patch_path': None, 'path': Normalized('schema_test/test_nothing_model_.sql'), 'raw_sql': "{{ config(severity='ERROR') }}{{ test.test_nothing(**_dbt_schema_test_kwargs) }}", 'refs': [['model']], 'resource_type': 'test', 'root_path': self.test_root_dir, 'schema': schema, 'sources': [], 'tags': ['schema'], 'unique_id': 'test.test.test_nothing_model_', 'test_metadata': { 'namespace': 'test', 'name': 'nothing', 'kwargs': { 'model': "{{ ref('model') }}", }, }, }, 'thread_id': ANY, 'timing': [ANY, ANY], 'skip': False, 'status': None }, { 'error': None, 'execution_time': AnyFloat(), 'fail': None, 'warn': None, 'node': { 'alias': 'unique_model_id', 'build_path': Normalized('target/compiled/test/models/schema.yml/schema_test/unique_model_id.sql'), 'column_name': 'id', 'columns': {}, 'compiled': True, 'compiled_sql': AnyStringWith('count(*)'), 'config': { 'column_types': {}, 'enabled': True, 'materialized': 'view', 'persist_docs': {}, 'post-hook': [], 'pre-hook': [], 'quoting': {}, 'vars': {}, 'tags': [], 'severity': 'ERROR', 'full_refresh': None, }, 'database': self.default_database, 'depends_on': { 'macros': ['macro.dbt.test_unique'], 'nodes': ['model.test.model'], }, 'description': '', 'docs': {'show': True}, 'extra_ctes': [], 'extra_ctes_injected': True, 'fqn': ['test', 'schema_test', 'unique_model_id'], 'injected_sql': AnyStringWith('count(*)'), 'meta': {}, 'name': 'unique_model_id', 'original_file_path': model_schema_yml_path, 'package_name': 'test', 'patch_path': None, 'path': Normalized('schema_test/unique_model_id.sql'), 'raw_sql': "{{ config(severity='ERROR') }}{{ test_unique(**_dbt_schema_test_kwargs) }}", 'refs': [['model']], 'resource_type': 'test', 'root_path': self.test_root_dir, 'schema': schema, 'sources': [], 'tags': ['schema'], 'unique_id': 'test.test.unique_model_id', 'test_metadata': { 'namespace': None, 'name': 'unique', 'kwargs': { 'column_name': 'id', 'model': "{{ ref('model') }}", }, }, }, 'thread_id': ANY, 'timing': [ANY, ANY], 'skip': False, 'status': None, }, ] def expected_postgres_references_run_results(self): my_schema_name = self.unique_schema() ephemeral_compiled_sql = ( '\n\nselect first_name, count(*) as ct from ' '__dbt__CTE__ephemeral_copy\ngroup by first_name\n' 'order by first_name asc' ) cte_sql = ( ' __dbt__CTE__ephemeral_copy as (\n\n\nselect * from {}."{}"."seed"\n)' ).format(self.default_database, my_schema_name) ephemeral_injected_sql = ( '\n\nwith{}select first_name, count(*) as ct from ' '__dbt__CTE__ephemeral_copy\ngroup by first_name\n' 'order by first_name asc' ).format(cte_sql) view_compiled_sql = ( '\n\nselect first_name, ct from "{}"."{}".ephemeral_summary\n' 'order by ct asc' ).format(self.default_database, my_schema_name) return [ { 'error': None, 'execution_time': AnyFloat(), 'fail': None, 'warn': None, 'node': { 'alias': 'ephemeral_summary', 'build_path': Normalized( 'target/compiled/test/ref_models/ephemeral_summary.sql' ), 'columns': { 'first_name': { 'description': 'The first name being summarized', 'name': 'first_name', 'data_type': None, 'meta': {}, 'tags': [], }, 'ct': { 'description': 'The number of instances of the first name', 'name': 'ct', 'data_type': None, 'meta': {}, 'tags': [], }, }, 'compiled': True, 'compiled_sql': ephemeral_compiled_sql, 'config': { 'enabled': True, 'materialized': 'table', 'persist_docs': {}, 'pre-hook': [], 'post-hook': [], 'vars': {}, 'column_types': {}, 'quoting': {}, 'tags': [], 'full_refresh': None, }, 'sources': [], 'depends_on': { 'nodes': ['model.test.ephemeral_copy'], 'macros': [] }, 'description': ( 'A summmary table of the ephemeral copy of the seed data' ), 'docs': {'show': True}, 'extra_ctes': [ {'id': 'model.test.ephemeral_copy', 'sql': cte_sql}, ], 'extra_ctes_injected': True, 'fqn': ['test', 'ephemeral_summary'], 'injected_sql': ephemeral_injected_sql, 'meta': {}, 'name': 'ephemeral_summary', 'original_file_path': self.dir('ref_models/ephemeral_summary.sql'), 'package_name': 'test', 'patch_path': self.dir('ref_models/schema.yml'), 'path': 'ephemeral_summary.sql', 'raw_sql': LineIndifferent( '{{\n config(\n materialized = "table"\n )\n}}\n' '\nselect first_name, count(*) as ct from ' "{{ref('ephemeral_copy')}}\ngroup by first_name\n" 'order by first_name asc' ), 'refs': [['ephemeral_copy']], 'resource_type': 'model', 'root_path': self.test_root_dir, 'schema': my_schema_name, 'database': self.default_database, 'tags': [], 'unique_id': 'model.test.ephemeral_summary', }, 'thread_id': ANY, 'timing': [ANY, ANY], 'skip': False, 'status': None, }, { 'error': None, 'execution_time': AnyFloat(), 'fail': None, 'warn': None, 'node': { 'alias': 'view_summary', 'build_path': Normalized( 'target/compiled/test/ref_models/view_summary.sql' ), 'alias': 'view_summary', 'columns': { 'first_name': { 'description': 'The first name being summarized', 'name': 'first_name', 'data_type': None, 'meta': {}, 'tags': [], }, 'ct': { 'description': 'The number of instances of the first name', 'name': 'ct', 'data_type': None, 'meta': {}, 'tags': [], }, }, 'compiled': True, 'compiled_sql': view_compiled_sql, 'config': { 'enabled': True, 'materialized': 'view', 'persist_docs': {}, 'pre-hook': [], 'post-hook': [], 'vars': {}, 'column_types': {}, 'quoting': {}, 'tags': [], 'full_refresh': None, }, 'sources': [], 'depends_on': { 'nodes': ['model.test.ephemeral_summary'], 'macros': [] }, 'description': ( 'A view of the summary of the ephemeral copy of the ' 'seed data' ), 'docs': {'show': True}, 'extra_ctes': [], 'extra_ctes_injected': True, 'fqn': ['test', 'view_summary'], 'injected_sql': view_compiled_sql, 'meta': {}, 'name': 'view_summary', 'original_file_path': self.dir('ref_models/view_summary.sql'), 'package_name': 'test', 'patch_path': self.dir('ref_models/schema.yml'), 'path': 'view_summary.sql', 'raw_sql': LineIndifferent( '{{\n config(\n materialized = "view"\n )\n}}\n\n' 'select first_name, ct from ' "{{ref('ephemeral_summary')}}\norder by ct asc" ), 'refs': [['ephemeral_summary']], 'resource_type': 'model', 'root_path': self.test_root_dir, 'schema': my_schema_name, 'database': self.default_database, 'tags': [], 'unique_id': 'model.test.view_summary', }, 'thread_id': ANY, 'timing': [ANY, ANY], 'skip': False, 'status': None, }, { 'error': None, 'execution_time': AnyFloat(), 'fail': None, 'warn': None, 'node': { 'alias': 'seed', 'build_path': None, 'columns': { 'id': { 'name': 'id', 'description': 'The user ID number', 'data_type': None, 'meta': {}, 'tags': [], }, 'first_name': { 'name': 'first_name', 'description': "The user's first name", 'data_type': None, 'meta': {}, 'tags': [], }, 'email': { 'name': 'email', 'description': "The user's email", 'data_type': None, 'meta': {}, 'tags': [], }, 'ip_address': { 'name': 'ip_address', 'description': "The user's IP address", 'data_type': None, 'meta': {}, 'tags': [], }, 'updated_at': { 'name': 'updated_at', 'description': "The last time this user's email was updated", 'data_type': None, 'meta': {}, 'tags': [], }, }, 'compiled': True, 'compiled_sql': '', 'config': { 'column_types': {}, 'enabled': True, 'materialized': 'seed', 'persist_docs': {}, 'post-hook': [], 'pre-hook': [], 'quoting': {}, 'vars': {}, 'tags': [], 'quote_columns': True, 'full_refresh': None, }, 'sources': [], 'depends_on': {'macros': [], 'nodes': []}, 'description': 'The test seed', 'docs': {'show': True}, 'extra_ctes': [], 'extra_ctes_injected': True, 'fqn': ['test', 'seed'], 'injected_sql': '', 'meta': {}, 'name': 'seed', 'original_file_path': self.dir('seed/seed.csv'), 'package_name': 'test', 'patch_path': self.dir('seed/schema.yml'), 'path': 'seed.csv', 'raw_sql': '', 'refs': [], 'resource_type': 'seed', 'root_path': self.test_root_dir, 'schema': my_schema_name, 'database': self.default_database, 'tags': [], 'unique_id': 'seed.test.seed', }, 'thread_id': ANY, 'timing': [ANY, ANY], 'skip': False, 'status': None, }, ] def verify_run_results(self, expected_run_results): run_result = _read_json('./target/run_results.json') self.assertIn('generated_at', run_result) self.assertIn('elapsed_time', run_result) self.assertBetween( run_result['generated_at'], start=self.generate_start_time ) self.assertGreater(run_result['elapsed_time'], 0) self.assertTrue( isinstance(run_result['elapsed_time'], float), "run_result['elapsed_time'] is of type {}, expected float".format( str(type(run_result['elapsed_time']))) ) # sort the results so we can make reasonable assertions run_result['results'].sort(key=lambda r: r['node']['unique_id']) self.assertEqual(run_result['results'], expected_run_results) @use_profile('postgres') def test__postgres__run_and_generate_no_compile(self): self.run_and_generate(alternate_db=self.default_database, args=['--no-compile']) self.verify_catalog(self.expected_postgres_catalog()) self.assertFalse(os.path.exists('./target/manifest.json')) @use_profile('postgres') def test__postgres__run_and_generate(self): self.run_and_generate(alternate_db=self.default_database) self.verify_catalog(self.expected_postgres_catalog()) self.verify_manifest(self.expected_seeded_manifest( model_database=self.default_database )) self.verify_run_results(self.expected_run_results( model_database=self.default_database )) @use_profile('postgres') def test__postgres_references(self): self.run_and_generate( {'source-paths': [self.dir('ref_models')]}, model_count=2, alternate_db=self.default_database ) self.verify_catalog(self.expected_postgres_references_catalog()) self.verify_manifest(self.expected_postgres_references_manifest()) self.verify_run_results(self.expected_postgres_references_run_results()) @use_profile('snowflake') def test__snowflake__run_and_generate(self): self.run_and_generate() self.verify_catalog(self.expected_snowflake_catalog()) self.verify_manifest(self.expected_seeded_manifest()) self.verify_run_results(self.expected_run_results( quote_schema=False, quote_model=False )) @use_profile('snowflake') def test__snowflake__run_and_generate_ignore_quoting_parameter(self): # with optional adapters, this package could easily just not exist! # accordingly, only run it when we think snowflake things should work from dbt.adapters.snowflake import connections as snowflake_conn old_connect = snowflake_conn.snowflake.connector.connect def connect(*args, **kwargs): kwargs['session_parameters'] = { 'QUOTED_IDENTIFIERS_IGNORE_CASE': True } return old_connect(*args, **kwargs) with patch.object(snowflake_conn.snowflake.connector, 'connect', connect): self.run_and_generate({ 'quoting': { 'identifier': True, } }) self.verify_catalog(self.expected_snowflake_catalog(case_columns=True)) self.verify_manifest(self.expected_seeded_manifest()) self.verify_run_results(self.expected_run_results(quote_schema=False, quote_model=True)) @use_profile('bigquery') def test__bigquery__run_and_generate(self): self.run_and_generate() self.verify_catalog(self.expected_bigquery_catalog()) self.verify_manifest(self.expected_seeded_manifest()) self.verify_run_results(self.expected_run_results()) @use_profile('bigquery') def test__bigquery__complex_models(self): self.run_and_generate( extra={'source-paths': [self.dir('bq_models')]}, model_count=4 ) self.verify_catalog(self.expected_bigquery_complex_catalog()) self.verify_manifest(self.expected_bigquery_complex_manifest()) @use_profile('redshift') def test__redshift__run_and_generate(self): self.run_and_generate(alternate_db=self.default_database) self.verify_catalog(self.expected_redshift_catalog()) self.verify_manifest(self.expected_seeded_manifest( model_database=self.default_database )) self.verify_run_results(self.expected_run_results( model_database=self.default_database )) @use_profile('redshift') def test__redshift__incremental_view(self): self.run_and_generate( {'source-paths': [self.dir('rs_models')]}, alternate_db=self.default_database ) self.verify_catalog(self.expected_redshift_incremental_catalog()) self.verify_manifest(self.expected_redshift_incremental_view_manifest()) @use_profile('presto') def test__presto__run_and_generate(self): self.run_and_generate(alternate_db=self.default_database) self.verify_catalog(self.expected_presto_catalog()) self.verify_manifest(self.expected_seeded_manifest( model_database=self.default_database )) self.verify_run_results(self.expected_run_results( model_database=self.default_database )) class TestDocsGenerateMissingSchema(DBTIntegrationTest): @property def schema(self): return 'docs_generate_029' @staticmethod def dir(path): return normalize(path) @property def models(self): return self.dir("bq_models_noschema") def setUp(self): super().setUp() self.extra_schema = self.unique_schema() + '_bq_test' def tearDown(self): with self.adapter.connection_named('__test'): self._drop_schema_named(self.default_database, self.extra_schema) super().tearDown() @use_profile('bigquery') def test_bigquery_docs_generate_noschema(self): self.run_dbt([ 'docs', 'generate', '--vars', "{{extra_schema: {}}}".format(self.extra_schema) ]) class TestDocsGenerateOverride(DBTIntegrationTest): @property def schema(self): return 'docs_generate_029' @staticmethod def dir(path): return normalize(path) @property def models(self): return self.dir("trivial_models") @property def project_config(self): return { 'config-version': 2, 'macro-paths': [self.dir('fail_macros')], } @use_profile('postgres') def test_postgres_override_used(self): self.assertEqual(len(self.run_dbt(['run'])), 1) # this should pick up our failure macro and raise a compilation exception with self.assertRaises(CompilationException) as exc: self.run_dbt(['docs', 'generate']) self.assertIn('rejected: no catalogs for you', str(exc.exception))