Merge branch 'master' into remove-list-relations

Jacob Beck · Jacob Beck · commit 660454e49e2f · 2018-10-17T12:24:37.000-06:00
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -18,7 +18,9 @@ jobs:
           command: |
             python3 -m venv venv
             . venv/bin/activate
+
             pip install dbt
+
             mkdir -p ~/.dbt
             cp integration_tests/ci/sample.profiles.yml ~/.dbt/profiles.yml
 
@@ -33,8 +35,8 @@ jobs:
             . venv/bin/activate
             cd integration_tests
             dbt deps
-            dbt seed
-            dbt run
+            dbt seed --full-refresh
+            dbt run --full-refresh
             dbt test
 
       - save_cache:
diff --git a/README.md b/README.md
@@ -175,7 +175,7 @@ Usage:
 ```
 
 #### star ([source](macros/sql/star.sql))
-This macro generates a list of all fields that exist in the `from` relation, excluding any fields listed in the `except` argument. The construction is identical to `select * from {{ref('my_model')}}`, replacing star (`*`) with the star macro.
+This macro generates a list of all fields that exist in the `from` relation, excluding any fields listed in the `except` argument. The construction is identical to `select * from {{ref('my_model')}}`, replacing star (`*`) with the star macro. This macro also has an optional `relation_alias` argument that will prefix all generated fields with an alias.
 
 Usage:
 ```
diff --git a/integration_tests/Makefile b/integration_tests/Makefile
@@ -1,22 +1,22 @@
 
 test-postgres:
-	dbt seed --target postgres
-	dbt run --target postgres --exclude test_insert_by_period
+	dbt seed --target postgres --full-refresh
+	dbt run --target postgres --full-refresh --exclude test_insert_by_period
 	dbt test --target postgres --exclude test_insert_by_period
 
 test-redshift:
-	dbt seed --target redshift
-	dbt run --target redshift
+	dbt seed --target redshift --full-refresh
+	dbt run --target redshift --full-refresh
 	dbt test --target redshift
 
 test-snowflake:
-	dbt seed --target snowflake
-	dbt run --target snowflake
+	dbt seed --target snowflake --full-refresh
+	dbt run --target snowflake --full-refresh
 	dbt test --target snowflake
 
 test-bigquery:
-	dbt seed --target bigquery
-	dbt run --target bigquery
+	dbt seed --target bigquery --full-refresh
+	dbt run --target bigquery --full-refresh
 	dbt test --target bigquery
 
 test-all: test-postgres test-redshift test-snowflake test-bigquery
diff --git a/integration_tests/data/sql/data_union_expected.csv b/integration_tests/data/sql/data_union_expected.csv
@@ -0,0 +1,6 @@
+id,name,favorite_color
+1,"drew",
+2,"bob",
+3,"alice",
+1,,"green"
+2,,"pink"
diff --git a/integration_tests/data/sql/data_union_table_1.csv b/integration_tests/data/sql/data_union_table_1.csv
@@ -0,0 +1,4 @@
+id,name
+1,drew
+2,bob
+3,alice
diff --git a/integration_tests/data/sql/data_union_table_2.csv b/integration_tests/data/sql/data_union_table_2.csv
@@ -0,0 +1,3 @@
+id,favorite_color
+1,green
+2,pink
diff --git a/integration_tests/models/sql/schema.yml b/integration_tests/models/sql/schema.yml
@@ -56,3 +56,8 @@ test_surrogate_key:
     constraints:
         assert_equal:
             - {actual: actual, expected: expected}
+
+test_union:
+    constraints:
+        dbt_utils.equality:
+            - ref('data_union_expected')
diff --git a/integration_tests/models/sql/test_get_column_values.sql b/integration_tests/models/sql/test_get_column_values.sql
@@ -3,6 +3,7 @@
 
 
 select
+    {% set columns = columns if columns is iterable else [] %}
     {% for column in columns -%}
 
         sum(case when field = '{{ column }}' then 1 else 0 end) as count_{{ column }}
diff --git a/integration_tests/models/sql/test_union.sql b/integration_tests/models/sql/test_union.sql
@@ -0,0 +1,8 @@
+
+select
+    id,
+    name,
+    favorite_color
+
+from {{ ref('test_union_base') }}
+
diff --git a/integration_tests/models/sql/test_union_base.sql b/integration_tests/models/sql/test_union_base.sql
@@ -0,0 +1,6 @@
+
+{{ dbt_utils.union_tables([
+        ref('data_union_table_1'),
+        ref('data_union_table_2')]
+) }} 
+
diff --git a/macros/cross_db_utils/literal.sql b/macros/cross_db_utils/literal.sql
@@ -0,0 +1,8 @@
+
+{% macro string_literal(value) %}
+  {{ adapter_macro('dbt_utils.string_literal', value) }}
+{% endmacro %}
+
+{% macro default__string_literal(value) -%}
+    '{{ value }}'
+{%- endmacro %}
diff --git a/macros/sql/star.sql b/macros/sql/star.sql
@@ -1,4 +1,4 @@
-{% macro star(from, except=[]) -%}
+{% macro star(from, relation_alias=False, except=[]) -%}
 
     {#-- Prevent querying of db in parsing mode. This works because this macro does not create any new refs. #}
     {%- if not execute -%}
@@ -23,7 +23,7 @@
 
     {%- for col in include_cols %}
 
-        "{{ col }}" {% if not loop.last %},
+        {% if relation_alias %} {{ relation_alias }}.{% endif %}"{{ col }}" {% if not loop.last %},
         {% endif %}
 
     {%- endfor -%}
diff --git a/macros/sql/union.sql b/macros/sql/union.sql
@@ -56,15 +56,15 @@
         (
             select
 
-                '{{ table }}'::text as _dbt_source_table,
+                {{ dbt_utils.safe_cast(dbt_utils.string_literal(table), dbt_utils.type_string()) }} as _dbt_source_table,
 
                 {% for col_name in ordered_column_names -%}
 
                     {%- set col = column_superset[col_name] %}
                     {%- set col_type = column_override.get(col.column, col.data_type) %}
                     {%- set col_name = adapter.quote(col_name) if col_name in table_columns[table] else 'null' %}
 
-                    {{ col_name }}::{{ col_type }} as {{ col.quoted }} {% if not loop.last %},{% endif %}
+                    {{ dbt_utils.safe_cast(col_name, col_type) }} as {{ col.quoted }} {% if not loop.last %},{% endif %}
                 {%- endfor %}
 
             from {{ table }}
diff --git a/macros/sql/unpivot.sql b/macros/sql/unpivot.sql
@@ -13,6 +13,8 @@ Arguments:
 
   {%- set exclude = exclude if exclude is not none else [] %}
 
+  {%- set include_cols = [] %}
+
   {%- set table_columns = {} %}
 
   {%- set _ = table_columns.update({table: []}) %}
@@ -26,18 +28,22 @@ Arguments:
   {%- set cols = adapter.get_columns_in_table(schema, table_name) %}
 
   {%- for col in cols -%}
-
-  {%- if col.column not in exclude -%}
-  select
-    {%- for exclude_col in exclude %}
-    {{ exclude_col }},
-    {%- endfor %}
-    cast('{{ col.column }}' as varchar) as field_name,
-    {{ dbt_utils.safe_cast(field=col.column, type=cast_to) }} as value
-  from {{ table }}
-  {% if not loop.last -%}
-  union all
-  {% endif -%}
-  {%- endif -%}
+    {%- if col.column not in exclude -%}
+      {% set _ = include_cols.append(col) %}
+    {%- endif %}
+  {%- endfor %}
+
+  {%- for col in include_cols -%}
+
+    select
+      {%- for exclude_col in exclude %}
+        {{ exclude_col }},
+      {%- endfor %}
+      cast('{{ col.column }}' as varchar) as field_name,
+      {{ dbt_utils.safe_cast(field=col.column, type=cast_to) }} as value
+    from {{ table }}
+    {% if not loop.last -%}
+      union all
+    {% endif -%}
   {%- endfor -%}
 {%- endmacro %}

-Original file line number
+Diff line change
@@ @@ -0,0 +1,4 @@ @@
 +id,name
 +1,drew
 +2,bob
 +3,alice
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+id,favorite_color`
	`2`	`+1,green`
	`3`	`+2,pink`