Merge pull request #143 from USEPA/tj_troy_elci3_development

m-jamieson · web-flow · commit 2232c41f2cb4 · 2020-08-26T05:34:02.000-04:00
ELCI3: integration of the International flows of Canada and mexico for the EPA-eGRID trading methods, surplus pools.etc.
diff --git a/electricitylci/__init__.py b/electricitylci/__init__.py
@@ -210,6 +210,40 @@ def write_generation_mix_database_to_dict(
     return genmix_dict
 
 
+def write_fuel_mix_database_to_dict(
+    genmix_database, gen_dict, regions=None
+):
+    from electricitylci.generation_mix import olcaschema_usaverage
+    if regions is None:
+        regions = config.model_specs.regional_aggregation
+    if regions in ["FERC","US","BA"]:
+        usaverage_dict = olcaschema_usaverage(
+                genmix_database, gen_dict, subregion="BA"
+        )
+    else:
+        usaverage_dict = olcaschema_usaverage(
+            genmix_database, gen_dict, subregion=regions
+        )
+    return usaverage_dict
+
+
+def write_international_mix_database_to_dict(
+    genmix_database, usfuelmix_dict, regions=None
+):
+    from electricitylci.generation_mix import olcaschema_international;
+    if regions is None:
+        regions = config.model_specs.regional_aggregation
+    if regions in ["FERC","US","BA"]:
+        international_dict = olcaschema_international(
+                genmix_database, usfuelmix_dict, subregion="BA"
+        )
+    else:
+        international_dict = olcaschema_international(
+            genmix_database, usfuelmix_dict, subregion=regions
+        )
+    return international_dict
+
+
 def write_surplus_pool_and_consumption_mix_dict():
     """
     Create olca formatted dictionaries for the consumption mix as calculated by
diff --git a/electricitylci/consumption_mix.py b/electricitylci/consumption_mix.py
@@ -55,7 +55,6 @@ def surplus_pool_dictionary(nerc_region, surplus_pool_trade_in, trade_matrix, ge
                 # chk = 1;
 
         final = process_table_creation_surplus(region, exchanges_list)
-        print(region+' NERC Surplus Process Created')
         surplus_dict['SurplusPool'+region] = final;
     return surplus_dict
 
@@ -84,7 +83,6 @@ def consumption_mix_dictionary(nerc_region, surplus_pool_trade_in, trade_matrix,
                         # name = surplus_dict[nerc_region[reg][0].value]['name']
 
                         if trade_matrix[nerc+1][j].value != None and trade_matrix[nerc+1][j].value !=0:
-                            print(nerc_region[reg][0].value)
                             exchange(exchange_table_creation_input_con_mix(surplus_pool_trade_in[reg][0].value, nerc_region[reg][0].value), exchanges_list)
                             chk=1;
                             break;
@@ -96,7 +94,6 @@ def consumption_mix_dictionary(nerc_region, surplus_pool_trade_in, trade_matrix,
             exchange(exchange_table_creation_input_con_mix(1, region), exchanges_list)
 
         final = process_table_creation_con_mix(region, exchanges_list)
-        print(region+' Consumption Mix Process Created')
         consumption_dict['Consumption'+region] = final;
     return consumption_dict
 
diff --git a/electricitylci/data/International_Electricity_Mix.csv b/electricitylci/data/International_Electricity_Mix.csv
@@ -0,0 +1,51 @@
+Subregion,FuelCategory,Electricity,Generation_Ratio,Year
+British Columbia,GAS,,0.02357189,2016
+British Columbia,COAL,,0,2016
+British Columbia,NUCLEAR,,0,2016
+British Columbia,GEOTHERMAL,,0.057344566,2016
+British Columbia,SOLAR,,2.94E-05,2016
+British Columbia,WIND,,0.009877884,2016
+British Columbia,HYDRO,,0.909176305,2016
+Alberta,GAS,,0.416603496,2016
+Alberta,COAL,,0.475944,2016
+Alberta,NUCLEAR,,0,2016
+Alberta,GEOTHERMAL,,0.022504382,2016
+Alberta,SOLAR,,1.18E-05,2016
+Alberta,WIND,,0.056722888,2016
+Alberta,HYDRO,,0.028213389,2016
+Saskatchewan,GAS,,0.250935862,2016
+Saskatchewan,COAL,,0.524041261,2016
+Saskatchewan,NUCLEAR,,0,2016
+Saskatchewan,GEOTHERMAL,,0.006280675,2016
+Saskatchewan,SOLAR,,4.16E-05,2016
+Saskatchewan,WIND,,0.030529906,2016
+Saskatchewan,HYDRO,,0.188170701,2016
+Manitoba,GAS,,0.001771549,2016
+Manitoba,COAL,,0.00153358,2016
+Manitoba,NUCLEAR,,0,2016
+Manitoba,GEOTHERMAL,,0.001401375,2016
+Manitoba,SOLAR,,2.64E-05,2016
+Manitoba,WIND,,0.022157589,2016
+Manitoba,HYDRO,,0.973109466,2016
+Ontario,GAS,,0.070797025,2016
+Ontario,COAL,,0,2016
+Ontario,NUCLEAR,,0.55173294,2016
+Ontario,GEOTHERMAL,,0.036202035,2016
+Ontario,SOLAR,,0.029588202,2016
+Ontario,WIND,,0.057992876,2016
+Ontario,HYDRO,,0.253686922,2016
+Quebec,GAS,,0.005132507,2016
+Quebec,COAL,,0,2016
+Quebec,NUCLEAR,,0,2016
+Quebec,GEOTHERMAL,,0.005777032,2016
+Quebec,SOLAR,,0,2016
+Quebec,WIND,,0.041349143,2016
+Quebec,HYDRO,,0.947741318,2016
+New Brunswick,GAS,,0.193446177,2016
+New Brunswick,COAL,,0.116984908,2016
+New Brunswick,NUCLEAR,,0.329775702,2016
+New Brunswick,GEOTHERMAL,,0.050529476,2016
+New Brunswick,SOLAR,,0,2016
+New Brunswick,WIND,,0.064537647,2016
+New Brunswick,HYDRO,,0.24472609,2016
+Mexico,WIND,,1,2016
diff --git a/electricitylci/data/eGRID_Consumption_Mix_new.xlsx b/electricitylci/data/eGRID_Consumption_Mix_new.xlsx
diff --git a/electricitylci/distribution.py b/electricitylci/distribution.py
@@ -10,6 +10,5 @@ def distribution_mix_dictionary():
         exchange(ref_exchange_creator(electricity_at_user_flow), exchanges_list)
         exchange(exchange_table_creation_input_con_mix(1/model_specs.efficiency_of_distribution_grid, reg, ref_to_consumption=True), exchanges_list)
         final = process_table_creation_distribution(reg, exchanges_list)
-        print(reg+' Distribution Process Created')
         distribution_dict['Distribution'+reg] = final;
     return distribution_dict
diff --git a/electricitylci/egrid_facilities.py b/electricitylci/egrid_facilities.py
@@ -70,3 +70,6 @@ def list_facilities_w_percent_generation_from_primary_fuel_category_greater_than
 
 # Merge back into facilities
 egrid_facilities = pd.merge(egrid_facilities, egrid_facilities_fuel_cat_per_gen, on=['FacilityID', 'FuelCategory'], how='left')
+
+international = pd.read_csv(data_dir+'/International_Electricity_Mix.csv')
+international_reg = list(pd.unique(international['Subregion']))
diff --git a/electricitylci/generation.py b/electricitylci/generation.py
@@ -1108,7 +1108,7 @@ def turn_data_to_dict(data, upstream_dict):
     process_df["@type"] = "Process"
     process_df["allocationFactors"] = ""
     process_df["defaultAllocationMethod"] = ""
-    process_df["location"] = ""
+    process_df["location"] = process_df[region_agg].values
     process_df["parameters"] = ""
 #    process_doc_dict = process_doc_creation(process_type)
 #    process_df["processDocumentation"] = [process_doc_dict]*len(process_df)
diff --git a/electricitylci/generation_mix.py b/electricitylci/generation_mix.py
@@ -5,6 +5,7 @@
 
 import numpy as np
 import pandas as pd
+from electricitylci.globals import data_dir
 from electricitylci.process_dictionary_writer import *
 from electricitylci.egrid_facilities import egrid_facilities, egrid_subregions
 from electricitylci.model_config import model_specs
@@ -328,10 +329,6 @@ def olcaschema_genmix(database, gen_dict, subregion=None):
                 database_reg["FuelCategory"] == fuelname
             ]
             if database_f1.empty != True:
-                ra = exchange_table_creation_input_genmix(
-                    database_f1, fuelname
-                )
-                ra["quantitativeReference"] = False
                 matching_dict = None
                 for generator in gen_dict:
                     if (
@@ -342,19 +339,151 @@ def olcaschema_genmix(database, gen_dict, subregion=None):
                         break
                 if matching_dict is None:
                     logging.warning(
-                        f"Trouble matching dictionary for {fuelname} - {reg}"
+                        f"Trouble matching dictionary for generation mix {fuelname} - {reg}. Skipping this flow for now"
                     )
                 else:
+                    ra = exchange_table_creation_input_genmix(
+                    database_f1, fuelname
+                    )
+                    ra["quantitativeReference"] = False                    
                     ra["provider"] = {
                         "name": matching_dict["name"],
                         "@id": matching_dict["uuid"],
                         "category": matching_dict["category"].split("/"),
                     }
-                exchange(ra, exchanges_list)
-                # Writing final file
+                    #if matching_dict is None:
+                    exchange(ra, exchanges_list)
+                    # Writing final file
 
         final = process_table_creation_genmix(reg, exchanges_list)
+        # print(reg +' Process Created')
+        generation_mix_dict[reg] = final
+    return generation_mix_dict
+
+
+def olcaschema_usaverage(database, gen_dict, subregion=None, excluded_regions = ['HIMS','HIOA','AKGD','AKMS']):
+    if subregion is None:
+        subregion = model_specs.regional_aggregation
+    generation_mix_dict = {}
+    # croppping the database according to the current fuel being considered
+    #Not choosing the Hawaiian and Alaskan regions.   
+    us_database = create_generation_mix_process_df_from_egrid_ref_data(subregion='US')
+    #Not choosing the Hawaiian and Alaskan regions.  
+    us_database=us_database.loc[~us_database["Subregion"].isin(excluded_regions),:]
+    df2 = us_database.groupby(['FuelCategory'])['Electricity'].agg('sum').reset_index()
+    df2['Electricity_fuel_total'] = df2['Electricity']
+    del df2['Electricity']
+    df3 = us_database.merge(df2,left_on ='FuelCategory',right_on = 'FuelCategory')
+    df3['Generation_Ratio'] = df3['Electricity']/df3['Electricity_fuel_total']
+    del df3['Electricity_fuel_total']
+    us_database = df3
+    if "FuelCategory" in us_database.columns:
+        fuels = list(pd.unique(us_database["FuelCategory"]))
+
+    for fuel in fuels:
+
+        database_reg = us_database[us_database["FuelCategory"] == fuel]
+        exchanges_list = []
+
+        # Creating the reference output
+        exchange(exchange_table_creation_ref(database_reg), exchanges_list)
+        for reg in list(us_database["Subregion"].unique()):
+            # Reading complete fuel name and heat content information
+            # fuelname = row['Fuelname']
+            # croppping the database according to the current fuel being considered
+            #Not choosing the Hawaiian and Alaskan regions. 
+            if reg in excluded_regions:
+                    continue
+            else:
+                    database_f1 = database_reg[
+                        database_reg["Subregion"] == reg
+                    ]
+                    if database_f1.empty != True:
+                        matching_dict = None
+                        for generator in gen_dict:
+                            if (
+                                gen_dict[generator]["name"]
+                                == "Electricity - " + fuel + " - " + reg
+                            ):
+                                matching_dict = gen_dict[generator]
+                                break
+                        if matching_dict is None:
+                            logging.warning(
+                                f"Trouble matching dictionary for creating fuel mix {fuel} - {reg}.Skipping this flow for now"
+                            )
+                        else:
+                            ra = exchange_table_creation_input_usaverage(
+                                database_f1, fuel
+                            )
+                            ra["quantitativeReference"] = False                    
+                            ra["provider"] = {
+                                "name": matching_dict["name"],
+                                "@id": matching_dict["uuid"],
+                                "category": matching_dict["category"].split("/"),
+                            }
+                            exchange(ra, exchanges_list)
+                            # Writing final file
+
+        final = process_table_creation_usaverage(fuel, exchanges_list)
+        # print(reg +' Process Created')
+        generation_mix_dict[fuel] = final 
+
+    return generation_mix_dict
+
+def olcaschema_international(database, gen_dict, subregion=None):
+    
+    intl_database = pd.read_csv(data_dir+'/International_Electricity_Mix.csv')
+    database = intl_database
+    generation_mix_dict = {}
+    if "Subregion" in database.columns:
+        region = list(pd.unique(database["Subregion"]))
+    else:
+        region = ["US"]
+        database["Subregion"] = "US"
+    for reg in region:
+
+        database_reg = database[database["Subregion"] == reg]
+        exchanges_list = []
+
+        # Creating the reference output
+        exchange(exchange_table_creation_ref(database_reg), exchanges_list)
+        for fuelname in list(database["FuelCategory"].unique()):
+            # Reading complete fuel name and heat content information
+            # fuelname = row['Fuelname']
+            # croppping the database according to the current fuel being considered
+            database_f1 = database_reg[
+                database_reg["FuelCategory"] == fuelname
+            ]
+            if database_f1.empty != True:
+                matching_dict = None
+                for generator in gen_dict:
+                    if (
+                        gen_dict[generator]["name"]
+                        == "Electricity; at grid; USaverage - " + fuelname
+                       ):
+                        matching_dict = gen_dict[generator]
+                        break
+                if matching_dict is None:
+                    logging.warning(
+                        f"Trouble matching dictionary for us average mix {fuelname} - USaverage. Skipping this flow for now"
+                    )
+                else:
+                    ra = exchange_table_creation_input_international_mix(
+                    database_f1, fuelname
+                    )
+                    ra["quantitativeReference"] = False                    
+                    ra["provider"] = {
+                       "name": matching_dict["name"],
+                       "@id": matching_dict["uuid"],
+                       "category": matching_dict["category"].split("/"),
+                    }
+                    #if matching_dict is None:
+                    exchange(ra, exchanges_list)
+                    # Writing final file
 
+        final = process_table_creation_genmix(reg, exchanges_list)
         # print(reg +' Process Created')
         generation_mix_dict[reg] = final
     return generation_mix_dict
+
+
diff --git a/electricitylci/main.py b/electricitylci/main.py
@@ -67,6 +67,7 @@ def main():
     generation_mix_dict = electricitylci.write_process_dicts_to_jsonld(
         generation_mix_dict
     )
+
     # At this point the two methods diverge from underlying functions enough that
     # it's just easier to split here.
     if config.model_specs.EPA_eGRID_trading is False:
@@ -100,24 +101,35 @@ def main():
             dist_mix_dicts[subreg] = electricitylci.write_process_dicts_to_jsonld(
                 dist_mix_dicts[subreg])
     else:
+        print("us average mix to dict")
+        usavegfuel_mix_dict = electricitylci.write_fuel_mix_database_to_dict(
+        generation_mix_df, generation_process_dict)
+        print("write us average mix to jsonld")
+        usavegfuel_mix_dict = electricitylci.write_process_dicts_to_jsonld(
+            usavegfuel_mix_dict
+        )    
+        print("international average mix to dict")
+        international_mix_dict = electricitylci.write_international_mix_database_to_dict(
+        generation_mix_df, usavegfuel_mix_dict)
+        international_mix_dict = electricitylci.write_process_dicts_to_jsonld(
+        international_mix_dict
+        ) 
         # Get surplus and consumption mix dictionary
         sur_con_mix_dict = electricitylci.write_surplus_pool_and_consumption_mix_dict()
         # Get dist dictionary
         dist_dict = electricitylci.write_distribution_dict()
         generation_mix_dict = electricitylci.write_process_dicts_to_jsonld(
             generation_mix_dict
         )
-        sur_con_mix_dict = fill_default_provider_uuids(
-            sur_con_mix_dict, generation_mix_dict
-        )
+        print('write surplus pool consumption mix to jsonld')
         sur_con_mix_dict = electricitylci.write_process_dicts_to_jsonld(sur_con_mix_dict)
+        print('Filling up UUID of surplus pool consumption mix')
         sur_con_mix_dict = fill_default_provider_uuids(
-            sur_con_mix_dict, sur_con_mix_dict, generation_mix_dict
+            sur_con_mix_dict, sur_con_mix_dict, generation_mix_dict, international_mix_dict
         )
         sur_con_mix_dict = electricitylci.write_process_dicts_to_jsonld(sur_con_mix_dict)
         dist_dict = fill_default_provider_uuids(dist_dict, sur_con_mix_dict)
         dist_dict = electricitylci.write_process_dicts_to_jsonld(dist_dict)
-
     logger.info(
         f'JSON-LD files have been saved in the "output" folder with the full path '
         f'{config.model_specs.namestr}'
diff --git a/electricitylci/process_dictionary_writer.py b/electricitylci/process_dictionary_writer.py
diff --git a/electricitylci/process_exchange_aggregator_uncertainty.py b/electricitylci/process_exchange_aggregator_uncertainty.py
