Added Summary Statistics Calculations and Batch Script

.gitignore

@@ -8,3 +8,5 @@ Intermediate/
 Examples/
 *.xlsm
 environmentExamples.yml
+Outputs/SCurveStats.csv
+Outputs/RimInflows_Summary*.csv

README.md

@@ -19,15 +19,19 @@ All the Upper American related files are named with the 'upper_american_' prefix
 
 The *upper_american_2022_extension_data.csv* file contains the data used in the previous extension. Where possible, the gap filling was removed and moved to be done in the code. Places where the DWR COMP model was used, were not removed.
 
-## To calculate the rim inflows
+## Calculating Rim Inflows
 
 ### To create an environment
 
 `conda env create -f environment.yml`
 
 `conda activate extension`
 
-### To recreate the rim inflows with the data in the repository:
+### To run the full rim inflow dataset calculations
+
+To run the full set of rim inflow scripts, run the run_rim_inflows.bat script. This batch file will run all rim inflows calculations for all locations within this respository as well as a summary script to caclutate summary metrics for all rim inflows.
+
+### To recreate the rim inflows for a basin with the data in the repository:
 
 Run `python upper_american_data_read.py` to read in the data and then `python upper_american_calculate_rim_inflows.py` to calculate the rim inflows.
 
@@ -40,6 +44,11 @@ The calculated flows will be in the *Outputs* folder in *upper_american_rim_infl
 
 The calculated flows will be in the *Outputs* folder in *rim_inflows.csv*.
 
+### To run summary statistics
+Run `python Summary_statistics.py` to calculate summary statistics for each rim inflow location. *Note rim inflows must be calculated prior to running the summary script
+
+## Developing Rim Inflows
+
 ### To incorporate additional locations
 Every location is different so the process to incorporate a new location is going to look different every time. For a new basin, create a new set of files named with the basin name. Generally, the following this must be added:
 1. Any data from the previous extension should be added, in TAF, to *Inputs/upper_american_2022_extension_data.csv*

Summary_statistics.py

@@ -0,0 +1,72 @@
+import pandas as pd
+
+# ----------------------------------------------
+# ---                INPUTS                  ---
+# ----------------------------------------------
+
+# Rim Inflow Basins (Names must match RimInflow Output files)
+sl_rim_inflow_basins = ["upper_american","upper_mokelumne"]
+
+# Summary Period
+i_final_year = 2021
+i_start_year = 1921
+
+# ----------------------------------------------
+# --- FORMAT & OUTPUT S-CURVE SUMMARY TABLE  ---
+# ----------------------------------------------
+
+# read in s_curve ouptut file
+s_scurveStats_fn = 'Outputs/SCurveStats.csv'
+df_scurve = pd.read_csv(s_scurveStats_fn)
+
+# remove the fit_slope and fit_intercept (these are duplicates, currently calculated in two functions of the rim inflow workflow, these values are exactly the same)
+df_scurve = df_scurve[[not(s_stat in ['fit_slope','fit_intercept']) for s_stat in df_scurve.stat]]
+
+# pivot the table
+df_scurve.drop_duplicates(inplace=True)
+df_scurve_format = df_scurve.pivot(index = ['reference location','target location'],columns='stat',values='value')
+df_scurve_format = df_scurve_format[['slope','intercept','r2']].reset_index()
+
+# To Do - add gage/lodation names, will require input table to cross reference gage #s and Names
+
+# save the output
+df_scurve_format.round(3).to_csv('Outputs/RimInflows_Summary_SCurveParameters.csv',index=False)
+
+# ----------------------------------------------
+# ---  CALCULATE & OUTPUT RIM INFLOW SUMMARY TABLE    ---
+# ----------------------------------------------
+
+# read in final rim inflow output csv files from each basin and merge into a single dataframe 
+dl_basins = [pd.read_csv(f'Outputs/{s}_rim_inflows.csv',index_col=0) for s in sl_rim_inflow_basins]
+df_rim_inflows = pd.concat(dl_basins,axis=1)
+df_rim_inflows.index = pd.to_datetime(df_rim_inflows.index)
+
+# clip to selected period
+# Question why does I_AMADR go to 2024-08-31??
+# Question why does the upper_mokelumne_rim_inflows.csv have a 1921-09-30
+df_rim_inflows = df_rim_inflows[df_rim_inflows.index < pd.to_datetime(str(i_final_year) + '-10-01')]
+df_rim_inflows = df_rim_inflows[df_rim_inflows.index > pd.to_datetime(str(i_start_year) + '-09-30')]
+
+# calculate mean and median monthly values for each rim inflow
+df_rim_inflows_monthlyTS = df_rim_inflows.resample('ME').sum()
+df_rim_inflows_meanMon = df_rim_inflows_monthlyTS.groupby(df_rim_inflows_monthlyTS.index.month_name()).mean().T 
+df_rim_inflows_medianMon = df_rim_inflows_monthlyTS.groupby(df_rim_inflows_monthlyTS.index.month_name()).median().T
+
+# calculate mean and median total annual flow for each rim infow (calculated over water years)
+df_rim_inflows['wy'] = df_rim_inflows.index.year.where(df_rim_inflows.index.month<10,df_rim_inflows.index.year+1)
+df_rim_inflows_annualTS = df_rim_inflows.groupby('wy').sum()
+df_rim_inflows_annualMetric = df_rim_inflows_annualTS.agg(["mean","median"]).T
+
+# join monthly and annual datasets
+df_rim_inflows_summaryMedian = pd.concat([df_rim_inflows_medianMon[['October','November','December','January','February','March','April','May','June','July','August','September']],
+                            df_rim_inflows_annualMetric],axis=1)
+df_rim_inflows_summaryMean = pd.concat([df_rim_inflows_meanMon[['October','November','December','January','February','March','April','May','June','July','August','September']],
+                            df_rim_inflows_annualMetric],axis=1)
+
+# output combined dataframes
+df_rim_inflows_summaryMedian.to_csv('Outputs/RimInflows_Summary_MonthlyMedianandAnnualFlows.csv')
+df_rim_inflows_summaryMean.to_csv('Outputs/RimInflows_Summary_MonthlyAvgandAnnualFlows.csv')
+
+# ouptut combined dataframes rounded to one decimal place for the final tables
+df_rim_inflows_summaryMedian.round(1).to_csv('Outputs/RimInflows_Summary_MonthlyMedianandAnnualFlows_rounded.csv')
+df_rim_inflows_summaryMean.round(1).to_csv('Outputs/RimInflows_Summary_MonthlyAvgandAnnualFlows_rounded.csv')

extension_functions.py

@@ -12,7 +12,8 @@
 
 
 def s_curve_disaggregation(df_x_data, df_y_data, i_x_start_year, i_x_end_year, i_y_start_year,
-                           i_y_end_year, b_use_all_y=False, s_strange_sheet=''):
+                           i_y_end_year, b_use_all_y=False, s_strange_sheet='',b_save_stats=False,
+                           s_reference_name = '',s_target_name = '',s_out_stats = 'Outputs/SCurveStats.csv'):
     """
     Takes in the x data and the y data and generated a full timeseries of synthetic y data.
     This is meant to replicate what the Excel/VBA does for the S-Curve disaggregation.
@@ -35,6 +36,14 @@ def s_curve_disaggregation(df_x_data, df_y_data, i_x_start_year, i_x_end_year, i
         Whether to use all y data or just the section in the Y years
     s_strange_sheet: string
         For a few sheets with strange modifications to the s-curve procedure, this is the sheet name with capital letters.
+    b_save_stats: boolean
+        Option to save s-curve stats
+    s_reference_name: str
+        Name of reference location, only needed if saving s_curve stats to file, i.e. b_save_stats = True
+    s_target_name: str
+        Name of target location, only needed if saving s_curve stats to file, i.e. b_save_stats = True
+    s_out_stats: string
+        Output file to save s-curve stats if b_save_stats is true
 
     Returns
     -------
@@ -155,10 +164,21 @@ def s_curve_disaggregation(df_x_data, df_y_data, i_x_start_year, i_x_end_year, i
     else:
         df_y_data_output.loc[i_y_start_year:i_y_end_year, :] = df_y_data.loc[i_y_start_year:i_y_end_year, :]
 
+    # save stats if indicated
+    # add a line to the output csv with location,slope,intercept
+    if b_save_stats:
+        df_stats = pd.DataFrame(
+            {'reference location':[s_reference_name]*2,
+            'target location':[s_target_name]*2,
+            'stat':['slope','intercept'],
+            'value':[d_slope,d_intercept]})
+        df_stats.to_csv(s_out_stats,mode='a',index=False,header=False) 
+
     return df_y_data_output, df_y_data_synthetic
 
 
-def s_curve_comparison_plots(df_final_y_dat, df_y_data_synthetic, df_x_data, df_y_data, s_current_location):
+def s_curve_comparison_plots(df_final_y_dat, df_y_data_synthetic, df_x_data, df_y_data, s_current_location,s_reference_location,
+                            b_save_stats=False,s_out_stats = 'Outputs/SCurveStats.csv'):
     """
     Generates two plots to understand the quality of the generated data.
     First plot compares the historical y data and the reference x data.
@@ -176,11 +196,20 @@ def s_curve_comparison_plots(df_final_y_dat, df_y_data_synthetic, df_x_data, df_
         Original y data
     s_current_location: str
         Current location of the data
+    s_reference_location: str
+        Reference location name
+    b_save_stats: boolean
+        Option to save s-curve stats
+    s_out_stats: string
+        Output file to save s-curve stats if b_save_stats is true
     Returns
     -------
     None
     """
 
+    # save dataset names
+
+
     # first remove nans so they wont get plotted as zeros
     df_x_data.dropna(inplace=True)
     df_y_data.dropna(inplace=True)
@@ -240,6 +269,15 @@ def s_curve_comparison_plots(df_final_y_dat, df_y_data_synthetic, df_x_data, df_
     plt.savefig(f'./Figures/{s_current_location} Monthly Flows Observed vs Synthetic', bbox_inches='tight', dpi=300)
     plt.close()
 
+    # save stats if indicated
+    # add a line to the output csv with location,slope,intercept
+    if b_save_stats:
+        df_stats = pd.DataFrame(
+            {'reference location':[s_reference_location]*3,
+            'target location':[s_current_location]*3,
+            'stat':['fit_slope','fit_intercept','r2'],
+            'value':[slope,intercept,r2]})
+        df_stats.to_csv(s_out_stats,mode='a',index=False,header=False) 
 
 def read_data(s_path):
     """
@@ -885,7 +923,7 @@ def read_previous_data(s_path, df_new_data):
 
 def extend_data(df_reference_data, df_current_data, df_extended_data, df_synthetic_data,
                 i_y_start_year, i_y_end_year, b_use_all_y_data, s_name, i_x_start_year=1922,
-                i_final_year=2021, s_strange_sheet=''):
+                i_final_year=2021, s_strange_sheet='',b_save_stats=False):
 
     """
     Extends data using the s-curve disaggregation. Also creates the plots and saves the data into dataframes.
@@ -912,6 +950,9 @@ def extend_data(df_reference_data, df_current_data, df_extended_data, df_synthet
         Final year for the x data
     s_strange_sheet: string
         For a few sheets with strange modifications to the s-curve procedure, this is the sheet name with capital letters.
+    b_save_stats: boolean
+        Option to save stats of fitted s-curve
+
     Returns
     -------
     None
@@ -921,11 +962,14 @@ def extend_data(df_reference_data, df_current_data, df_extended_data, df_synthet
                                                                         df_current_data,
                                                                         i_x_start_year, i_final_year,
                                                                         i_y_start_year, i_y_end_year,
-                                                                        b_use_all_y_data, s_strange_sheet)
+                                                                        b_use_all_y_data, s_strange_sheet,
+                                                                        s_target_name = s_name,
+                                                                        s_reference_name= df_reference_data.name,
+                                                                        b_save_stats=b_save_stats)
     # generate the comparison plots
     s_curve_comparison_plots(df_curr_final_data, df_curr_synthetic_data,
                              timeseries_to_monthly(df_reference_data), timeseries_to_monthly(df_current_data),
-                             s_name)
+                             s_name,df_reference_data.name,b_save_stats=b_save_stats)
 
     # put the data into the two final dataframes
     df_extended_data[s_name] = monthly_to_timeseries(df_curr_final_data)
@@ -935,7 +979,7 @@ def extend_data_multi_model(df_reference_data_1, df_current_data_1, df_reference
                 df_extended_data, df_synthetic_data,
                 i_y1_start_year, i_y1_end_year, i_y2_start_year, i_y2_end_year, b_use_all_y_data, s_name,
                 s_model_name_1, s_model_name_2, i_x_start_year=1922,
-                i_final_year=2021, s_strange_sheet=''):
+                i_final_year=2021, s_strange_sheet='',b_save_stats=False):
 
     """
     Extends data using the s-curve disaggregation and compares two different models.
@@ -971,6 +1015,8 @@ def extend_data_multi_model(df_reference_data_1, df_current_data_1, df_reference
         Final year for the x data
     s_strange_sheet: string
         For a few sheets with strange modifications to the s-curve procedure, this is the sheet name with capital letters.
+    b_save_stats: boolean
+        Option to save stats of fitted s-curve
     Returns
     -------
     None
@@ -980,18 +1026,25 @@ def extend_data_multi_model(df_reference_data_1, df_current_data_1, df_reference
                                                                         df_current_data_1,
                                                                         i_x_start_year, i_final_year,
                                                                         i_y1_start_year, i_y1_end_year,
-                                                                        b_use_all_y_data, s_strange_sheet)
+                                                                        b_use_all_y_data, s_strange_sheet,
+                                                                        s_target_name = s_name + '_ref1',
+                                                                        s_reference_name= df_reference_data_1.name,
+                                                                        b_save_stats=b_save_stats)
     # do the s-curve disaggregation for model 2
     df_curr_final_data_2, df_curr_synthetic_data_2 = s_curve_disaggregation(df_reference_data_2,
                                                                             df_current_data_2,
                                                                             i_x_start_year, i_final_year,
                                                                             i_y2_start_year, i_y2_end_year,
-                                                                            b_use_all_y_data, s_strange_sheet)
+                                                                            b_use_all_y_data, s_strange_sheet,
+                                                                            s_target_name = s_name + '_ref2',
+                                                                            s_reference_name = df_reference_data_2.name,
+                                                                            b_save_stats=b_save_stats)
 
     # generate the comparison plots
     two_s_curves_comparison_plots(df_curr_final_data_1, timeseries_to_monthly(df_reference_data_1),
                                   df_curr_final_data_2, timeseries_to_monthly(df_reference_data_2), s_name,
-                                  s_model_name_1, s_model_name_2)
+                                  s_model_name_1, s_model_name_2,s_ref_name_1 = df_reference_data_1.name, 
+                                  s_ref_name_2 = df_reference_data_2.name,b_save_stats=b_save_stats)
 
     # put the data into the two final dataframes
     df_extended_data[s_name+s_model_name_1] = monthly_to_timeseries(df_curr_final_data_1)
@@ -1261,7 +1314,9 @@ def read_replication_data(ls_sheet_info, df_before, df_after):
         df_after[sheet[0]] = monthly_to_timeseries(df_temp)
 
 def two_s_curves_comparison_plots(df_final_y_dat_1, df_x_data_1,
-                                  df_final_y_dat_2, df_x_data_2, s_current_location, s_model_name_1, s_model_name_2):
+                                  df_final_y_dat_2, df_x_data_2, s_current_location, s_model_name_1, s_model_name_2,
+                                  s_ref_name_1='',s_ref_name_2='',
+                                  s_out_stats = 'Outputs/SCurveStats.csv',b_save_stats=False):
     """
     Generates a plot to compare model 1 and model 2 for a sheet.
 
@@ -1281,6 +1336,14 @@ def two_s_curves_comparison_plots(df_final_y_dat_1, df_x_data_1,
         The name for model 1
     s_model_name_2: str
         The name for model 2
+    s_ref_name_1: str
+        Reference gage name for model 1
+    s_ref_name_2: str
+        Reference gage name for model 2
+    b_save_stats: boolean
+        Option to save s-curve stats
+    s_out_stats: string
+        Output file to save s-curve stats if b_save_stats is true
     Returns
     -------
     None
@@ -1336,3 +1399,13 @@ def two_s_curves_comparison_plots(df_final_y_dat_1, df_x_data_1,
     plt.savefig(f'./Figures/Model_Comparison/{s_current_location} Comparison of Two Models, {s_model_name_1} and {s_model_name_2}'
                 , bbox_inches='tight', dpi=300)
     plt.close()
+
+    # save stats if indicated
+    # add a line to the output csv with location,slope,intercept
+    if b_save_stats:
+        df_stats = pd.DataFrame(
+            {'reference location':[s_ref_name_1]*3+[s_ref_name_2]*3,
+            'target location':[s_current_location + s_model_name_1]*3 + [s_current_location + s_model_name_2]*3,
+            'stat':['fit_slope','fit_intercept','r2']*2,
+            'value':[slope_1,intercept_1,r2_1,slope_2,intercept_2,r2_2]})
+        df_stats.to_csv(s_out_stats,mode='a',index=False,header=False) 

run_rim_inflows.bat

@@ -0,0 +1,45 @@
+@echo off 
+
+:: Start Python Environment
+echo Starting Python Environment
+call conda activate extension
+
+:: Remove previous output files series of steps to ensure the correct files are deleted
+echo WARNING removing previous output files
+pause
+
+set "TARGET_FOLDER=Outputs"
+set "FULL_PATH=%~dp0%TARGET_FOLDER%"
+
+if "%TARGET_FOLDER%"=="" (
+    echo ERROR: TARGET_FOLDER variable is empty! Aborting.
+    pause
+    exit /b
+)
+if not exist "%FULL_PATH%\" (
+    echo ERROR: Target folder "%FULL_PATH%" does not exist! Aborting.
+    pause
+    exit /b
+)
+
+del /f /q "%FULL_PATH%\*.*"
+echo Done! All files deleted safely.
+pause
+
+
+echo Starting Rim Inflow Python script sequence
+
+:: Upper American
+echo Running Upper American Module
+python  upper_american_data_read.py
+python upper_american_calculate_rim_inflows.py
+
+:: Upper Mokelumne
+echo Running Upper Mokelumne
+python upper_mokelumne_data_read.py
+python upper_mokelumne_calculate_rim_inflows.py
+
+:: Generate Summary Tables and Figures
+echo Summarizing Rim Inflow Output
+python Summary_statistics.py
+pause