diff --git a/.gitignore b/.gitignore index b536ace..ea9e92d 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,5 @@ Intermediate/ Examples/ *.xlsm environmentExamples.yml +Outputs/SCurveStats.csv +Outputs/RimInflows_Summary*.csv diff --git a/README.md b/README.md index 0276340..25dea9c 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ All the Upper American related files are named with the 'upper_american_' prefix The *upper_american_2022_extension_data.csv* file contains the data used in the previous extension. Where possible, the gap filling was removed and moved to be done in the code. Places where the DWR COMP model was used, were not removed. -## To calculate the rim inflows +## Calculating Rim Inflows ### To create an environment @@ -27,7 +27,11 @@ The *upper_american_2022_extension_data.csv* file contains the data used in the `conda activate extension` -### To recreate the rim inflows with the data in the repository: +### To run the full rim inflow dataset calculations + +To run the full set of rim inflow scripts, run the run_rim_inflows.bat script. This batch file will run all rim inflows calculations for all locations within this respository as well as a summary script to caclutate summary metrics for all rim inflows. + +### To recreate the rim inflows for a basin with the data in the repository: Run `python upper_american_data_read.py` to read in the data and then `python upper_american_calculate_rim_inflows.py` to calculate the rim inflows. @@ -40,6 +44,11 @@ The calculated flows will be in the *Outputs* folder in *upper_american_rim_infl The calculated flows will be in the *Outputs* folder in *rim_inflows.csv*. +### To run summary statistics +Run `python Summary_statistics.py` to calculate summary statistics for each rim inflow location. *Note rim inflows must be calculated prior to running the summary script + +## Developing Rim Inflows + ### To incorporate additional locations Every location is different so the process to incorporate a new location is going to look different every time. For a new basin, create a new set of files named with the basin name. Generally, the following this must be added: 1. Any data from the previous extension should be added, in TAF, to *Inputs/upper_american_2022_extension_data.csv* diff --git a/Summary_statistics.py b/Summary_statistics.py new file mode 100644 index 0000000..23b85c8 --- /dev/null +++ b/Summary_statistics.py @@ -0,0 +1,72 @@ +import pandas as pd + +# ---------------------------------------------- +# --- INPUTS --- +# ---------------------------------------------- + +# Rim Inflow Basins (Names must match RimInflow Output files) +sl_rim_inflow_basins = ["upper_american","upper_mokelumne"] + +# Summary Period +i_final_year = 2021 +i_start_year = 1921 + +# ---------------------------------------------- +# --- FORMAT & OUTPUT S-CURVE SUMMARY TABLE --- +# ---------------------------------------------- + +# read in s_curve ouptut file +s_scurveStats_fn = 'Outputs/SCurveStats.csv' +df_scurve = pd.read_csv(s_scurveStats_fn) + +# remove the fit_slope and fit_intercept (these are duplicates, currently calculated in two functions of the rim inflow workflow, these values are exactly the same) +df_scurve = df_scurve[[not(s_stat in ['fit_slope','fit_intercept']) for s_stat in df_scurve.stat]] + +# pivot the table +df_scurve.drop_duplicates(inplace=True) +df_scurve_format = df_scurve.pivot(index = ['reference location','target location'],columns='stat',values='value') +df_scurve_format = df_scurve_format[['slope','intercept','r2']].reset_index() + +# To Do - add gage/lodation names, will require input table to cross reference gage #s and Names + +# save the output +df_scurve_format.round(3).to_csv('Outputs/RimInflows_Summary_SCurveParameters.csv',index=False) + +# ---------------------------------------------- +# --- CALCULATE & OUTPUT RIM INFLOW SUMMARY TABLE --- +# ---------------------------------------------- + +# read in final rim inflow output csv files from each basin and merge into a single dataframe +dl_basins = [pd.read_csv(f'Outputs/{s}_rim_inflows.csv',index_col=0) for s in sl_rim_inflow_basins] +df_rim_inflows = pd.concat(dl_basins,axis=1) +df_rim_inflows.index = pd.to_datetime(df_rim_inflows.index) + +# clip to selected period +# Question why does I_AMADR go to 2024-08-31?? +# Question why does the upper_mokelumne_rim_inflows.csv have a 1921-09-30 +df_rim_inflows = df_rim_inflows[df_rim_inflows.index < pd.to_datetime(str(i_final_year) + '-10-01')] +df_rim_inflows = df_rim_inflows[df_rim_inflows.index > pd.to_datetime(str(i_start_year) + '-09-30')] + +# calculate mean and median monthly values for each rim inflow +df_rim_inflows_monthlyTS = df_rim_inflows.resample('ME').sum() +df_rim_inflows_meanMon = df_rim_inflows_monthlyTS.groupby(df_rim_inflows_monthlyTS.index.month_name()).mean().T +df_rim_inflows_medianMon = df_rim_inflows_monthlyTS.groupby(df_rim_inflows_monthlyTS.index.month_name()).median().T + +# calculate mean and median total annual flow for each rim infow (calculated over water years) +df_rim_inflows['wy'] = df_rim_inflows.index.year.where(df_rim_inflows.index.month<10,df_rim_inflows.index.year+1) +df_rim_inflows_annualTS = df_rim_inflows.groupby('wy').sum() +df_rim_inflows_annualMetric = df_rim_inflows_annualTS.agg(["mean","median"]).T + +# join monthly and annual datasets +df_rim_inflows_summaryMedian = pd.concat([df_rim_inflows_medianMon[['October','November','December','January','February','March','April','May','June','July','August','September']], + df_rim_inflows_annualMetric],axis=1) +df_rim_inflows_summaryMean = pd.concat([df_rim_inflows_meanMon[['October','November','December','January','February','March','April','May','June','July','August','September']], + df_rim_inflows_annualMetric],axis=1) + +# output combined dataframes +df_rim_inflows_summaryMedian.to_csv('Outputs/RimInflows_Summary_MonthlyMedianandAnnualFlows.csv') +df_rim_inflows_summaryMean.to_csv('Outputs/RimInflows_Summary_MonthlyAvgandAnnualFlows.csv') + +# ouptut combined dataframes rounded to one decimal place for the final tables +df_rim_inflows_summaryMedian.round(1).to_csv('Outputs/RimInflows_Summary_MonthlyMedianandAnnualFlows_rounded.csv') +df_rim_inflows_summaryMean.round(1).to_csv('Outputs/RimInflows_Summary_MonthlyAvgandAnnualFlows_rounded.csv') diff --git a/extension_functions.py b/extension_functions.py index 3519d31..15309dd 100644 --- a/extension_functions.py +++ b/extension_functions.py @@ -12,7 +12,8 @@ def s_curve_disaggregation(df_x_data, df_y_data, i_x_start_year, i_x_end_year, i_y_start_year, - i_y_end_year, b_use_all_y=False, s_strange_sheet=''): + i_y_end_year, b_use_all_y=False, s_strange_sheet='',b_save_stats=False, + s_reference_name = '',s_target_name = '',s_out_stats = 'Outputs/SCurveStats.csv'): """ Takes in the x data and the y data and generated a full timeseries of synthetic y data. This is meant to replicate what the Excel/VBA does for the S-Curve disaggregation. @@ -35,6 +36,14 @@ def s_curve_disaggregation(df_x_data, df_y_data, i_x_start_year, i_x_end_year, i Whether to use all y data or just the section in the Y years s_strange_sheet: string For a few sheets with strange modifications to the s-curve procedure, this is the sheet name with capital letters. + b_save_stats: boolean + Option to save s-curve stats + s_reference_name: str + Name of reference location, only needed if saving s_curve stats to file, i.e. b_save_stats = True + s_target_name: str + Name of target location, only needed if saving s_curve stats to file, i.e. b_save_stats = True + s_out_stats: string + Output file to save s-curve stats if b_save_stats is true Returns ------- @@ -155,10 +164,21 @@ def s_curve_disaggregation(df_x_data, df_y_data, i_x_start_year, i_x_end_year, i else: df_y_data_output.loc[i_y_start_year:i_y_end_year, :] = df_y_data.loc[i_y_start_year:i_y_end_year, :] + # save stats if indicated + # add a line to the output csv with location,slope,intercept + if b_save_stats: + df_stats = pd.DataFrame( + {'reference location':[s_reference_name]*2, + 'target location':[s_target_name]*2, + 'stat':['slope','intercept'], + 'value':[d_slope,d_intercept]}) + df_stats.to_csv(s_out_stats,mode='a',index=False,header=False) + return df_y_data_output, df_y_data_synthetic -def s_curve_comparison_plots(df_final_y_dat, df_y_data_synthetic, df_x_data, df_y_data, s_current_location): +def s_curve_comparison_plots(df_final_y_dat, df_y_data_synthetic, df_x_data, df_y_data, s_current_location,s_reference_location, + b_save_stats=False,s_out_stats = 'Outputs/SCurveStats.csv'): """ Generates two plots to understand the quality of the generated data. First plot compares the historical y data and the reference x data. @@ -176,11 +196,20 @@ def s_curve_comparison_plots(df_final_y_dat, df_y_data_synthetic, df_x_data, df_ Original y data s_current_location: str Current location of the data + s_reference_location: str + Reference location name + b_save_stats: boolean + Option to save s-curve stats + s_out_stats: string + Output file to save s-curve stats if b_save_stats is true Returns ------- None """ + # save dataset names + + # first remove nans so they wont get plotted as zeros df_x_data.dropna(inplace=True) df_y_data.dropna(inplace=True) @@ -240,6 +269,15 @@ def s_curve_comparison_plots(df_final_y_dat, df_y_data_synthetic, df_x_data, df_ plt.savefig(f'./Figures/{s_current_location} Monthly Flows Observed vs Synthetic', bbox_inches='tight', dpi=300) plt.close() + # save stats if indicated + # add a line to the output csv with location,slope,intercept + if b_save_stats: + df_stats = pd.DataFrame( + {'reference location':[s_reference_location]*3, + 'target location':[s_current_location]*3, + 'stat':['fit_slope','fit_intercept','r2'], + 'value':[slope,intercept,r2]}) + df_stats.to_csv(s_out_stats,mode='a',index=False,header=False) def read_data(s_path): """ @@ -885,7 +923,7 @@ def read_previous_data(s_path, df_new_data): def extend_data(df_reference_data, df_current_data, df_extended_data, df_synthetic_data, i_y_start_year, i_y_end_year, b_use_all_y_data, s_name, i_x_start_year=1922, - i_final_year=2021, s_strange_sheet=''): + i_final_year=2021, s_strange_sheet='',b_save_stats=False): """ Extends data using the s-curve disaggregation. Also creates the plots and saves the data into dataframes. @@ -912,6 +950,9 @@ def extend_data(df_reference_data, df_current_data, df_extended_data, df_synthet Final year for the x data s_strange_sheet: string For a few sheets with strange modifications to the s-curve procedure, this is the sheet name with capital letters. + b_save_stats: boolean + Option to save stats of fitted s-curve + Returns ------- None @@ -921,11 +962,14 @@ def extend_data(df_reference_data, df_current_data, df_extended_data, df_synthet df_current_data, i_x_start_year, i_final_year, i_y_start_year, i_y_end_year, - b_use_all_y_data, s_strange_sheet) + b_use_all_y_data, s_strange_sheet, + s_target_name = s_name, + s_reference_name= df_reference_data.name, + b_save_stats=b_save_stats) # generate the comparison plots s_curve_comparison_plots(df_curr_final_data, df_curr_synthetic_data, timeseries_to_monthly(df_reference_data), timeseries_to_monthly(df_current_data), - s_name) + s_name,df_reference_data.name,b_save_stats=b_save_stats) # put the data into the two final dataframes df_extended_data[s_name] = monthly_to_timeseries(df_curr_final_data) @@ -935,7 +979,7 @@ def extend_data_multi_model(df_reference_data_1, df_current_data_1, df_reference df_extended_data, df_synthetic_data, i_y1_start_year, i_y1_end_year, i_y2_start_year, i_y2_end_year, b_use_all_y_data, s_name, s_model_name_1, s_model_name_2, i_x_start_year=1922, - i_final_year=2021, s_strange_sheet=''): + i_final_year=2021, s_strange_sheet='',b_save_stats=False): """ Extends data using the s-curve disaggregation and compares two different models. @@ -971,6 +1015,8 @@ def extend_data_multi_model(df_reference_data_1, df_current_data_1, df_reference Final year for the x data s_strange_sheet: string For a few sheets with strange modifications to the s-curve procedure, this is the sheet name with capital letters. + b_save_stats: boolean + Option to save stats of fitted s-curve Returns ------- None @@ -980,18 +1026,25 @@ def extend_data_multi_model(df_reference_data_1, df_current_data_1, df_reference df_current_data_1, i_x_start_year, i_final_year, i_y1_start_year, i_y1_end_year, - b_use_all_y_data, s_strange_sheet) + b_use_all_y_data, s_strange_sheet, + s_target_name = s_name + '_ref1', + s_reference_name= df_reference_data_1.name, + b_save_stats=b_save_stats) # do the s-curve disaggregation for model 2 df_curr_final_data_2, df_curr_synthetic_data_2 = s_curve_disaggregation(df_reference_data_2, df_current_data_2, i_x_start_year, i_final_year, i_y2_start_year, i_y2_end_year, - b_use_all_y_data, s_strange_sheet) + b_use_all_y_data, s_strange_sheet, + s_target_name = s_name + '_ref2', + s_reference_name = df_reference_data_2.name, + b_save_stats=b_save_stats) # generate the comparison plots two_s_curves_comparison_plots(df_curr_final_data_1, timeseries_to_monthly(df_reference_data_1), df_curr_final_data_2, timeseries_to_monthly(df_reference_data_2), s_name, - s_model_name_1, s_model_name_2) + s_model_name_1, s_model_name_2,s_ref_name_1 = df_reference_data_1.name, + s_ref_name_2 = df_reference_data_2.name,b_save_stats=b_save_stats) # put the data into the two final dataframes df_extended_data[s_name+s_model_name_1] = monthly_to_timeseries(df_curr_final_data_1) @@ -1261,7 +1314,9 @@ def read_replication_data(ls_sheet_info, df_before, df_after): df_after[sheet[0]] = monthly_to_timeseries(df_temp) def two_s_curves_comparison_plots(df_final_y_dat_1, df_x_data_1, - df_final_y_dat_2, df_x_data_2, s_current_location, s_model_name_1, s_model_name_2): + df_final_y_dat_2, df_x_data_2, s_current_location, s_model_name_1, s_model_name_2, + s_ref_name_1='',s_ref_name_2='', + s_out_stats = 'Outputs/SCurveStats.csv',b_save_stats=False): """ Generates a plot to compare model 1 and model 2 for a sheet. @@ -1281,6 +1336,14 @@ def two_s_curves_comparison_plots(df_final_y_dat_1, df_x_data_1, The name for model 1 s_model_name_2: str The name for model 2 + s_ref_name_1: str + Reference gage name for model 1 + s_ref_name_2: str + Reference gage name for model 2 + b_save_stats: boolean + Option to save s-curve stats + s_out_stats: string + Output file to save s-curve stats if b_save_stats is true Returns ------- None @@ -1336,3 +1399,13 @@ def two_s_curves_comparison_plots(df_final_y_dat_1, df_x_data_1, plt.savefig(f'./Figures/Model_Comparison/{s_current_location} Comparison of Two Models, {s_model_name_1} and {s_model_name_2}' , bbox_inches='tight', dpi=300) plt.close() + + # save stats if indicated + # add a line to the output csv with location,slope,intercept + if b_save_stats: + df_stats = pd.DataFrame( + {'reference location':[s_ref_name_1]*3+[s_ref_name_2]*3, + 'target location':[s_current_location + s_model_name_1]*3 + [s_current_location + s_model_name_2]*3, + 'stat':['fit_slope','fit_intercept','r2']*2, + 'value':[slope_1,intercept_1,r2_1,slope_2,intercept_2,r2_2]}) + df_stats.to_csv(s_out_stats,mode='a',index=False,header=False) diff --git a/run_rim_inflows.bat b/run_rim_inflows.bat new file mode 100644 index 0000000..d986e09 --- /dev/null +++ b/run_rim_inflows.bat @@ -0,0 +1,45 @@ +@echo off + +:: Start Python Environment +echo Starting Python Environment +call conda activate extension + +:: Remove previous output files series of steps to ensure the correct files are deleted +echo WARNING removing previous output files +pause + +set "TARGET_FOLDER=Outputs" +set "FULL_PATH=%~dp0%TARGET_FOLDER%" + +if "%TARGET_FOLDER%"=="" ( + echo ERROR: TARGET_FOLDER variable is empty! Aborting. + pause + exit /b +) +if not exist "%FULL_PATH%\" ( + echo ERROR: Target folder "%FULL_PATH%" does not exist! Aborting. + pause + exit /b +) + +del /f /q "%FULL_PATH%\*.*" +echo Done! All files deleted safely. +pause + + +echo Starting Rim Inflow Python script sequence + +:: Upper American +echo Running Upper American Module +python upper_american_data_read.py +python upper_american_calculate_rim_inflows.py + +:: Upper Mokelumne +echo Running Upper Mokelumne +python upper_mokelumne_data_read.py +python upper_mokelumne_calculate_rim_inflows.py + +:: Generate Summary Tables and Figures +echo Summarizing Rim Inflow Output +python Summary_statistics.py +pause diff --git a/upper_american_calculate_rim_inflows.py b/upper_american_calculate_rim_inflows.py index 85440e6..4811628 100644 --- a/upper_american_calculate_rim_inflows.py +++ b/upper_american_calculate_rim_inflows.py @@ -6,11 +6,16 @@ if __name__ == "__main__": i_final_year = 2021 + # option to save s-curve parameters + b_save_stats = True + if b_save_stats and not os.path.exists('Outputs/SCurveStats.csv'): # if s-curve parameter output is desired and the output file doesn't exist, create initial output file to write to + pd.DataFrame(columns=['reference location','target location','stat','value']).to_csv('Outputs/SCurveStats.csv',index=False) + # this holds the already extended evap rates s_evap_dss_path = r".\Inputs\evaporation_rates.dss" # option to plot comparison - b_compareData = True + b_compareData = False s_prev_rim_inflows_fn = "CS3_Sac_ReadAllInflowDatatoDSS_05.18.23.xlsm" # file path and name must be provided to plot/calculate comparison s_prev_rim_inflow_sheet = "Inflows" @@ -98,48 +103,48 @@ print("Extending flows...") # extend all with the s-curve disaggregation - extend_data(df_full_data['AMF'], df_pos_unimpaired_data['11439501'], df_extended_data, df_synthetic_data, 1923, i_final_year, False, '11439501', i_final_year=i_final_year) - extend_data(df_extended_data['11439501'], df_full_data['11427700'], df_extended_data, df_synthetic_data, 1961, i_final_year, False, '11427700', i_final_year=i_final_year) - extend_data(df_extended_data['11439501'], df_pos_unimpaired_data['11427500'], df_extended_data, df_synthetic_data, 1966, 2007,True, '11427500', i_final_year=i_final_year) - extend_data(df_extended_data['11439501'], df_pos_unimpaired_data['11427760'], df_extended_data, df_synthetic_data, 1966, 2007, False, '11427760', i_final_year=i_final_year) - extend_data(df_extended_data['11439501'], df_pos_unimpaired_data['11428000'], df_extended_data, df_synthetic_data, 1957, 1986, False, '11428000', i_final_year=i_final_year) - extend_data(df_extended_data['11439501'], df_unimpaired_data['11428400'], df_extended_data, df_synthetic_data, 1991, 2015, False, '11428400', i_final_year=i_final_year) - extend_data(df_extended_data['11439501'], df_pos_unimpaired_data['11428800'], df_extended_data, df_synthetic_data, 1966, 2007, True, '11428800', i_final_year=i_final_year) - extend_data(df_extended_data['11439501'], df_pos_unimpaired_data['11429500'], df_extended_data, df_synthetic_data, 1963, i_final_year, False, '11429500', i_final_year=i_final_year) - extend_data(df_extended_data['11439501'], df_unimpaired_data['11430000'], df_extended_data, df_synthetic_data, 1963, 2021, False, '11430000', i_final_year=i_final_year) - extend_data(df_extended_data['11439501'], df_unimpaired_data['11433040'], df_extended_data, df_synthetic_data, 1962, 2017, True, '11433040', i_final_year=i_final_year) - extend_data(df_extended_data['11439501'], df_unimpaired_data['11433100'], df_extended_data, df_synthetic_data, 1967, 1992, False, '11433100', i_final_year=i_final_year) - extend_data(df_full_data['AMF'], df_unimpaired_data['11433100'], df_extended_data, df_synthetic_data, 1967, 1992, False, '11433100_AMF', i_final_year=i_final_year) - extend_data(df_extended_data['11439501'], df_full_data['11433260'], df_extended_data, df_synthetic_data, 1966, 1985, False, '11433260', i_final_year=i_final_year) - extend_data(df_full_data['AMF'], df_pos_unimpaired_data['11433300'], df_extended_data, df_synthetic_data, 1959, i_final_year, False, '11433300', i_final_year=i_final_year) + extend_data(df_full_data['AMF'], df_pos_unimpaired_data['11439501'], df_extended_data, df_synthetic_data, 1923, i_final_year, False, '11439501', i_final_year=i_final_year,b_save_stats=b_save_stats) + extend_data(df_extended_data['11439501'], df_full_data['11427700'], df_extended_data, df_synthetic_data, 1961, i_final_year, False, '11427700', i_final_year=i_final_year,b_save_stats=b_save_stats) + extend_data(df_extended_data['11439501'], df_pos_unimpaired_data['11427500'], df_extended_data, df_synthetic_data, 1966, 2007,True, '11427500', i_final_year=i_final_year,b_save_stats=b_save_stats) + extend_data(df_extended_data['11439501'], df_pos_unimpaired_data['11427760'], df_extended_data, df_synthetic_data, 1966, 2007, False, '11427760', i_final_year=i_final_year,b_save_stats=b_save_stats) + extend_data(df_extended_data['11439501'], df_pos_unimpaired_data['11428000'], df_extended_data, df_synthetic_data, 1957, 1986, False, '11428000', i_final_year=i_final_year,b_save_stats=b_save_stats) + extend_data(df_extended_data['11439501'], df_unimpaired_data['11428400'], df_extended_data, df_synthetic_data, 1991, 2015, False, '11428400', i_final_year=i_final_year,b_save_stats=b_save_stats) + extend_data(df_extended_data['11439501'], df_pos_unimpaired_data['11428800'], df_extended_data, df_synthetic_data, 1966, 2007, True, '11428800', i_final_year=i_final_year,b_save_stats=b_save_stats) + extend_data(df_extended_data['11439501'], df_pos_unimpaired_data['11429500'], df_extended_data, df_synthetic_data, 1963, i_final_year, False, '11429500', i_final_year=i_final_year,b_save_stats=b_save_stats) + extend_data(df_extended_data['11439501'], df_unimpaired_data['11430000'], df_extended_data, df_synthetic_data, 1963, 2021, False, '11430000', i_final_year=i_final_year,b_save_stats=b_save_stats) + extend_data(df_extended_data['11439501'], df_unimpaired_data['11433040'], df_extended_data, df_synthetic_data, 1962, 2017, True, '11433040', i_final_year=i_final_year,b_save_stats=b_save_stats) + extend_data(df_extended_data['11439501'], df_unimpaired_data['11433100'], df_extended_data, df_synthetic_data, 1967, 1992, False, '11433100', i_final_year=i_final_year,b_save_stats=b_save_stats) + extend_data(df_full_data['AMF'], df_unimpaired_data['11433100'], df_extended_data, df_synthetic_data, 1967, 1992, False, '11433100_AMF', i_final_year=i_final_year,b_save_stats=b_save_stats) + extend_data(df_extended_data['11439501'], df_full_data['11433260'], df_extended_data, df_synthetic_data, 1966, 1985, False, '11433260', i_final_year=i_final_year,b_save_stats=b_save_stats) + extend_data(df_full_data['AMF'], df_pos_unimpaired_data['11433300'], df_extended_data, df_synthetic_data, 1959, i_final_year, False, '11433300', i_final_year=i_final_year,b_save_stats=b_save_stats) df_extended_data['11433500'] = flow_from_two_unimp(df_unimpaired_data['11433500'], df_unimpaired_data['11433300'], 1.06) - extend_data(df_full_data['AMF'], df_unimpaired_data['11435100'][~df_unimpaired_data.index.isin(pd.date_range(datetime(2002, 10, 31), datetime(2012, 9, 30)))], df_extended_data, df_synthetic_data, 1971, 2002, True, '11435100_C', i_final_year=i_final_year) - extend_data(df_full_data['AMF'], df_unimpaired_data['11435100'].loc[datetime(2011, 9, 30):], df_extended_data, df_synthetic_data, 2012, 2021, False, '11435100_A', i_final_year=i_final_year) - extend_data(df_extended_data['11439501'], df_unimpaired_data['11435100'].loc[datetime(2011, 9, 30):], df_extended_data, df_synthetic_data, 2012, 2021, False, '11435100_B', i_final_year=i_final_year) - extend_data(df_full_data['AMF'], df_pos_unimpaired_data['11437000'], df_extended_data, df_synthetic_data, 1923, 1992, True, '11437000_A', i_final_year=i_final_year) - extend_data(df_extended_data['11439501'], df_pos_unimpaired_data['11437000'], df_extended_data, df_synthetic_data, 1923, 1992, False, '11437000_B', i_final_year=i_final_year) - extend_data(df_extended_data['11439501'], df_pos_unimpaired_data['11436000'], df_extended_data, df_synthetic_data, 1923, i_final_year, False, '11436000', i_final_year=i_final_year) - extend_data(df_full_data['AMF'], df_full_data['11440000'], df_extended_data, df_synthetic_data, 1923, 1981, False, '11440000_A', i_final_year=i_final_year) - extend_data(df_extended_data['11439501'], df_full_data['11440000'], df_extended_data, df_synthetic_data, 1923, 1981, False, '11440000_B', i_final_year=i_final_year) - extend_data(df_full_data['AMF'], df_full_data['11440500'], df_extended_data, df_synthetic_data, 1923, 1939, False, '11440500_A', i_final_year=i_final_year) - extend_data(df_extended_data['11439501'], df_full_data['11440500'], df_extended_data, df_synthetic_data, 1923, 1939, False, '11440500_B', i_final_year=i_final_year) - extend_data(df_extended_data['11439501'], monthly_to_timeseries(timeseries_to_monthly(df_unimpaired_data['11441000']).dropna(how='any', axis=0).drop(1962, axis=0))['TAF'], df_extended_data, df_synthetic_data, 1925, 1960, True, '11441000', i_final_year=i_final_year) + extend_data(df_full_data['AMF'], df_unimpaired_data['11435100'][~df_unimpaired_data.index.isin(pd.date_range(datetime(2002, 10, 31), datetime(2012, 9, 30)))], df_extended_data, df_synthetic_data, 1971, 2002, True, '11435100_C', i_final_year=i_final_year,b_save_stats=b_save_stats) + extend_data(df_full_data['AMF'], df_unimpaired_data['11435100'].loc[datetime(2011, 9, 30):], df_extended_data, df_synthetic_data, 2012, 2021, False, '11435100_A', i_final_year=i_final_year,b_save_stats=b_save_stats) + extend_data(df_extended_data['11439501'], df_unimpaired_data['11435100'].loc[datetime(2011, 9, 30):], df_extended_data, df_synthetic_data, 2012, 2021, False, '11435100_B', i_final_year=i_final_year,b_save_stats=b_save_stats) + extend_data(df_full_data['AMF'], df_pos_unimpaired_data['11437000'], df_extended_data, df_synthetic_data, 1923, 1992, True, '11437000_A', i_final_year=i_final_year,b_save_stats=b_save_stats) + extend_data(df_extended_data['11439501'], df_pos_unimpaired_data['11437000'], df_extended_data, df_synthetic_data, 1923, 1992, False, '11437000_B', i_final_year=i_final_year,b_save_stats=b_save_stats) + extend_data(df_extended_data['11439501'], df_pos_unimpaired_data['11436000'], df_extended_data, df_synthetic_data, 1923, i_final_year, False, '11436000', i_final_year=i_final_year,b_save_stats=b_save_stats) + extend_data(df_full_data['AMF'], df_full_data['11440000'], df_extended_data, df_synthetic_data, 1923, 1981, False, '11440000_A', i_final_year=i_final_year,b_save_stats=b_save_stats) + extend_data(df_extended_data['11439501'], df_full_data['11440000'], df_extended_data, df_synthetic_data, 1923, 1981, False, '11440000_B', i_final_year=i_final_year,b_save_stats=b_save_stats) + extend_data(df_full_data['AMF'], df_full_data['11440500'], df_extended_data, df_synthetic_data, 1923, 1939, False, '11440500_A', i_final_year=i_final_year,b_save_stats=b_save_stats) + extend_data(df_extended_data['11439501'], df_full_data['11440500'], df_extended_data, df_synthetic_data, 1923, 1939, False, '11440500_B', i_final_year=i_final_year,b_save_stats=b_save_stats) + extend_data(df_extended_data['11439501'], monthly_to_timeseries(timeseries_to_monthly(df_unimpaired_data['11441000']).dropna(how='any', axis=0).drop(1962, axis=0))['TAF'], df_extended_data, df_synthetic_data, 1925, 1960, True, '11441000', i_final_year=i_final_year,b_save_stats=b_save_stats) # wy 1981 and 1994 use the synthetic values df_extended_data.loc[pd.date_range(datetime(1980, 10, 31), datetime(1981, 9, 30), freq='ME'), '11441000'] = df_synthetic_data.loc[pd.date_range(datetime(1980, 10, 31), datetime(1981, 9, 30), freq='ME'), '11441000'].values df_extended_data.loc[pd.date_range(datetime(1993, 10, 31), datetime(1994, 9, 30), freq='ME'), '11441000'] = df_synthetic_data.loc[pd.date_range(datetime(1993, 10, 31), datetime(1994, 9, 30), freq='ME'), '11441000'].values - extend_data(df_full_data['11442000'], df_pos_unimpaired_data['11441500'], df_extended_data, df_synthetic_data, 1925, i_final_year, True, '11441500', i_final_year=1961) + extend_data(df_full_data['11442000'], df_pos_unimpaired_data['11441500'], df_extended_data, df_synthetic_data, 1925, i_final_year, True, '11441500', i_final_year=1961,b_save_stats=b_save_stats) # wy 1962 and on are blank, fill with the original values df_extended_data.fillna({'11441500': df_pos_unimpaired_data['11441500']}, inplace=True) - extend_data(df_extended_data['11439501'], df_full_data['11442000'], df_extended_data, df_synthetic_data, 1923, 1961, False, '11442000', i_final_year=i_final_year) - extend_data(df_full_data['AMF'], df_pos_unimpaired_data['11443500'], df_extended_data, df_synthetic_data, 1974, 2021, True, '11443500_A', i_final_year=i_final_year) + extend_data(df_extended_data['11439501'], df_full_data['11442000'], df_extended_data, df_synthetic_data, 1923, 1961, False, '11442000', i_final_year=i_final_year,b_save_stats=b_save_stats) + extend_data(df_full_data['AMF'], df_pos_unimpaired_data['11443500'], df_extended_data, df_synthetic_data, 1974, 2021, True, '11443500_A', i_final_year=i_final_year,b_save_stats=b_save_stats) df_extended_data.loc[df_pos_unimpaired_data.loc[:datetime(2021,9,30), '11443500'].dropna().index, '11443500_A'] = df_pos_unimpaired_data.loc[:datetime(2021,9,30), '11443500'].dropna() - extend_data(df_pos_unimpaired_data['11444500'], df_pos_unimpaired_data['11443500'], df_extended_data, df_synthetic_data, 1974, 2021, True, '11443500_D', i_x_start_year=1965, i_final_year=i_final_year) + extend_data(df_pos_unimpaired_data['11444500'], df_pos_unimpaired_data['11443500'], df_extended_data, df_synthetic_data, 1974, 2021, True, '11443500_D', i_x_start_year=1965, i_final_year=i_final_year,b_save_stats=b_save_stats) df_extended_data.fillna({'11443500_D': df_pos_unimpaired_data['11443500']}, inplace=True) - extend_data(df_full_data['AMF'], df_unimpaired_data['11444201'], df_extended_data, df_synthetic_data, 1987, 2008, True, '11444201', i_final_year=i_final_year) + extend_data(df_full_data['AMF'], df_unimpaired_data['11444201'], df_extended_data, df_synthetic_data, 1987, 2008, True, '11444201', i_final_year=i_final_year,b_save_stats=b_save_stats) # replace the end of wy 2017 df_extended_data.loc[datetime(2016, 11, 30): datetime(2017, 9, 30), '11444201'] = df_unimpaired_data.loc[datetime(2016, 11, 30): datetime(2017, 9, 30), '11444201'] - extend_data(df_full_data['AMF'], df_pos_unimpaired_data['11444500'], df_extended_data, df_synthetic_data, 1965, i_final_year, False, '11444500', i_final_year=i_final_year) - extend_data(df_full_data['AMF'], df_full_data['11446000'], df_extended_data, df_synthetic_data, 1944, 1959, False, '11446000', i_final_year=i_final_year) + extend_data(df_full_data['AMF'], df_pos_unimpaired_data['11444500'], df_extended_data, df_synthetic_data, 1965, i_final_year, False, '11444500', i_final_year=i_final_year,b_save_stats=b_save_stats) + extend_data(df_full_data['AMF'], df_full_data['11446000'], df_extended_data, df_synthetic_data, 1944, 1959, False, '11446000', i_final_year=i_final_year,b_save_stats=b_save_stats) # save to csv df_extended_data.to_csv('./Intermediate/upper_american_extended_data.csv') diff --git a/upper_mokelumne_calculate_rim_inflows.py b/upper_mokelumne_calculate_rim_inflows.py index 31f1720..17a4f56 100644 --- a/upper_mokelumne_calculate_rim_inflows.py +++ b/upper_mokelumne_calculate_rim_inflows.py @@ -14,6 +14,9 @@ # option to plot comparison b_compare_data = True + # option to save s-curve dissagregation data (slope, intercept, r2) + b_save_stats = True + # option to run each element using "upstream" (antecedent) SV INPUT values from sheets rather than from the values # calculated within the Python code. b_use_upstream_sv_inputs = True @@ -43,6 +46,12 @@ # path for some extra sv inputs s_prev_rim_inflows_extra = r".\Inputs\upper_mokelumne_2022_sv_inputs.csv" + + # if s-curve parameter output is desired, create initial output file to write to + if b_save_stats and not os.path.exists('Outputs/SCurveStats.csv'): + pd.DataFrame(columns=['location','stat','value']).to_csv('Outputs/SCurveStats.csv') + + # first if the needed output folders don't exist, create them os.makedirs('./Intermediate', exist_ok=True) os.makedirs('./Figures', exist_ok=True) @@ -65,22 +74,22 @@ # create a list of lists. inner elements are ['column_name', 'path to before csv', 'path to after csv'] ls_sheet_info = [['COL003', './Inputs/s_curve_replication/col003_input_to_s_curve.csv', - './Inputs/s_curve_replication/col003_output_from_s_curve.csv'], - ['SLTSP', './Inputs/s_curve_replication/sltsp_input_to_s_curve.csv', - './Inputs/s_curve_replication/sltsp_output_from_s_curve.csv'], - ['UBEAR', './Inputs/s_curve_replication/ubear_input_to_s_curve.csv', - './Inputs/s_curve_replication/ubear_output_from_s_curve.csv'], - ['NFM010', './Inputs/s_curve_replication/nfm010_input_to_s_curve.csv', - './Inputs/s_curve_replication/nfm010_output_from_s_curve.csv'], - ['TGC003', './Inputs/s_curve_replication/nfm010_input_to_s_curve.csv', - './Inputs/s_curve_replication/tgc003_output_from_s_curve.csv'], - ['CMP001', './Inputs/s_curve_replication/cmp001_input_to_s_curve.csv', - './Inputs/s_curve_replication/cmp001_output_from_s_curve.csv'], - ['CMP014', './Inputs/s_curve_replication/cmp014_input_to_s_curve.csv', - './Inputs/s_curve_replication/cmp014_output_from_s_curve.csv'], - ['DEE023', './Inputs/s_curve_replication/dee023_input_to_s_curve.csv', - './Inputs/s_curve_replication/dee023_output_from_s_curve.csv'] - ] + './Inputs/s_curve_replication/col003_output_from_s_curve.csv'], + ['SLTSP', './Inputs/s_curve_replication/sltsp_input_to_s_curve.csv', + './Inputs/s_curve_replication/sltsp_output_from_s_curve.csv'], + ['UBEAR', './Inputs/s_curve_replication/ubear_input_to_s_curve.csv', + './Inputs/s_curve_replication/ubear_output_from_s_curve.csv'], + ['NFM010', './Inputs/s_curve_replication/nfm010_input_to_s_curve.csv', + './Inputs/s_curve_replication/nfm010_output_from_s_curve.csv'], + ['TGC003', './Inputs/s_curve_replication/nfm010_input_to_s_curve.csv', + './Inputs/s_curve_replication/tgc003_output_from_s_curve.csv'], + ['CMP001', './Inputs/s_curve_replication/cmp001_input_to_s_curve.csv', + './Inputs/s_curve_replication/cmp001_output_from_s_curve.csv'], + ['CMP014', './Inputs/s_curve_replication/cmp014_input_to_s_curve.csv', + './Inputs/s_curve_replication/cmp014_output_from_s_curve.csv'], + ['DEE023', './Inputs/s_curve_replication/dee023_input_to_s_curve.csv', + './Inputs/s_curve_replication/dee023_output_from_s_curve.csv'] + ] # create the dataframes where we keep the before and after data df_before_s = pd.DataFrame() df_after_s = pd.DataFrame() @@ -92,7 +101,7 @@ # ---------------------------------- # in CMP001, fill the JNKSN_STORAGE in December 1965 with linear interpolation of the adjacent months. df_full_data.loc['1965-12-31', 'JNKSN_STORAGE'] = (df_full_data.loc['1965-11-30', 'JNKSN_STORAGE'] - + df_full_data.loc['1966-01-31', 'JNKSN_STORAGE']) / 2 + + df_full_data.loc['1966-01-31', 'JNKSN_STORAGE']) / 2 # for JNKSN, create a copy of data with dropped WY1955 df_full_data.rename(columns={'11332500': '11332500_v1'}, inplace=True) df_full_data["11332500_v2"] = df_full_data["11332500_v1"].copy() @@ -201,11 +210,11 @@ if b_replicate_sheets: print("Checking inputs to s-curve, part 1...") compare_two_df(df_full_data['11317000'].drop(df_full_data['11317000'].index[0]).round(2), df_sv_inputs['I_MFM008'], '11317000', - 'SV_INPUT_MFM008') # for MFM008 + 'SV_INPUT_MFM008') # for MFM008 compare_two_df(df_unimpaired_data['11335000_v1'], df_before_s['CMP001'], '11335000_v1', - 'before_s_CMP001') # for CMP001 + 'before_s_CMP001') # for CMP001 compare_two_df(df_unimpaired_data['11335000_v2'], df_before_s['CMP014'], '11335000_v2', - 'before_s_CMP014') # for CMP014 + 'before_s_CMP014') # for CMP014 @@ -213,26 +222,27 @@ # extend with the s-curve disaggregation, round 1 extend_data(df_full_data['11317000'], df_full_data['11318500'], df_extended_data, df_synthetic_data, 1934, i_final_year, False, - '11318500', i_final_year=i_final_year) # see SFM005 + '11318500', i_final_year=i_final_year, b_save_stats=False) # see SFM005 # for JNKSN, s-curve extend_data(df_unimpaired_data['11335000_v2'], df_full_data['11332500_v2'], df_extended_data, df_synthetic_data, 1947, 1954, False, - '11332500', i_final_year=i_final_year) # see JNKSN + '11332500', i_final_year=i_final_year,b_save_stats=b_save_stats) # see JNKSN extend_data(df_unimpaired_data['11335000_v1'], df_unimpaired_data['11333000'], df_extended_data, df_synthetic_data, 1956, 2004, False, - '11333000', i_final_year=i_final_year) # see CMP001 + '11333000', i_final_year=i_final_year,b_save_stats=b_save_stats) # see CMP001 extend_data(df_unimpaired_data['11335000_v2'], df_full_data['11331500'], df_extended_data, df_synthetic_data, 1949, 1954, False, - '11331500', i_final_year=i_final_year) # see CMP014 + '11331500', i_final_year=i_final_year,b_save_stats=b_save_stats) # see CMP014 extend_data(df_unimpaired_data['11335000_v2'], df_full_data['11326300'], df_extended_data, df_synthetic_data, 1961, 1970, False, - '11326300', i_final_year=i_final_year) # see DSC035 + '11326300', i_final_year=i_final_year,b_save_stats=b_save_stats) # see DSC035 extend_data(df_unimpaired_data['11335000_v2'], df_full_data['11327000_v2'], df_extended_data, df_synthetic_data, 1961, 1980, False, - '11327000_v2', i_final_year=i_final_year) # see DSC035 + '11327000_v2', i_final_year=i_final_year,b_save_stats=b_save_stats) # see DSC035 extend_data(df_unimpaired_data['11335000_v2'], df_full_data['11335700'], df_extended_data, df_synthetic_data, 1961, 1977, False, - '11335700', i_final_year=i_final_year, s_strange_sheet='DEE023') # see DEE023 + '11335700', i_final_year=i_final_year, s_strange_sheet='DEE023', + b_save_stats=b_save_stats) # see DEE023 # ----------------------------- # --- UNIMPAIRMENT, ROUND 2 --- @@ -265,19 +275,19 @@ df_full_data['11318500'], df_extended_data, df_synthetic_data, 1934, 2021, 1934, 2021, False, '11318500', "Model A", "Model B", i_x_start_year=1922, - i_final_year=2021, s_strange_sheet='') # SFM005 A/B + i_final_year=2021, s_strange_sheet='',b_save_stats=b_save_stats) # SFM005 A/B extend_data(df_unimpaired_data['11319500_v1'], df_full_data['11315000'], df_extended_data, df_synthetic_data, 1928, i_final_year, False, - '11315000', i_x_start_year=1922, i_final_year=i_final_year, s_strange_sheet='COL003') # see COL003 + '11315000', i_x_start_year=1922, i_final_year=i_final_year, s_strange_sheet='COL003',b_save_stats=b_save_stats) # see COL003 extend_data(df_unimpaired_data['11319500_v1'], df_unimpaired_data['LBearSS_v1'], df_extended_data, df_synthetic_data, 1989, i_final_year, False, - 'LBearSS_v1', i_final_year=i_final_year) # see SLTSP + 'LBearSS_v1', i_final_year=i_final_year,b_save_stats=b_save_stats) # see SLTSP extend_data(df_unimpaired_data['11319500_v1'], df_unimpaired_data['LBearSS_v2'], df_extended_data, df_synthetic_data, 1989, i_final_year, False, - 'LBearSS_v2', i_final_year=i_final_year) # see UBEAR + 'LBearSS_v2', i_final_year=i_final_year,b_save_stats=b_save_stats) # see UBEAR extend_data(df_unimpaired_data['11319500_v1'], df_unimpaired_data['11316600'], df_extended_data, df_synthetic_data, 1986, i_y_end_year=2001, - b_use_all_y_data=False, s_name='11316600', i_final_year=i_final_year) # see NFM010 + b_use_all_y_data=False, s_name='11316600', i_final_year=i_final_year,b_save_stats=b_save_stats) # see NFM010 # ------------------------------- # --- FILL DATA AFTER S-CURVE ---