| @@ -150,8 +150,8 @@ |
| return [ |
| 'state', \ |
| 'county', \ |
| - 'latitude', \ |
| - 'longitude', \ |
| + 'Latitude', \ |
| + 'Longitude', \ |
| 'multipolygon_key', \ |
| 'WtAvgWaterSoilLoss', \ |
| 'WtAvgWindSoilLoss', \ |
| @@ -258,9 +258,6 @@ |
| if not(resq_erosion.is_finished()): |
| e = resq_erosion.get_error() |
| logging.warning(f"{service_name} | Attempt #{attempt} failed: {e} |{latitude}|{longitude}|{multipolygon}") |
| -# increment_counter(counter) |
| -# if counter.value > acceptable_failure_num: |
| -# raise MaxFailuresException() |
| resq_erosion = erosion_service_call(latitude, longitude, multipolygon, attempt+1) |
| else: |
| logging.info(f"{service_name} | Attempt #{attempt} succeded: {resq_erosion.data} |{latitude}|{longitude}|{multipolygon}") |
| @@ -278,77 +275,77 @@ |
| resq_Centroid, multipolygon = centroid_service_call(latitude, longitude) |
| # logging.info(resq_Centroid.data) |
| |
| -# resq_erosion = erosion_service_call(multipolygon) |
| - #logging.info(resq_erosion) |
| + resq_erosion = erosion_service_call(multipolygon) |
| + logging.info(resq_erosion) |
| |
| -# if resq_erosion is None: |
| -# raise TypeError() |
| + if resq_erosion is None: |
| + raise TypeError() |
| |
| -# c_res = Results(idx, latitude, longitude) |
| -# c_res.add_mupolygonkey(multipolygon) |
| -# c_res.add_state(resq_erosion.data['state']['value']) |
| -# c_res.add_county(resq_erosion.data['county']['value']) |
| + c_res = Results(idx, latitude, longitude) |
| + c_res.add_mupolygonkey(multipolygon) |
| + c_res.add_state(resq_erosion.data['state']['value']) |
| + c_res.add_county(resq_erosion.data['county']['value']) |
| |
| -# jl_all = [ s for s in resq_erosion.data['detailed_output']['value'] if s[0]['value'] == True ] |
| + jl_all = [ s for s in resq_erosion.data['detailed_output']['value'] if s[0]['value'] == True ] |
| |
| -# if len(jl_all) > 1: |
| -# ## check which is the main component |
| -# max_comppct_r = 0 |
| -# for j in jl_all: |
| -# c_comppct_r = float(extract_json_key_value(j, key='comppct_r')) |
| -# if c_comppct_r > max_comppct_r: |
| -# max_comppct_r = c_comppct_r |
| -# jl = j |
| -# else: |
| -# jl = jl_all[0] |
| + if len(jl_all) > 1: |
| + ## check which is the main component |
| + max_comppct_r = 0 |
| + for j in jl_all: |
| + c_comppct_r = float(extract_json_key_value(j, key='comppct_r')) |
| + if c_comppct_r > max_comppct_r: |
| + max_comppct_r = c_comppct_r |
| + jl = j |
| + else: |
| + jl = jl_all[0] |
| |
| -# # cokey |
| -# c_res.add_cokey(extract_json_key_value(jl, key='cokey')) |
| -# # compname |
| -# c_res.add_compname(extract_json_key_value(jl, key='compname')) |
| -# # claytotal_r |
| -# c_res.add_wepp_clay(extract_json_key_value(jl, key='sol_wepp_surface_clay')) |
| -# c_res.add_weps_clay(extract_json_key_value(jl, key='ifc_weps_surface_clay')) |
| -# # sandtotal_r |
| -# c_res.add_wepp_sand(extract_json_key_value(jl, key='sol_wepp_surface_sand')) |
| -# c_res.add_weps_sand(extract_json_key_value(jl, key='ifc_weps_surface_sand')) |
| -# # silttotal_r |
| -# c_res.add_weps_silt(extract_json_key_value(jl, key='ifc_weps_surface_silt')) |
| -# # wilting point |
| -# c_res.add_weps_wp(extract_json_key_value(jl, key='ifc_weps_surface_wilting_point')) |
| + # cokey |
| + c_res.add_cokey(extract_json_key_value(jl, key='cokey')) |
| + # compname |
| + c_res.add_compname(extract_json_key_value(jl, key='compname')) |
| + # claytotal_r |
| + c_res.add_wepp_clay(extract_json_key_value(jl, key='sol_wepp_surface_clay')) |
| + c_res.add_weps_clay(extract_json_key_value(jl, key='ifc_weps_surface_clay')) |
| + # sandtotal_r |
| + c_res.add_wepp_sand(extract_json_key_value(jl, key='sol_wepp_surface_sand')) |
| + c_res.add_weps_sand(extract_json_key_value(jl, key='ifc_weps_surface_sand')) |
| + # silttotal_r |
| + c_res.add_weps_silt(extract_json_key_value(jl, key='ifc_weps_surface_silt')) |
| + # wilting point |
| + c_res.add_weps_wp(extract_json_key_value(jl, key='ifc_weps_surface_wilting_point')) |
| |
| -# c_res.add_WtAvgWaterSoilLoss(resq_erosion.data["WtAvgWaterSoilLoss"]['value']) |
| -# c_res.add_WtAvgWindSoilLoss(resq_erosion.data["WtAvgWindSoilLoss"]['value']) |
| -# c_res.add_WtAvgTotalSoilLoss(resq_erosion.data["WtAvgTotalSoilLoss"]['value']) |
| -# c_res.add_WtAvgWaterEI(resq_erosion.data["WtAvgWaterEI"]['value']) |
| -# c_res.add_WtAvgWindEI(resq_erosion.data["WtAvgWindEI"]['value']) |
| + c_res.add_WtAvgWaterSoilLoss(resq_erosion.data["WtAvgWaterSoilLoss"]['value']) |
| + c_res.add_WtAvgWindSoilLoss(resq_erosion.data["WtAvgWindSoilLoss"]['value']) |
| + c_res.add_WtAvgTotalSoilLoss(resq_erosion.data["WtAvgTotalSoilLoss"]['value']) |
| + c_res.add_WtAvgWaterEI(resq_erosion.data["WtAvgWaterEI"]['value']) |
| + c_res.add_WtAvgWindEI(resq_erosion.data["WtAvgWindEI"]['value']) |
| |
| -# legacy_arr = resq_erosion.data["legacy_data_by_mapunit"]['value'][0] |
| + legacy_arr = resq_erosion.data["legacy_data_by_mapunit"]['value'][0] |
| |
| -# soil_slope = [el['value'] for el in legacy_arr if el['name']=='soil_slp_lgth_fctr'][0] |
| -# soil_loss = [el['value'] for el in legacy_arr if el['name']=='soil_loss_tolr_fctr'][0] |
| -# c_res.add_soilSlpLgthFctr(soil_slope) |
| -# c_res.add_soilLossTolrFctr(soil_loss) |
| + soil_slope = [el['value'] for el in legacy_arr if el['name']=='soil_slp_lgth_fctr'][0] |
| + soil_loss = [el['value'] for el in legacy_arr if el['name']=='soil_loss_tolr_fctr'][0] |
| + c_res.add_soilSlpLgthFctr(soil_slope) |
| + c_res.add_soilLossTolrFctr(soil_loss) |
| |
| -# legacy_r_factor = [el['value'] for el in legacy_arr if el['name']=='legacy_r_factor'][0] |
| -# legacy_waterEI = [el['value'] for el in legacy_arr if el['name']=='wtr_erod_fctr'][0] |
| -# c_res.add_legacyRfactor(legacy_r_factor) |
| -# c_res.add_wtrErodFctr(legacy_waterEI) |
| -# water_erosion_value = [el['value'] for el in legacy_arr if el['name']=='water_erosion'][0] |
| -# c_res.add_legacyWtAvgWaterEI(water_erosion_value) |
| -# c_res.eval_legacyWaterErosionValue() |
| + legacy_r_factor = [el['value'] for el in legacy_arr if el['name']=='legacy_r_factor'][0] |
| + legacy_waterEI = [el['value'] for el in legacy_arr if el['name']=='wtr_erod_fctr'][0] |
| + c_res.add_legacyRfactor(legacy_r_factor) |
| + c_res.add_wtrErodFctr(legacy_waterEI) |
| + water_erosion_value = [el['value'] for el in legacy_arr if el['name']=='water_erosion'][0] |
| + c_res.add_legacyWtAvgWaterEI(water_erosion_value) |
| + c_res.eval_legacyWaterErosionValue() |
| |
| -# legacy_c_factor = [el['value'] for el in legacy_arr if el['name']=='legacy_c_factor'][0] |
| -# legacy_windEI = [el['value'] for el in legacy_arr if el['name']=='wind_erod_idx'][0] |
| -# c_res.add_legacyCfactor(legacy_c_factor) |
| -# c_res.add_windErodIndex(legacy_windEI) |
| -# wind_erosion_value = [el['value'] for el in legacy_arr if el['name']=='wind_erosion'][0] |
| -# c_res.add_legacyWtAvgWindEI(wind_erosion_value) |
| -# c_res.eval_legacyWindErosionValue() |
| + legacy_c_factor = [el['value'] for el in legacy_arr if el['name']=='legacy_c_factor'][0] |
| + legacy_windEI = [el['value'] for el in legacy_arr if el['name']=='wind_erod_idx'][0] |
| + c_res.add_legacyCfactor(legacy_c_factor) |
| + c_res.add_windErodIndex(legacy_windEI) |
| + wind_erosion_value = [el['value'] for el in legacy_arr if el['name']=='wind_erosion'][0] |
| + c_res.add_legacyWtAvgWindEI(wind_erosion_value) |
| + c_res.eval_legacyWindErosionValue() |
| |
| -# results.append(c_res) |
| -# logging.info("List of results length: " + str(len(results))) |
| -# del [c_res] |
| + results.append(c_res) |
| + logging.info("List of results length: " + str(len(results))) |
| + del [c_res] |
| |
| except MaxAttemptsException as e: |
| # Handle the "max attempts" exception |
| @@ -388,15 +385,13 @@ |
| |
| ## setup output file |
| output_file = '''{repo_basepath}data/{datetime:%Y%m%dT%H%M%S}.csv'''.format(repo_basepath=repo_basepath, datetime=ref_datetime) |
| +output_failed = '''{repo_basepath}data/{datetime:%Y%m%dT%H%M%S}_failed.csv'''.format(repo_basepath=repo_basepath, datetime=ref_datetime) |
| |
| #read coordinates csv file |
| d = pd.read_csv('''{repo_basepath}data/Results.csv'''.format(repo_basepath=repo_basepath)) |
| # d = pd.read_csv('''{repo_basepath}data/test.csv'''.format(repo_basepath=repo_basepath)) |
| df = pd.DataFrame(data = d) |
| points_num = df.shape[0] |
| -#acceptable_failure_percentage = 20 |
| -#acceptable_failure_num = int(points_num*acceptable_failure_percentage/100) |
| -#counter = Value('i', 0) # 'i' represents the type of value (integer) and 0 is the initial value |
| count = cpu_count() |
| # count = 1 |
| |
| @@ -408,10 +403,20 @@ |
| |
| logging.info("Number of results: " + str(len(results))) |
| |
| -#if len(results) > 0: |
| -# with open(output_file, 'w', newline='') as csvfile: |
| -# writer = csv.writer(csvfile) |
| -# writer.writerow(results[0].write_header_line()) |
| -# [writer.writerow(r.write_results_line()) for r in results] |
| -# |
| -# csvfile.close() |
| +if len(results) > 0: |
| + with open(output_file, 'w', newline='') as csvfile: |
| + writer = csv.writer(csvfile) |
| + writer.writerow(results[0].write_header_line()) |
| + [writer.writerow(r.write_results_line()) for r in results] |
| + |
| + csvfile.close() |
| + |
| +output_df = pd.read_csv(output_file) |
| +output_df = output_df[["Latitude", "Longitude"]] |
| + |
| +a = pd.merge(d, output_df, how='left', on=["Latitude", "Longitude"], indicator=True) |
| +failed_df = a.loc[a._merge=='left_only',a.columns!='_merge'] |
| + |
| +failed_df = failed_df.set_index(d.columns[0]) |
| +failed_df.to_csv(output_failed) |
| +del failed_df |
| \ No newline at end of file |