PR: Cache correlation coefficients

cdprep/gapfill_data/gapfill_weather_algorithm.py

@@ -142,6 +142,8 @@ def __init__(self):
         self.wxdatasets.sig_task_progress.connect(self.sig_task_progress.emit)
         self.wxdatasets.sig_status_message.connect(
             self.sig_status_message.emit)
+        self.wxdatasets.sig_corrcoeff_calculated.connect(
+            lambda: self.wxdatasets.save_to_binary(self.inputDir))
 
         self.inputDir = None
         self.isParamsValid = False
@@ -280,23 +282,11 @@ def set_target_station(self, station_id):
             raise ValueError("No data currently loaded for station '{}'."
                              .format(station_id))
         else:
-            station_name = self.wxdatasets.metadata.loc[
-                station_id]['Station Name']
-
-            message = ("Calculating correlation coefficients "
-                       "for target station {}...".format(station_name))
-            print(message)
-            self.sig_status_message.emit(message)
-
             self.target = station_id
             self.alt_and_dist = self.wxdatasets.alt_and_dist_calc(station_id)
             self.corcoef = (
                 self.wxdatasets.compute_correlation_coeff(station_id))
 
-            print("Correlation coefficients calculated "
-                  "for target station {}.".format(station_name))
-            self.sig_status_message.emit('')
-
     def get_valid_neighboring_stations(self, hdist_limit, vdist_limit):
         """
         Return the list of neighboring stations that are within the
@@ -482,6 +472,9 @@ def gapfill_data(self, time_start, time_end, max_neighbors,
         for varname in VARNAMES:
             gapfilled_data[varname] = gapfilled_data[varname].round(1)
 
+        # Replace nan values by an empty string.
+        gapfilled_data = gapfilled_data.fillna(value='')
+
         # Make sure the columns are in the right order.
         gapfilled_data = gapfilled_data[
             ['Year', 'Month', 'Day', 'Tmax', 'Tmin', 'Tavg', 'Ptot']]
@@ -827,12 +820,14 @@ class WeatherData(QObject):
     """
     sig_task_progress = QSignal(int)
     sig_status_message = QSignal(str)
+    sig_corrcoeff_calculated = QSignal()
 
     def __init__(self):
         super().__init__()
 
         self.data = None
         self.metadata = None
+        self._corrcoef = None
 
     @property
     def filenames(self):
@@ -885,6 +880,7 @@ def load_and_format_data(self, paths):
         """
         self.data = {var: pd.DataFrame([]) for var in VARNAMES}
         self.metadata = pd.DataFrame([])
+        self._corrcoef = None
         if len(paths) == 0:
             return
 
@@ -939,12 +935,16 @@ def load_from_binary(self, dirname):
             ).item()
         self.data = A['data']
         self.metadata = A['metadata']
+        self._corrcoef = A.get('corrcoef', None)
 
     def save_to_binary(self, dirname):
         """Save the data and metadata to binary files."""
+        print('Caching data...')
         os.makedirs(osp.join(dirname, '__cache__'), exist_ok=True)
-        A = {'data': self.data, 'metadata': self.metadata}
+        A = {'data': self.data, 'metadata': self.metadata,
+             'corrcoef': self._corrcoef}
         np.save(osp.join(dirname, '__cache__', 'fdata.npy'), A)
+        print('Data cached succesfully.')
 
     # ---- Utilities
     def alt_and_dist_calc(self, target_station_id):
@@ -974,14 +974,27 @@ def compute_correlation_coeff(self, target_station_id):
         Compute the correlation coefficients between the target
         station and the neighboring stations for each meteorological variable.
         """
+        if self._corrcoef is None:
+            message = "Calculating correlation coefficients..."
+            print(message)
+            self.sig_status_message.emit(message)
+            self._corrcoef = {}
+            for var in VARNAMES:
+                self._corrcoef[var] = (
+                    self.data[var].corr(min_periods=365//2))
+            print("Correlation coefficients calculated sucessfully.")
+            self.sig_corrcoeff_calculated.emit()
+            self.sig_status_message.emit('')
+
         correl_target = None
         for var in VARNAMES:
-            corr_matrix = self.data[var].corr(min_periods=365//2).rename(
-                {target_station_id: var}, axis='columns')
+            corr_var_sta = (
+                self._corrcoef[var][[target_station_id]]
+                .rename({target_station_id: var}, axis='columns'))
             if correl_target is None:
-                correl_target = corr_matrix[[var]]
+                correl_target = corr_var_sta
             else:
-                correl_target = correl_target.join(corr_matrix[[var]])
+                correl_target = correl_target.join(corr_var_sta)
         return correl_target
 
     def generate_html_summary_table(self):

cdprep/gapfill_data/gapfill_weather_gui.py

@@ -338,6 +338,7 @@ def update_corrcoeff(self):
                 self._pending_corrcoeff_update = station_id
             else:
                 self._corrcoeff_update_inprogress = True
+                self.corrcoeff_textedit.setText('')
                 self.gapfill_manager.set_target_station(
                     station_id, callback=self._handle_corrcoeff_updated)