Implemented profile sparsity in the ejson v3

requirements.txt

@@ -22,3 +22,5 @@ Cython
 ortools
 pyproj
 folium
+pyarrow
+fastparquet

setup.py

@@ -67,6 +67,8 @@
                 "h5py>=2.9.0",
                 "numba>=0.54",
                 "folium",
+                "pyarrow",
+                "fastparquet",
                 "pytest>=3.8"]
 
 # Arguments marked as "Required" below must be included for upload to PyPI.

src/GridCal/Engine/Core/Compilers/circuit_to_newton_pa.py

@@ -432,22 +432,54 @@ def get_hvdc_data(circuit: MultiCircuit, npa_circuit: "npa.HybridCircuit", bus_d
                   HvdcControlType.type_1_Pset: npa.HvdcControlMode.HvdcControlPfix}
 
     for i, elm in enumerate(circuit.hvdc_lines):
+        """
+        (uuid: str = '', 
+        secondary_id: str = '', 
+        name: str = '', 
+        calc_node_from: newtonpa.CalculationNode = None, 
+        calc_node_to: newtonpa.CalculationNode = None, 
+        cn_from: newtonpa.ConnectivityNode = None, 
+        cn_to: newtonpa.ConnectivityNode = None, 
+        time_steps: int = 1, 
+        active_default: int = 1, 
+        rate: float = 9999, 
+        contingency_rate: float = 9999, 
+        monitor_loading_default: int = 1, 
+        monitor_contingency_default: int = 1, 
+        P: float = 0.0, 
+        Vf: float = 1.0, 
+        Vf: float = 1.0, 
+        r: float = 1e-20, 
+        angle_droop: float = 360.0, 
+        length: float = 0.0, 
+        min_firing_angle_f: float = -1.0, 
+        max_firing_angle_f: float = 1.0, 
+        min_firing_angle_t: float = -1.0, 
+        max_firing_angle_t: float = -1.0, 
+        control_mode: newtonpa.HvdcControlMode = <HvdcControlMode.HvdcControlPfix: 1>)
+        """
         hvdc = npa.HvdcLine(uuid=elm.idtag,
-                            code=elm.code,
+                            secondary_id=elm.code,
                             name=elm.name,
                             calc_node_from=bus_dict[elm.bus_from.idtag],
                             calc_node_to=bus_dict[elm.bus_to.idtag],
+                            cn_from=None,
+                            cn_to=None,
                             time_steps=ntime,
-                            length=elm.length,
+                            active_default=int(elm.active),
                             rate=elm.rate,
-                            active_default=elm.active,
+                            contingency_rate=elm.rate * elm.contingency_factor,
+                            monitor_loading_default=1,
+                            monitor_contingency_default=1,
+                            P=elm.Pset,
+                            Vf=elm.Vset_f,
+                            Vt=elm.Vset_t,
                             r=elm.r,
-                            Pset=elm.Pset,
-                            v_set_f=elm.Vset_f,
-                            v_set_t=elm.Vset_t,
+                            angle_droop=elm.angle_droop,
+                            length=elm.length,
                             min_firing_angle_f=elm.min_firing_angle_f,
-                            min_firing_angle_t=elm.min_firing_angle_t,
                             max_firing_angle_f=elm.max_firing_angle_f,
+                            min_firing_angle_t=elm.min_firing_angle_t,
                             max_firing_angle_t=elm.max_firing_angle_t,
                             control_mode=cmode_dict[elm.control_mode])
 
@@ -457,8 +489,8 @@ def get_hvdc_data(circuit: MultiCircuit, npa_circuit: "npa.HybridCircuit", bus_d
         if time_series:
             hvdc.active = elm.active_prof.astype(BINT)
             hvdc.rates = elm.rate_prof
-            hvdc.v_set_f = elm.Vset_f_prof
-            hvdc.v_set_t = elm.Vset_t_prof
+            hvdc.Vf = elm.Vset_f_prof
+            hvdc.Vt = elm.Vset_t_prof
             hvdc.contingency_rates = elm.rate_prof * elm.contingency_factor
             hvdc.angle_droop = elm.angle_droop_prof
         else:
@@ -644,10 +676,15 @@ def newton_pa_pf(circuit: MultiCircuit, opt: PowerFlowOptions, time_series=False
 
     if time_series:
         time_indices = [i for i in range(circuit.get_time_number())]
+        n_threads = 0  # max threads
     else:
         time_indices = [0]
+        n_threads = 1
 
-    pf_res = npa.runPowerFlow(npaCircuit, pf_options, time_indices)
+    pf_res = npa.runPowerFlow(numeric_circuit=npaCircuit,
+                              pf_options=pf_options,
+                              time_indices=time_indices,
+                              n_threads=n_threads)
 
     return pf_res
 
@@ -763,13 +800,13 @@ def debug_newton_pa_circuit_at(npa_circuit: "npa.HybridCircuit", t: int = None):
 
     from GridCal.Engine import *
 
-    fname = '/home/santi/Documentos/Git/GitHub/GridCal/Grids_and_profiles/grids/IEEE14_from_raw.gridcal'
-
+    # fname = '/home/santi/Documentos/Git/GitHub/GridCal/Grids_and_profiles/grids/IEEE14_from_raw.gridcal'
+    fname = '/home/santi/Documentos/Git/GitHub/GridCal/Grids_and_profiles/grids/IEEE39.gridcal'
     _grid = FileOpen(fname).open()
 
     # _newton_grid = to_newton_pa(circuit=_grid, time_series=False)
     _options = PowerFlowOptions()
-    _res = newton_pa_pf(circuit=_grid, opt=_options, time_series=False)
+    _res = newton_pa_pf(circuit=_grid, opt=_options, time_series=True)
 
     _res2 = translate_newton_pa_pf_results(_grid, _res)
 

src/GridCal/Engine/Core/multi_circuit.py

@@ -837,7 +837,7 @@ def get_profiles_dict(self):
         """
         """
         if self.time_profile is not None:
-            t = self.time_profile.view(int).tolist()
+            t = (self.time_profile.view(int) * 1e-9).tolist()  # UNIX time in seconds
         else:
             t = list()
         return {'time': t}
@@ -928,6 +928,22 @@ def format_profiles(self, index):
             for elm in branch_list:
                 elm.create_profiles(index)
 
+    def set_time_profile(self, unix_data):
+
+        # try parse
+        try:
+            self.time_profile = pd.to_datetime(np.array(unix_data), unit='s', origin='unix')
+        except Exception as e:
+            # it may come in nanoseconds instead of seconds...
+            self.time_profile = pd.to_datetime(np.array(unix_data) / 1e9, unit='s', origin='unix')
+
+        for elm in self.buses:
+            elm.create_profiles(self.time_profile)
+
+        for branch_list in self.get_branch_lists():
+            for elm in branch_list:
+                elm.create_profiles(self.time_profile)
+
     def ensure_profiles_exist(self):
         """
         Format the pandas profiles in place using a time index.
@@ -2023,11 +2039,12 @@ def get_automatic_precision(self):
 
         return tolerance, exponent
 
-    def fill_xy_from_lat_lon(self, destructive=True, factor=0.01):
+    def fill_xy_from_lat_lon(self, destructive=True, factor=0.01, remove_offset=True):
         """
         fill the x and y value from the latitude and longitude values
         :param destructive: if true, the values are overwritten regardless, otherwise only if x and y are 0
         :param factor: Explosion factor
+        :param remove_offset: remove the sometimes huge offset coming from pyproj
         """
 
         n = len(self.buses)
@@ -2038,17 +2055,32 @@ def fill_xy_from_lat_lon(self, destructive=True, factor=0.01):
             lat[i] = bus.latitude
 
         # perform the coordinate transformation
-        import pyproj
+        logger = Logger()
+        try:
+            import pyproj
+        except ImportError:
+            logger.add_error("pyproj is not installed")
+            return logger
+
         x, y = pyproj.Transformer.from_crs(4326, 25830, always_xy=True).transform(lon, lat)
         x *= factor
         y *= factor
 
+        # remove the offset
+        if remove_offset:
+            x_min = np.min(x)
+            y_max = np.max(y)
+            x -= x_min + 100  # 100 is a healthy offset
+            y -= y_max - 100  # 100 is a healthy offset
+
         # assign the values
         for i, bus in enumerate(self.buses):
             if destructive or (bus.x == 0.0 and bus.y == 0.0):
                 bus.x = x[i]
                 bus.y = -y[i]
 
+        return logger
+
     def fill_lat_lon_from_xy(self, destructive=True, factor=1.0, offset_x=0, offset_y=0):
         """
         Convert the coordinates to some random lat lon
@@ -2065,7 +2097,13 @@ def fill_lat_lon_from_xy(self, destructive=True, factor=1.0, offset_x=0, offset_
             x[i] = bus.x * factor + offset_x
             y[i] = bus.y * factor + offset_y
 
-        import pyproj
+        logger = Logger()
+        try:
+            import pyproj
+        except ImportError:
+            logger.add_error("pyproj is not installed")
+            return logger
+
         proj_latlon = pyproj.Proj(proj='latlong', datum='WGS84')
         proj_xy = pyproj.Proj(proj="utm", zone=33, datum='WGS84')
         lonlat = pyproj.transform(proj_xy, proj_latlon, x, y)
@@ -2079,13 +2117,15 @@ def fill_lat_lon_from_xy(self, destructive=True, factor=1.0, offset_x=0, offset_
                 bus.latitude = lat[i]
                 bus.longitude = lon[i]
 
+        return logger
+
     def import_bus_lat_lon(self, df: pd.DataFrame, bus_col, lat_col, lon_col):
         """
 
-        :param df:
-        :param bus_col:
-        :param lat_col:
-        :param lon_col:
+        :param df: Pandas DataFrame with the information
+        :param bus_col: bus column name
+        :param lat_col: latitude column name
+        :param lon_col: longitude column name
         :return:
         """
         logger = Logger()
@@ -2164,8 +2204,6 @@ def import_plexos_generation_profiles(self, df: pd.DataFrame):
                 gen.P_prof = df[col_name].values
             except KeyError:
                 logger.add_error("Missing in the model", col_name)
-                # gen.P_prof = np.zeros(nn)
-                # gen.Q_prof = np.zeros(nn)
 
         return logger
 
@@ -2416,4 +2454,33 @@ def correct_inconsistencies(self, logger: Logger, maximum_difference=0.1, min_vs
         for elm in self.get_generators():
             elm.fix_inconsistencies(logger,
                                     min_vset=min_vset,
-                                    max_vset=max_vset)
+                                    max_vset=max_vset)
+
+    def normalize_bus_positions_offset(self, base_offset=100):
+        """
+        Normalize the massive offset that may be in the Qt graphic objects' positions
+        :param base_offset:
+        :return:
+        """
+        n = len(self.buses)
+        x = np.zeros(n, dtype=int)
+        y = np.zeros(n, dtype=int)
+
+        # read values
+        for i, bus in enumerate(self.buses):
+            gr = bus.graphic_obj
+            x[i] = gr.x()
+            y[i] = gr.y()
+
+        # correct values
+        x_min = np.min(x)
+        y_max = np.max(y)
+        x -= x_min + base_offset  # 100 is a healthy offset
+        y -= y_max - base_offset  # 100 is a healthy offset
+
+        # assign values
+        for i, bus in enumerate(self.buses):
+            bus.x = x[i]
+            bus.y = y[i]
+            if bus.graphic_obj is not None:
+                bus.graphic_obj.set_position(x[i], y[i])

src/GridCal/Engine/Devices/bus.py

@@ -365,7 +365,6 @@ def determine_bus_type_at(self, t) -> BusMode:
             # Nothing special; set it as PQ
             return BusMode.PQ
 
-
     def determine_bus_type_prof(self):
         """
         Array of bus types according to the profile

src/GridCal/Engine/Devices/editable_device.py

@@ -107,7 +107,10 @@ def __lt__(self, other):
         return self.__hash__() < other.__hash__()
 
     def __eq__(self, other):
-        return self.idtag == other.idtag
+        if hasattr(other, 'idtag'):
+            return self.idtag == other.idtag
+        else:
+            return False
 
     @property
     def name(self):

src/GridCal/Engine/Devices/load.py

@@ -202,6 +202,7 @@ def get_profiles_dict(self, version=3):
             Ii_prof = self.Ii_prof.tolist()
             G_prof = self.G_prof.tolist()
             B_prof = self.B_prof.tolist()
+
         else:
             active_profile = list()
             P_prof = list()

src/GridCal/Engine/IO/file_handler.py

@@ -19,7 +19,7 @@
 
 from GridCal.Engine.basic_structures import Logger
 
-from GridCal.Engine.IO.json_parser import save_json_file
+from GridCal.Engine.IO.json_parser import save_json_file_v3, save_json_file_v4
 from GridCal.Engine.IO.cim.cim_parser import CIMExport
 from GridCal.Engine.IO.excel_interface import save_excel, load_from_xls, interpret_excel_v3, interprete_excel_v2
 from GridCal.Engine.IO.pack_unpack import create_data_frames, data_frames_to_circuit
@@ -171,6 +171,14 @@ def open(self, text_func=None, progress_func=None):
                     else:
                         self.logger.add_error('Unknown json format')
 
+                elif file_extension.lower() == '.ejson3':
+                    data = json.load(open(self.file_name))
+                    self.circuit = parse_json_data_v3(data, self.logger)
+
+                elif file_extension.lower() == '.ejson4':
+                    data = json.load(open(self.file_name))
+                    # self.circuit = parse_json_data_v4(data, self.logger)
+
                 elif file_extension.lower() == '.raw':
                     parser = PSSeParser(self.file_name, text_func=text_func,  progress_func=progress_func)
                     self.circuit = parser.circuit
@@ -236,8 +244,11 @@ def save(self):
         elif self.file_name.endswith('.sqlite'):
             logger = self.save_sqlite()
 
-        elif self.file_name.endswith('.json'):
-            logger = self.save_json()
+        elif self.file_name.endswith('.ejson3'):
+            logger = self.save_json_v3()
+
+        elif self.file_name.endswith('.ejson4'):
+            logger = self.save_json_v4()
 
         elif self.file_name.endswith('.xml'):
             logger = self.save_cim()
@@ -299,13 +310,22 @@ def save_sqlite(self):
 
         return logger
 
-    def save_json(self):
+    def save_json_v3(self):
+        """
+        Save the circuit information in json format
+        :return:logger with information
+        """
+
+        logger = save_json_file_v3(self.file_name, self.circuit, self.simulation_drivers)
+        return logger
+
+    def save_json_v4(self):
         """
         Save the circuit information in json format
         :return:logger with information
         """
 
-        logger = save_json_file(self.file_name, self.circuit, self.simulation_drivers)
+        logger = save_json_file_v4(self.file_name, self.circuit, self.simulation_drivers)
         return logger
 
     def save_cim(self):