Merge pull request #474 from oist/unique-output-dir

[#429] outputディレクトリのユニーク化

optinist/api/dataclass/image.py

@@ -11,7 +11,7 @@
 
 
 class ImageData(BaseData):
-    def __init__(self, data, file_name='image'):
+    def __init__(self, data, output_dir=DIRPATH.OUTPUT_DIR, file_name='image'):
         super().__init__(file_name)
 
         self.json_path = None
@@ -23,7 +23,7 @@ def __init__(self, data, file_name='image'):
         elif isinstance(data, list) and isinstance(data[0], str):
             self.path = data
         else:
-            _dir = join_filepath([DIRPATH.OUTPUT_DIR, "tiff", file_name])
+            _dir = join_filepath([output_dir, "tiff", file_name])
             create_directory(_dir)
 
             _path = join_filepath([_dir, f'{file_name}.tif'])
@@ -49,12 +49,12 @@ def save_json(self, json_dir):
 
 
 class RoiData(BaseData):
-    def __init__(self, data, file_name='roi'):
+    def __init__(self, data, output_dir=DIRPATH.OUTPUT_DIR, file_name='roi'):
         super().__init__(file_name)
 
         images = create_images_list(data)
 
-        _dir = join_filepath([DIRPATH.OUTPUT_DIR, "tiff", file_name])
+        _dir = join_filepath([output_dir, "tiff", file_name])
         create_directory(_dir)
         self.path = join_filepath([_dir, f'{file_name}.tif'])
         tifffile.imsave(self.path, images)

optinist/api/rules/runner.py

@@ -30,6 +30,7 @@ def run(cls, __rule: Rule, last_output):
             output_info = cls.execute_function(
                 __rule.path,
                 __rule.params,
+                os.path.dirname(__rule.output),
                 input_info
             )
 
@@ -90,13 +91,13 @@ def save_all_nwb(cls, save_path, all_nwbfile):
         )
 
     @classmethod
-    def execute_function(cls, path, params, input_info):
+    def execute_function(cls, path, params, output_dir, input_info):
         wrapper = cls.dict2leaf(
             wrapper_dict,
             path.split('/')
         )
         func = copy.deepcopy(wrapper["function"])
-        output_info = func(params=params, **input_info)
+        output_info = func(params=params, output_dir=output_dir, **input_info)
         del func
         gc.collect()
 

optinist/routers/const.py

@@ -2,4 +2,4 @@
 ACCEPT_CSV_EXT = [".csv"]
 ACCEPT_HDF5_EXT = [".hdf5", ".nwb", ".HDF5", ".NWB"]
 
-NOT_DISPLAY_ARGS_LIST = ['params', 'nwbfile']
+NOT_DISPLAY_ARGS_LIST = ['params', 'output_dir', 'nwbfile']

optinist/wrappers/caiman_wrapper/cnmf.py

@@ -57,9 +57,10 @@ def get_roi(A, thr, thr_method, swap_dim, dims):
 
 
 def caiman_cnmf(
-        images: ImageData,
-        params: dict=None
-    ) -> dict(fluorescence=FluoData, iscell=IscellData):
+    images: ImageData,
+    output_dir: str,
+    params: dict = None
+) -> dict(fluorescence=FluoData, iscell=IscellData):
     from caiman import local_correlations, stop_server
     from caiman.paths import memmap_frames_filename
     from caiman.mmapping import prepare_shape
@@ -209,12 +210,12 @@ def caiman_cnmf(
     ])
 
     info = {
-        'images': ImageData(np.array(Cn * 255, dtype=np.uint8), file_name='images'),
+        'images': ImageData(np.array(Cn * 255, dtype=np.uint8), output_dir=output_dir, file_name='images'),
         'fluorescence': FluoData(fluorescence, file_name='fluorescence'),
         'iscell': IscellData(iscell, file_name='iscell'),
-        'all_roi': RoiData(all_roi, file_name='all_roi'),
-        'cell_roi': RoiData(cell_roi, file_name='cell_roi'),
-        'non_cell_roi': RoiData(non_cell_roi, file_name='non_cell_roi'),
+        'all_roi': RoiData(all_roi, output_dir=output_dir, file_name='all_roi'),
+        'cell_roi': RoiData(cell_roi, output_dir=output_dir, file_name='cell_roi'),
+        'non_cell_roi': RoiData(non_cell_roi, output_dir=output_dir, file_name='non_cell_roi'),
         'nwbfile': nwbfile
     }
 

optinist/wrappers/caiman_wrapper/motion_correction.py

@@ -3,9 +3,10 @@
 
 
 def caiman_mc(
-        image: ImageData,
-        params: dict=None
-    ) -> dict(mc_images=ImageData):
+    image: ImageData,
+    output_dir: str,
+    params: dict = None
+) -> dict(mc_images=ImageData):
     import numpy as np
     from caiman import load, save_memmap, load_memmap, stop_server
     from caiman.source_extraction.cnmf.params import CNMFParams
@@ -55,7 +56,7 @@ def caiman_mc(
     xy_trans_data = (np.array(mc.x_shifts_els), np.array(mc.y_shifts_els)) \
                     if params['pw_rigid'] else np.array(mc.shifts_rig)
 
-    mc_images = ImageData(images, file_name='mc_images')
+    mc_images = ImageData(images, output_dir=output_dir, file_name='mc_images')
 
     nwbfile = {}
     nwbfile[NWBDATASET.MOTION_CORRECTION] = {
@@ -67,8 +68,8 @@ def caiman_mc(
 
     info = {
         'mc_images': mc_images,
-        'meanImg': ImageData(meanImg, file_name='meanImg'),
-        'rois': RoiData(rois, file_name='rois'),
+        'meanImg': ImageData(meanImg, output_dir=output_dir, file_name='meanImg'),
+        'rois': RoiData(rois, output_dir=output_dir, file_name='rois'),
         'nwbfile': nwbfile,
     }
 

optinist/wrappers/lccd_wrapper/lccd_detection.py

@@ -3,7 +3,9 @@
 
 
 def lccd_detect(
-    mc_images: ImageData, params: dict = None
+    mc_images: ImageData,
+    output_dir: str,
+    params: dict = None
 ) -> dict(fluorescence=FluoData, cell_roi=RoiData):
     from .lccd_python.lccd import LCCD
 
@@ -40,7 +42,7 @@ def lccd_detect(
                 timeseries_dff[i, k] = (timeseries[i, k] - f0) / f0
 
     info = {
-        'rois': RoiData(np.nanmax(roi_list, axis=0), file_name='cell_roi'),
+        'rois': RoiData(np.nanmax(roi_list, axis=0), output_dir=output_dir, file_name='cell_roi'),
         'fluorescence': FluoData(timeseries, file_name='fluorescence'),
         'dff': FluoData(timeseries_dff, file_name='dff'),
     }

optinist/wrappers/optinist_wrapper/basic_neural_analysis/cell_grouping.py

@@ -2,10 +2,11 @@
 
 
 def cell_grouping(
-        neural_data: TimeSeriesData,
-        nwbfile: NWBFile=None,
-        params: dict=None
-    ) -> dict():
+    neural_data: TimeSeriesData,
+    nwbfile: NWBFile=None,
+    output_dir: str,
+    params: dict = None
+) -> dict():
     import numpy as np
 
     neural_data = neural_data.data

optinist/wrappers/optinist_wrapper/basic_neural_analysis/eta.py

@@ -35,11 +35,12 @@ def calc_trigger_average(neural_data, trigger_idx, start_time, end_time):
 
 
 def ETA(
-        neural_data: FluoData,
-        behaviors_data: BehaviorData,
-        iscell: IscellData=None,
-        params: dict=None
-    ) -> dict(mean=TimeSeriesData):
+    neural_data: FluoData,
+    behaviors_data: BehaviorData,
+    output_dir: str,
+    iscell: IscellData = None,
+    params: dict = None
+) -> dict(mean=TimeSeriesData):
     import numpy as np
 
     neural_data = neural_data.data

optinist/wrappers/optinist_wrapper/dimension_reduction/cca.py

@@ -3,11 +3,12 @@
 from optinist.api.nwb.nwb import NWBDATASET
 
 def CCA(
-        neural_data: FluoData,
-        behaviors_data: BehaviorData,
-        iscell: IscellData=None,
-        params: dict=None
-    ) -> dict():
+    neural_data: FluoData,
+    behaviors_data: BehaviorData,
+    output_dir: str,
+    iscell: IscellData = None,
+    params: dict = None
+) -> dict():
 
     from sklearn.cross_decomposition import CCA
 

optinist/wrappers/optinist_wrapper/dimension_reduction/pca.py

@@ -3,10 +3,11 @@
 from optinist.api.nwb.nwb import NWBDATASET
 
 def PCA(
-        neural_data: FluoData,
-        iscell: IscellData=None,
-        params: dict=None
-    ) -> dict():
+    neural_data: FluoData,
+    output_dir: str,
+    iscell: IscellData = None,
+    params: dict = None
+) -> dict():
 
     # modules specific to function
     from sklearn.decomposition import PCA

optinist/wrappers/optinist_wrapper/dimension_reduction/tsne.py

@@ -3,10 +3,11 @@
 from optinist.api.nwb.nwb import NWBDATASET
 
 def TSNE(
-        neural_data: FluoData,
-        iscell: IscellData=None,
-        params: dict=None
-    ) -> dict():
+    neural_data: FluoData,
+    output_dir: str,
+    iscell: IscellData = None,
+    params: dict = None
+) -> dict():
 
     from sklearn.manifold import TSNE
 

optinist/wrappers/optinist_wrapper/neural_decoding/glm.py

@@ -9,11 +9,12 @@
 from optinist.api.nwb.nwb import NWBDATASET
 
 def GLM(
-        neural_data: FluoData,
-        behaviors_data: BehaviorData,
-        iscell: IscellData=None,
-        params: dict=None
-    ) -> dict():
+    neural_data: FluoData,
+    behaviors_data: BehaviorData,
+    output_dir: str,
+    iscell: IscellData = None,
+    params: dict = None
+) -> dict():
 
     # modules specific to function
     import statsmodels.api as sm

optinist/wrappers/optinist_wrapper/neural_decoding/lda.py

@@ -3,11 +3,12 @@
 from optinist.api.nwb.nwb import NWBDATASET
 
 def LDA(
-        neural_data: FluoData,
-        behaviors_data: BehaviorData,
-        iscell: IscellData=None,
-        params: dict=None
-    ) -> dict():
+    neural_data: FluoData,
+    behaviors_data: BehaviorData,
+    output_dir: str,
+    iscell: IscellData = None,
+    params: dict = None
+) -> dict():
 
     # modules specific to function
     from sklearn.discriminant_analysis import LinearDiscriminantAnalysis as LDA

optinist/wrappers/optinist_wrapper/neural_decoding/svm.py

@@ -3,11 +3,12 @@
 from optinist.api.nwb.nwb import NWBDATASET
 
 def SVM(
-        neural_data: FluoData,
-        behaviors_data: BehaviorData,
-        iscell: IscellData=None,
-        params: dict=None
-    ) -> dict():
+    neural_data: FluoData,
+    behaviors_data: BehaviorData,
+    output_dir: str,
+    iscell: IscellData = None,
+    params: dict = None
+) -> dict():
 
     # modules specific to function
     import numpy as np

optinist/wrappers/optinist_wrapper/neural_population_analysis/correlation.py

@@ -3,10 +3,11 @@
 
 
 def correlation(
-        neural_data: FluoData,
-        iscell: IscellData=None,
-        params: dict=None
-    ) -> dict():
+    neural_data: FluoData,
+    output_dir: str,
+    iscell: IscellData = None,
+    params: dict = None
+) -> dict():
 
     neural_data = neural_data.data
 

optinist/wrappers/optinist_wrapper/neural_population_analysis/cross_correlation.py

@@ -3,10 +3,11 @@
 
 
 def cross_correlation(
-        neural_data: FluoData,
-        iscell: IscellData=None,
-        params: dict=None
-    ) -> dict():
+    neural_data: FluoData,
+    output_dir: str,
+    iscell: IscellData = None,
+    params: dict = None
+) -> dict():
 
     import scipy.signal as ss
     import itertools

optinist/wrappers/optinist_wrapper/neural_population_analysis/granger.py

@@ -4,10 +4,11 @@
 
 
 def Granger(
-        neural_data: FluoData,
-        iscell: IscellData=None,
-        params: dict=None
-    ) -> dict():
+    neural_data: FluoData,
+    output_dir: str,
+    iscell: IscellData = None,
+    params: dict = None
+) -> dict():
 
     # modules specific to function
     # from sklearn.preprocessing import StandardScaler

optinist/wrappers/suite2p_wrapper/file_convert.py

@@ -1,14 +1,14 @@
 import os
 
 from optinist.api.dataclass.dataclass import *
-from optinist.api.dir_path import DIRPATH
 from optinist.api.utils.filepath_creater import join_filepath
 
 
 def suite2p_file_convert(
-        image: ImageData,
-        params: dict=None
-    ) -> dict(ops=Suite2pData):
+    image: ImageData,
+    output_dir: str,
+    params: dict = None
+) -> dict(ops=Suite2pData):
     from suite2p import io, default_ops
     print('start suite2_file_convert')
 
@@ -24,7 +24,7 @@ def suite2p_file_convert(
     db = {
         'data_path': data_path_list,
         'tiff_list': data_name_list,
-        'save_path0': DIRPATH.OUTPUT_DIR,
+        'save_path0': output_dir,
         'save_folder': 'suite2p'
     }
 
@@ -39,7 +39,7 @@ def suite2p_file_convert(
     ops = io.tiff_to_binary(ops.copy())
 
     info = {
-        'meanImg': ImageData(ops['meanImg'], file_name='meanImg'),
+        'meanImg': ImageData(ops['meanImg'], output_dir=output_dir, file_name='meanImg'),
         'ops': Suite2pData(ops, file_name='ops')
     }
 

optinist/wrappers/suite2p_wrapper/registration.py

@@ -2,9 +2,10 @@
 
 
 def suite2p_registration(
-        ops: Suite2pData,
-        params: dict=None
-    ) -> dict(ops=Suite2pData):
+    ops: Suite2pData,
+    output_dir: str,
+    params: dict = None
+) -> dict(ops=Suite2pData):
     from suite2p import registration, default_ops
     ops = ops.data
     refImg = ops['meanImg']
@@ -24,8 +25,8 @@ def suite2p_registration(
         ops = registration.get_pc_metrics(ops)
 
     info = {
-        'refImg': ImageData(ops['refImg'], file_name='refImg'),
-        'meanImgE': ImageData(ops['meanImgE'], file_name='meanImgE'),
+        'refImg': ImageData(ops['refImg'], output_dir=output_dir, file_name='refImg'),
+        'meanImgE': ImageData(ops['meanImgE'], output_dir=output_dir, file_name='meanImgE'),
         'ops': Suite2pData(ops, file_name='ops'),
     }