BN-102 First result of migration to full paths - test-project work on…

… linux

BlendNet/AgentTask.py

@@ -63,7 +63,7 @@ def _executionWatcher(self):
  finally:
  self._parent._fc.workspaceClean(self.name())
 
- print('INFO: Execution of the task "%s" is ended' % self.name())
+ print('INFO: Execution of the task "%s" is ended' % (self.name(),))
  self.stateStop()
 
  with self._execution_lock:

BlendNet/Config.py

@@ -64,11 +64,13 @@ def _setattr(self, name, value):
 
  def configsSet(self, configs):
  '''Walk through the defined configs and set configs values'''
+ results = []
  for name, conf in self._defs.items():
  if name in configs:
- self._setattr(name, configs[name])
+ results.append(self._setattr(name, configs[name]))
  elif conf.get('value') is not None:
- self._setattr(name, conf['value'](self) if callable(conf['value']) else conf['value'])
+ results.append(self._setattr(name, conf['value'](self) if callable(conf['value']) else conf['value']))
+ return all(results)
 
  def configsGet(self):
  '''Returns set configs or defaults if defined'''

BlendNet/ManagerTask.py

@@ -149,7 +149,7 @@ def _mergeWorker(self, to_merge):
  else:
  files = dict([ (blob + '.exr', blob) for blob in to_merge[1] ])
  cfg = {
- 'images': list(files.keys()),
+ 'images': [ 'project/' + f for f in files.keys() ],
  'result': 'result.exr',
  }
  with self.prepareWorkspace(files) as ws_path:
@@ -196,7 +196,7 @@ def _composeWorker(self):
  cfg = {
  'use_compositing_nodes': self._cfg.use_compositing_nodes,
  'frame': self._cfg.frame,
- 'render_file_path': render_name + '.exr',
+ 'render_file_path': 'project/' + render_name + '.exr',
  'result_dir': render_name + '-result',
  }
  print('DEBUG: Files to use in workspace:')
@@ -210,7 +210,7 @@ def _composeWorker(self):
  for filename in os.listdir(os.path.join(ws_path, cfg['result_dir'])):
  blob = self._parent._fc.blobStoreFile(os.path.join(ws_path, cfg['result_dir'], filename), True)
  if not blob:
- print('ERROR: Unable to store blob for compose result of "%s"' % self.name())
+ print('ERROR: Unable to store blob for compose result of', self.name())
  return
  self.statusComposeSet(blob['id'])
  break
@@ -221,7 +221,7 @@ def _composeWorker(self):
  print('ERROR: Exception occurred during composing the result for task "%s": %s' % (self.name(), e))
  self.stateError({self.name(): 'Exception occurred during composing the result: %s' % (e,)})
 
- print('DEBUG: Compositing completed for task "%s"' % (self.name(),))
+ print('DEBUG: Compositing completed for task', self.name())
 
  def _processOutputs(self, process, show_out = False):
  '''Shows info from the process'''

BlendNet/Server.py

@@ -134,6 +134,11 @@ def task_file_info(self, req, parts):
  @SimpleREST.put('task/*/file/**')
  def put_task_file(self, req, parts):
  '''Upload file required to execute task'''
+ # The cross-system paths are a pain. In blender we have 3 types (including '//'), so the rules are:
+ # * Path delimiter is always '/'. Backslash '\' is for escaping only
+ # * The path should be straight and can't contain '/../' (parent dir usage)
+ # * Relative paths (dir/file.txt) will be transformed with 'cwd' to absolute path during config
+ # * Project paths (//something) will be transformed with 'path' to absolute path during config
  length = req.headers['content-length']
  if not length:
  return { 'success': False, 'message': 'Unable to find "Content-Length" header' }
@@ -153,6 +158,9 @@ def put_task_file(self, req, parts):
  if not task.fileAdd(parts[1], result['id']):
  return { 'success': False, 'message': 'Error during add file to the task' }
 
+ if self._e.taskGet(parts[0]).filesPathsFix(parts[1]) == False:
+ return { 'success': False, 'message': 'Error during task file fixing' }
+
  return { 'success': True, 'message': 'Uploaded task file',
  'data': result,
  }
@@ -176,6 +184,9 @@ def task_set_config(self, req, parts):
  if not self._e.taskGet(parts[0]).configsSet(conf):
  return { 'success': False, 'message': 'Error during task configuration' }
 
+ if self._e.taskGet(parts[0]).filesPathsFix() != True:
+ return { 'success': False, 'message': 'Error during task files fixing' }
+
  return { 'success': True, 'message': 'Task configured' }
 
  @SimpleREST.get('task/*/run')

BlendNet/TaskBase.py

@@ -15,13 +15,23 @@
 from abc import ABC, abstractmethod
 
 from .Config import Config
+from . import utils
 
 class TaskConfig(Config):
  _defs = {
  'project': {
  'description': '''Set the project file will be used to render''',
  'type': str,
- 'validation': lambda cfg, val: cfg._parent.fileGet(val),
+ },
+ 'path': {
+ 'description': '''Absolute path to the project dir, required to resolve `//../dir/file`''',
+ 'type': str,
+ 'validation': lambda cfg, val: utils.isPathAbsolute(val) and utils.isPathStraight(val),
+ },
+ 'cwd': {
+ 'description': '''Absolute path to the current working dir, required to resolve `dir/file`''',
+ 'type': str,
+ 'validation': lambda cfg, val: utils.isPathAbsolute(val) and utils.isPathStraight(val),
  },
  'samples': {
  'description': '''How much samples to process for the task''',
@@ -234,7 +244,7 @@ def stateSet(self, state):
 
  def fileAdd(self, path, file_id):
  '''Add file to the files map'''
- if path.startswith('/') or '../' in path:
+ if '../' in path:
  return print('WARN: Unable to use path with absolute path or contains parent dir symlink')
 
  if not self.canBeChanged():
@@ -255,14 +265,41 @@ def filesGet(self):
  with self._files_lock:
  return self._files.copy()
 
+ def filesPathsFix(self, path = None):
+ '''Fixes the files mapping to be absolute paths'''
+ tocheck = [path] if path else self._files.keys()
+ for p in tocheck:
+ if not utils.isPathStraight(p):
+ print('ERROR: Path is not straight:', p)
+ return False
+ if not self._cfg.path or not self._cfg.cwd:
+ # The required configs are not set - skipping fix
+ return None
+
+ if not utils.isPathAbsolute(p):
+ new_p = p
+ if p.startswith('//'):
+ # Project-based file
+ new_p = self._cfg.path + ('' if self._cfg.path.endswith('/') else '/') + p[2:]
+ else:
+ # Relative path to CWD
+ new_p = self._cfg.cwd + ('' if self._cfg.cwd.endswith('/') else '/') + p
+ with self._files_lock:
+ self._files[new_p] = self._files.pop(p)
+ return True
+
  def run(self):
  '''Trigger the task to execute'''
  with self._state_lock:
  if self._state not in (TaskState.CREATED, TaskState.STOPPED):
  print('WARN: Unable to run already started task')
  return True
 
- print('DEBUG: Running task %s' % self.name())
+ print('DEBUG: Running task', self.name())
+
+ if not self.check():
+ print('ERROR: Task check fail:', self.name())
+ return False
 
  return self._parent.taskAddToPending(self)
 
@@ -351,6 +388,10 @@ def check(self):
  errors = []
  with self._files_lock:
  for path, sha1 in self._files.items():
+ if not utils.isPathAbsolute(path):
+ errors.append({self.name(): 'The file path "%s" is not absolute' % (path,)})
+ if not utils.isPathStraight(path):
+ errors.append({self.name(): 'The file path "%s" is contains parent dir usage' % (path,)})
  if not self._parent._fc.blobGet(sha1):
  errors.append({self.name(): 'Unable to find required file "%s" with id "%s" in file cache' % (path, sha1)})
  for err in errors:
@@ -359,7 +400,22 @@ def check(self):
 
  def prepareWorkspace(self, files_map):
  '''Preparing workspace to process files'''
- ws_dir = self._parent._fc.workspaceCreate(self.name(), files_map)
+ # Change the absolute paths to the required relative ones
+ # and placing files to the proper folders
+ new_files_map = {}
+ for path in files_map:
+ p = ''
+ if path.startswith(self._cfg.path):
+ p = path.replace(self._cfg.path, 'project/', 1)
+ elif utils.isPathAbsolute(path):
+ # Windows don't like colon and other spec symbols in the path
+ p = 'ext_deps' + '/' + path.replace(':', '_')
+ else:
+ # Just a regular relative files are going to the project dir
+ # they could be here because inner logic is using them to create files
+ p = 'project/' + path
+ new_files_map[p] = files_map[path]
+ ws_dir = self._parent._fc.workspaceCreate(self.name(), new_files_map)
  if not ws_dir:
  raise Exception('ERROR: Unable to prepare workspace to execute task')
 
@@ -368,20 +424,22 @@ def prepareWorkspace(self, files_map):
  def runBlenderScriptProcessor(self, workspace_path, script_suffix, cfg, blendfile = None):
  '''Running blender in workspace with providing a script path'''
 
- config_name = 'config-%s.json' % script_suffix
+ config_name = 'config-%s.json' % (script_suffix,)
  with open(os.path.join(workspace_path, config_name), 'w') as f:
  json.dump(cfg, f)
 
  script_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'script-%s.py' % script_suffix)
  command = [sys.executable, '-b', '-noaudio', '-y']
  if blendfile:
- command.append(os.path.join(workspace_path, blendfile))
+ command.append(os.path.join(workspace_path, 'project', blendfile))
  # Position of script is important - if it's after blend file,
  # than it will be started after blend loading
  command.append('-P')
  command.append(script_path)
  command.append('--')
  command.append(config_name)
+
+ print('DEBUG: Run subprocess "%s" in "%s"' % (command, workspace_path))
  return subprocess.Popen(
  command,
  cwd = workspace_path,