Fixed daemon installation

 *Changed version to 0.0.2

Author: dariogf

History.txt

@@ -1,3 +1,7 @@
+=== 0.0.2 2011-07-12
+
+Fixed daemon installation
+
 === 0.0.1 2011-06-28
 
 * 1 major enhancement:

Manifest.txt

@@ -1,11 +1,23 @@
+bin/queue_manager.rb
+bin/sqstat
+bin/sqsub
+bin/sqs_install_daemon
 History.txt
+lib/scbi_queue_system/autolaunch/com.scbi_queue_system.plist
+lib/scbi_queue_system/autolaunch/sqsd-linux
+lib/scbi_queue_system/done_job_list.rb
+lib/scbi_queue_system/internal_config/internal_config.json
+lib/scbi_queue_system/job_list.rb
+lib/scbi_queue_system/queued_job_list.rb
+lib/scbi_queue_system/running_job_list.rb
+lib/scbi_queue_system.rb
 Manifest.txt
 PostInstall.txt
-README.rdoc
 Rakefile
-lib/scbi_queue_system.rb
+README.rdoc
 script/console
 script/destroy
 script/generate
+test/submit_script.sh
 test/test_helper.rb
 test/test_scbi_queue_system.rb

README.rdoc

@@ -1,32 +1,122 @@
 = scbi_queue_system
 
-* http://github.com/#{github_username}/#{project_name}
+* http://www.scbi.uma.es/downloads
 
 == DESCRIPTION:
 
-FIX (describe your package)
+scbi_queue_system (SQS) handles a simple queue of jobs executions over multiple machines (clustered installation) or your own personal computer.
 
 == FEATURES/PROBLEMS:
 
-* FIX (list of features or problems)
+* SQS can be used as very simple batch queue system for personal multicore computers as well as for small clusters
+* It handles machines with different number of cores/CPUs
+* It believes on well intentioned users, and because of that it doesn't kill jobs if they are using more CPUs than requested
+* All jobs run under the same user (the one that SQS manager run under)
 
 == SYNOPSIS:
 
-  FIX (code sample of usage)
+Once SQS is installed and queue manager is running, you can start using it:
+
+=== To submit a new job:
+
+  sqsub file.sh
+
+where file.sh is a script file where you run your programs. Example file issuing ls and hostname commands:
+
+  $> cat file.sh 
+  #!/usr/bin/env bash
+
+  ls
+  hostname
+
+=== To submit a new job using 4 cpus:
+
+  sqsub file.sh 4
+
+You can also set the cpu count inside the submit script this way:
+
+  $> cat file.sh 
+  #!/usr/bin/env bash
+  # CPUS = 4
+
+  ls
+  hostname
+
+=== To view queue status
+
+  sqstat
+  
+To view already done jobs:
+  
+  sqstat -d
+  
 
 == REQUIREMENTS:
 
-* FIX (list of requirements)
+* OS X / linux operating systems.
 
 == INSTALL:
 
-* FIX (sudo gem install, anything else)
+=== 1.- Personal/standalone computer:
+
+In a personal installation, all pieces of the queue system are installed on the same computer. To install the queue manager on one computer:
+
+  gem install scbi_queue_system
+
+Once installed, you should start the queue manager:
+
+  queue_manager.rb
+
+By default, it is configured to use localhost with 1 cpu as testing
+  
+<b>NOTE:</b> See automatic startup
+
+
+=== 2.-Cluster:
+
+On a clustered installation you need to choose a node to act as the queue manager. 
+
+==== -On the manager machine
+
+Install SQS on the manager machine:
+
+  gem install scbi_queue_system
+
+Start the queue manager:
+
+  queue_manager.rb
+
+Share your QUEUED folder with frontend machines and give write-only permission to others (chmod 772)
+
+==== -On the frontends machines:
+
+Later on, you should install the sqsub command on your cluster's frontend machines (the one/ones from where you are going to submit jobs).
+
+  gem install scbi_queue_system
+
+=== 3.- Environment variables:
+
+* SQS_BASE_PATH : sets the location of sqs directoies. 
+
+* SQS_QUEUED_PATH : sets the location of QUEUED directory. This location must be shared and with write only permissions for all SQS clients (those that need to use sqsub).
+
+* SQS_LOG_FILE: sets the location of log files.
+
+=== 4.- Automatic startup
+
+Once the gem is installed you can start the queue manager (it is recommended to setup automatic startup so you don't have to start it manually after each reboot). The queue manager only should be started up in the manager machine, and not on frontends.
+
+==== -On Linux
+
+
+==== -On Mac OS X
+
 
 == LICENSE:
 
 (The MIT License)
 
-Copyright (c) 2011 FIXME full name
+Copyright (c) 2011 Dario guerrero
 
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the

Rakefile

@@ -11,10 +11,10 @@ Hoe.plugin :newgem
 # Generate all the Rake tasks
 # Run 'rake -T' to see list of generated tasks (from gem root directory)
 $hoe = Hoe.spec 'scbi_queue_system' do
-  self.developer 'FIXME full name', 'FIXME email'
+  self.developer 'Dario Guerrero', '[email protected]'
   self.post_install_message = 'PostInstall.txt' # TODO remove if post-install message not required
   self.rubyforge_name       = self.name # TODO this is default value
-  # self.extra_deps         = [['activesupport','>= 2.0.2']]
+  self.extra_deps         = [['json','>= 1.5.3']]
 
 end
 

bin/queue_manager.rb

@@ -1,6 +1,6 @@
 #!/usr/bin/env ruby
 
-$: << File.join(File.dirname(__FILE__),'..','lib')
+# $: << File.join(File.dirname(__FILE__),'..','lib')
 
 require 'scbi_queue_system'
 require 'logger'
@@ -9,9 +9,19 @@
 require 'queued_job_list'
 require 'done_job_list'
 
+system_log_message="SCBI SQS queue at: #{QUEUED_PATH}"
+`logger "#{system_log_message}"`
 
-# $LOG = Logger.new(LOG_FILE, 10, 1024000)
-$LOG = Logger.new(STDOUT, 10, 1024000)
+user=`whoami`
+system_log_message="SCBI SQS user: #{user}"
+`logger "#{system_log_message}"`
+
+
+$LOG = Logger.new(LOG_FILE, 10, 1024000)
+
+# $LOG = Logger.new(STDOUT, 10, 1024000)
+
+$LOG.level = Logger::INFO
 
 $LOG.info 'Starting up SQS'
 
@@ -35,7 +45,16 @@ def load_config
 
     config[:polling_time] = 10
     config[:machine_list] = []
-    machine={:name=>'localhost', :cpus => 1}
+    
+    cpus=1
+    
+    if RUBY_PLATFORM.downcase.include?("darwin")
+      cpus=`hwprefs -cpu_count`.chomp.to_i
+    else
+      cpus=`grep processor /proc/cpuinfo |wc -l`.chomp.to_i
+    end
+    
+    machine={:name=>'localhost', :cpus => cpus}
 
     config[:machine_list] << machine
     # config[:sqs_user] = 'dariogf'

bin/sqstat

@@ -1,6 +1,6 @@
 #!/usr/bin/env ruby
 
-$: << File.join(File.dirname(__FILE__),'..','lib')
+# $: << File.join(File.dirname(__FILE__),'..','lib')
 
 require 'scbi_queue_system'
 require 'optparse'

bin/sqsub

@@ -1,6 +1,6 @@
 #!/usr/bin/env ruby
 
-$: << File.join(File.dirname(__FILE__),'..','lib')
+# $: << File.join(File.dirname(__FILE__),'..','lib')
 
 require 'scbi_queue_system'
 

lib/scbi_queue_system.rb

@@ -13,29 +13,33 @@
 if File.exists?(INTERNAL_CONFIG_PATH)
   config_txt=File.read(INTERNAL_CONFIG_PATH)
   internal_config=JSON::parse(config_txt, :symbolize_names => true)
+  if ENV['SQS_BASE_PATH']
+    internal_config[:base_path] = ENV['SQS_BASE_PATH']
+  end
 else
   # default config
   internal_config={}
-  internal_config[:base_path] = '/drives'
+  internal_config[:base_path] = ENV['SQS_BASE_PATH'] || '~/.sqs'
   
   f=File.open(INTERNAL_CONFIG_PATH,'w')
   f.puts JSON::pretty_generate(internal_config)
-  f.close  
+  f.close
 end
 
 JOBS_PATH=File.join(internal_config[:base_path],'sqs','jobs')
 
-CONFIG_PATH=File.join('/drives','sqs','config')
+CONFIG_PATH=File.join(internal_config[:base_path],'sqs','config')
 
-LOGS_PATH=File.join('/drives','sqs','logs')
+LOGS_PATH=File.join(internal_config[:base_path],'sqs','logs')
 
 CONFIG_FILE=File.join(CONFIG_PATH,'config.json')
-LOG_FILE=File.join(LOGS_PATH,'sqs.log')
+
+LOG_FILE=ENV['SQS_LOG_PATH'] || File.join(LOGS_PATH,'sqs.log')
 
 # create paths
 FileUtils.mkdir_p([CONFIG_PATH,LOGS_PATH])
 
-QUEUED_PATH=File.join(JOBS_PATH,'queued')
+QUEUED_PATH=ENV['SQS_QUEUED_PATH'] || File.join(JOBS_PATH,'queued')
 DONE_PATH=File.join(JOBS_PATH,'done')
 RUNNING_PATH=File.join(JOBS_PATH,'running')
 SENT_PATH=File.join(JOBS_PATH,'sent')
@@ -49,5 +53,5 @@
 
 
 module ScbiQueueSystem
-  VERSION = '0.0.1'
-end
+   VERSION = '0.0.2'
+end