support r (alibaba#147)

* support r lang

bin/fun-install.js

@@ -17,7 +17,8 @@ program
   .option('-r, --runtime <runtime>', 'function runtime, avaliable choice is: python2.7, python3, nodejs6, nodejs8, java8, php7.2')
   .option('--save', 'add module to fun.yml file.')
   .option('-R, --recursive', 'recursive install fun.yml in subdirectory.')
-  .option('-p, --package-type <type>', 'avaliable package type option: module, pip, apt, defaults to \'module\'')
+  .option('-p, --package-type <type>', 'avaliable package type option: pip, apt.')
+  .option('-e, --env [env]', 'environment variable, ex. -e PATH=/code/bin', (e, envs) => (envs.push(e), envs), [])
   .arguments('[packageNames...]')
   .description('install dependencies which are described in fun.yml file.')
   .action(async (packageNames, program) => {
@@ -37,6 +38,11 @@ program
 
     opts.verbose = parseInt(process.env.FUN_VERBOSE) > 0;
 
+    // [ 'A=B', 'B=C' ] => { A: 'B', B: 'C' }
+    opts.env = (options.env || []).map( e => _.split(e, '=', 2))
+      .filter( e => e.length === 2 )
+      .reduce((acc, cur) => (acc[cur[0]] = cur[1], acc), {});
+
     install(packageNames, opts).catch(handler);
   });
 

examples/install/pyzbar_example/fun.yml

@@ -1,8 +1,7 @@
 runtime: python3
 tasks:
   - apt: libzbar0
-    local: true
+  - shell: ln -sf libzbar.so.0.2.0 libzbar.so
+    cwd: /code/.fun/root/usr/lib
   - pip: Pillow
-    local: true
   - pip: pyzbar
-    local: true

examples/install/pyzbar_example/index.py

@@ -1,3 +1,7 @@
+import os
+for a in os.environ:
+    print('Var: ', a, 'Value: ', os.getenv(a))
+
 from pyzbar.pyzbar import decode
 from pyzbar.pyzbar import ZBarSymbol
 from PIL import Image

examples/install/rlang/fun.yml

@@ -0,0 +1,25 @@
+runtime: python3
+tasks:
+  - apt: libblas3
+  - pip: pandas
+  - name: compile R
+    shell: |-
+      apt-get build-dep -y r-base;
+      curl -L https://fc-demo-public.oss-cn-hangzhou.aliyuncs.com/fun/examples/R-3.5.2.tar.gz | tar -zxf -;
+      cd R-3.5.2 ; 
+      ./configure --prefix=/code/.fun/R/ --enable-R-shlib --with-blas --with-lapack ; 
+      make ; 
+      make install;
+  - pip: rpy2
+    env: 
+      PATH: /code/.fun/R/bin
+      LD_LIBRARY_PATH: /code/.fun/root/usr/lib/libblas
+  - name: silm size
+    shell: |-
+      rm -rf R-3.5.2; 
+      rm -rf .fun/root/usr/share; 
+      rm -rf .fun/R/share; 
+      rm -rf .fun/R/lib/R/doc;
+      rm -rf .fun/python/lib/python3.6/site-packages/pandas/tests;
+      rm -rf .fun/R/lib/R/library/*/{help,demo,html,doc};
+      find .fun -type f -name "*.so" -exec strip --strip-all {} \;

examples/install/rlang/index.py

@@ -0,0 +1,6 @@
+import rpy2.robjects as robjects
+from rpy2.robjects import pandas2ri 
+
+def handler(event, context):  
+    pandas2ri.activate()
+    return str(robjects.r('paste0("1 + 1 = ", 1 + 1)'))

examples/install/rlang/template.yml

@@ -0,0 +1,14 @@
+ROSTemplateFormatVersion: '2015-09-01'
+Transform: 'Aliyun::Serverless-2018-04-03'
+Resources:
+  srv-01:
+    Type: 'Aliyun::Serverless::Service'
+    fun-01:
+      Type: 'Aliyun::Serverless::Function'
+      Properties:
+        Handler: index.handler
+        Runtime: python3
+        CodeUri: .
+        EnvironmentVariables:
+          LD_LIBRARY_PATH: /code/.fun/root/usr/lib/libblas
+          R_HOME: /code/.fun/R/lib/R

lib/commands/install.js

@@ -69,7 +69,7 @@ function getRuntime(options) {
 
 }
 
-async function save(runtime, codeUri, pkgType, packages) {
+async function save(runtime, codeUri, pkgType, packages, env) {
   const ymlPath = path.join(codeUri, 'fun.yml');
 
   var funModule;
@@ -84,13 +84,15 @@ async function save(runtime, codeUri, pkgType, packages) {
     case 'pip':
       funModule.addTask(new FunTask('pip', {
         pip: pkg,
-        local: true
+        local: true,
+        env
       }));
       break;
     case 'apt':
       funModule.addTask(new FunTask('apt', {
         apt: pkg,
-        local: true
+        local: true,
+        env
       }));
       break;
     default:
@@ -114,7 +116,7 @@ async function install(packages, options) {
   }
 
   if (options.save) {
-    await save(runtime, options.codeUri, pkgType, packages);
+    await save(runtime, options.codeUri, pkgType, packages, options.env);
   }
 }
 

lib/docker.js

@@ -16,6 +16,7 @@ const http = require('http');
 var containers = new Set();
 const { addEnv } = require('./install/env');
 const devnull = require('dev-null');
+const _ = require('lodash');
 
 // exit container, when use ctrl + c
 function waitingForContainerStopped() {
@@ -216,9 +217,9 @@ function resolveRuntimeToDockerImage(runtime, isBuild) {
     const name = runtimeImageMap[runtime];
     var imageName;
     if (isBuild) {
-      imageName = `aliyunfc/runtime-${name}:build-1.2.0`;
+      imageName = `aliyunfc/runtime-${name}:build-1.4.0`;
     } else {
-      imageName = `aliyunfc/runtime-${name}:1.2.0`;
+      imageName = `aliyunfc/runtime-${name}:1.4.0`;
     }
 
     debug('imageName: ' + imageName);
@@ -508,12 +509,13 @@ async function startInstallationContainer({ runtime, imageName, codeUri }) {
       await container.stop();
       containers.delete(containers.id);
     },
-    exec: async (cmd, env, verbose = false) => {
+    exec: async (cmd, {cwd = '', env = {}, verbose = false} = {}) => {      
       const options = {
         Cmd: cmd,
-        Env: env || [],
+        Env: _.map(addEnv(env), (v, k) => `${k}=${v}`),
         AttachStdout: true,
-        AttachStderr: true
+        AttachStderr: true,
+        WorkingDir: cwd
       };
 
       const exec = await container.exec(options);

lib/install/env.js

@@ -4,17 +4,19 @@ function addEnv(envVars) {
   const envs = Object.assign({}, envVars);
   const prefix = '/code/.fun/root';
 
+  const libPath = [`${prefix}/usr/lib`, `${prefix}/usr/lib/x86_64-linux-gnu`].join(':');
+  const defaultLibPath = `${libPath}:/code:/code/lib:/usr/local/lib`;
   if (envs['LD_LIBRARY_PATH']) {
-    envs['LD_LIBRARY_PATH'] = `${prefix}/usr/lib/x86_64-linux-gnu:${envs['LD_LIBRARY_PATH']}`;
+    envs['LD_LIBRARY_PATH'] = `${envs['LD_LIBRARY_PATH']}:${defaultLibPath}`;
   } else {
-    envs['LD_LIBRARY_PATH'] = `${prefix}/usr/lib/x86_64-linux-gnu:/code:/code/lib:/usr/local/lib`;
+    envs['LD_LIBRARY_PATH'] = defaultLibPath;
   }
 
   const paths = ['/usr/local/bin', '/usr/local/sbin', '/usr/bin', '/usr/sbin', '/sbin', '/bin'];
   const defaultPath = paths.join(':');
   const customPath = paths.map(p => `${prefix}${p}`).join(':') + ':/code/.fun/python/bin';
   if (envs['PATH']) {
-    envs['PATH'] = `${customPath}:${envs['PATH']}`;
+    envs['PATH'] = `${envs['PATH']}:${customPath}:${defaultPath}`;
   } else {
     envs['PATH'] = `${customPath}:${defaultPath}`;
   }

lib/install/install.js

@@ -13,11 +13,11 @@ async function installPackage(runtime, pkgType, pkgName, options) {
     switch (pkgType) {
     case 'apt':
       await new AptTask(options.name, runtime, options.codeUri, 
-        pkgName, options.local, ctx, options.verbose).run();
+        pkgName, options.local, options.env, ctx, options.verbose).run();
       break;
     case 'pip':
       await new PipTask(options.name, runtime, options.codeUri, 
-        pkgName, options.local, ctx, options.verbose).run();
+        pkgName, options.local, options.env, ctx, options.verbose).run();
       break;
     case 'module':
       // TODO
@@ -42,15 +42,15 @@ async function installFromYaml(file, verbose) {
     for (const t of funModule.tasks) {
       if (t.type === 'pip') {
         const pipTask = new PipTask(t.attrs.name, runtime, codeUri, 
-          t.attrs.pip, t.attrs.local, ctx, verbose);
+          t.attrs.pip, t.attrs.local, t.attrs.env,  ctx, verbose);
         await pipTask.run();
       } else if (t.type === 'apt') {
         const aptTask = new AptTask(t.attrs.name, runtime, codeUri, 
-          t.attrs.apt, t.attrs.local, ctx, verbose);
+          t.attrs.apt, t.attrs.local, t.attrs.env, ctx, verbose);
         await aptTask.run();
       } else if (t.type === 'shell') {
         const shellTask = new ShellTask(t.attrs.name, runtime, codeUri, 
-          t.attrs.shell, ctx, verbose);
+          t.attrs.shell, t.attrs.cwd, t.attrs.env, ctx, verbose);
         await shellTask.run();
       } else {
         console.error('unkown task %s', t);

lib/install/module.js

@@ -4,6 +4,7 @@ const yaml = require('js-yaml');
 const fs = require('fs');
 const debug = require('debug')('fun:install');
 const { red } = require('colors');
+const _ = require('lodash');
 
 class FunTask {
 
@@ -14,20 +15,36 @@ class FunTask {
 
   static parse(attrs) {
     if (attrs.pip) {
-      return new FunTask('pip', attrs);
+      return new FunTask('pip', Object.assign({ local: true }, attrs));
     } else if (attrs.apt) {
-      return new FunTask('apt', attrs);
+      return new FunTask('apt', Object.assign({ local: true }, attrs));
     } else if (attrs.shell) {
       return new FunTask('shell', attrs);
     }
     throw new Error('Unknown task.');
   }
 
   isEqual(task) {
-    if (!(task instanceof FunTask)) {return false;}
-    if (this.type !== task.type) {return false;}
+    if (!(task instanceof FunTask)) { return false; }
+    if (this.type !== task.type) { return false; }
     return this.attrs[this.type] === task.attrs[this.type];
   }
+
+  canonical() {
+    let attrs = _(this.attrs)
+      .toPairs()
+      .reject(item => {
+        switch (item[0]) {
+        case 'local': return item[1] === true;
+        case 'env': return _.isEmpty(item[1]);
+        default: return false;
+        }
+      })
+      .fromPairs()
+      .value();
+
+    return new FunTask(this.type, attrs);
+  }
 }
 class FunModule {
   constructor(runtime) {
@@ -39,7 +56,7 @@ class FunModule {
   addTask(task) {
     for (var i = 0; i < this.tasks.length; i++) {
       if (this.tasks[i].isEqual(task)) {
-        debug(`Task is already in list.`);
+        this.tasks[i] = task;
         return;
       }
     }
@@ -97,7 +114,7 @@ class FunModule {
     if (funModule.tasks) {
       doc.tasks = [];
       funModule.tasks.forEach((t) => {
-        doc.tasks.push(t.attrs);
+        doc.tasks.push(t.canonical().attrs);
       });
     }
     fs.writeFileSync(file, yaml.safeDump(doc));