Feature/source pythonpath (conan-io#624)

* adding failing test for source command

* local commands improved

* fixed broken test

* checking dst!=src folder, and improved help

conans/client/command.py

@@ -456,39 +456,48 @@ def build(self, *args):
         self._manager.build(root_path, current_path, filename=args.file, profile_name=args.profile)
 
     def package(self, *args):
-        """ Calls your conanfile.py "package" method for a specific package or
-            regenerates the existing package's manifest.
-            It will not create a new package, use 'install' or 'test_package' instead.
+        """ Calls your conanfile.py "package" method for a specific package recipe.
+            It will not create a new package, use 'install' or 'test_package' instead for
+            packages in the conan local cache, or `build' for conanfile.py in user space.
             Intended for package creators, for regenerating a package without
             recompiling the source, i.e. for troubleshooting,
             and fixing the package() method, not normal operation. It requires
             the package has been built locally, it will not re-package otherwise.
             E.g. conan package MyPackage/1.2@user/channel 9cf83afd07b678da9c1645f605875400847ff3
+
+            When used in a user space project, it will execute from the build folder specified
+            as parameter, and the current directory. This is useful while creating package recipes
+            or just for extracting artifacts from the current project, without even being a package
         """
         parser = argparse.ArgumentParser(description=self.package.__doc__, prog="conan package")
-        parser.add_argument("reference", help='package recipe reference name. '
-                            'e.g., MyPackage/1.2@user/channel')
+        parser.add_argument("reference", help='package recipe reference '
+                            'e.g. MyPkg/0.1@user/channel, or local path to the build folder'
+                            ' (relative or absolute)')
         parser.add_argument("package", nargs="?", default="",
                             help='Package ID to regenerate. e.g., '
                                  '9cf83afd07b678d38a9c1645f605875400847ff3'
                                  ' If not specified, ALL binaries for this recipe are re-packaged')
 
         args = parser.parse_args(*args)
 
+        current_path = os.getcwd()
         try:
             reference = ConanFileReference.loads(args.reference)
+            self._manager.package(reference, args.package)
         except:
-            raise ConanException("Invalid package recipe reference. "
-                                 "e.g., MyPackage/1.2@user/channel")
-
-        self._manager.package(reference, args.package)
+            if "@" in args.reference:
+                raise
+            build_folder = args.reference
+            if not os.path.isabs(build_folder):
+                build_folder = os.path.normpath(os.path.join(current_path, build_folder))
+            self._manager.local_package(current_path, build_folder)
 
     def source(self, *args):
         """ Calls your conanfile.py "source" method to configure the source directory.
             I.e., downloads and unzip the package source.
         """
         parser = argparse.ArgumentParser(description=self.source.__doc__, prog="conan source")
-        parser.add_argument("reference", help="package recipe reference. e.g., MyPackage/1.2@user/channel or ./my_project/")
+        parser.add_argument("reference", nargs='?', default="", help="package recipe reference. e.g., MyPackage/1.2@user/channel or ./my_project/")
         parser.add_argument("-f", "--force", default=False, action="store_true", help="force remove the source directory and run again.")
 
         args = parser.parse_args(*args)
@@ -497,7 +506,12 @@ def source(self, *args):
         try:
             reference = ConanFileReference.loads(args.reference)
         except:
-            reference = os.path.normpath(os.path.join(current_path, args.reference))
+            if "@" in args.reference:
+                raise
+            if not os.path.isabs(args.reference):
+                reference = os.path.normpath(os.path.join(current_path, args.reference))
+            else:
+                reference = args.reference
 
         self._manager.source(current_path, reference, args.force)
 

conans/client/generators/env.py

@@ -1,11 +1,12 @@
 from conans.model import Generator
+from conans.paths import CONANENV
 
 
 class ConanEnvGenerator(Generator):
 
     @property
     def filename(self):
-        return "conanenv.txt"
+        return CONANENV
 
     @property
     def content(self):

conans/client/installer.py

@@ -4,7 +4,7 @@
 import fnmatch
 import shutil
 
-from conans.paths import CONANINFO, BUILD_INFO
+from conans.paths import CONANINFO, BUILD_INFO, CONANENV
 from conans.util.files import save, rmdir
 from conans.model.ref import PackageReference
 from conans.util.log import logger
@@ -16,6 +16,7 @@
 from conans.model.env_info import EnvInfo
 from conans.client.file_copier import report_copied_files
 from conans.client.source import config_source
+from conans.client.generators.env import ConanEnvGenerator
 
 
 def init_package_info(deps_graph, paths):
@@ -203,6 +204,8 @@ def _build_node(self, conan_ref, conan_file, build_mode):
             output.info("Generated %s" % CONANINFO)
             save(os.path.join(build_folder, BUILD_INFO), TXTGenerator(conan_file).content)
             output.info("Generated %s" % BUILD_INFO)
+            save(os.path.join(build_folder, CONANENV), ConanEnvGenerator(conan_file).content)
+            output.info("Generated %s" % CONANENV)
 
             os.chdir(build_folder)
             create_package(conan_file, build_folder, package_folder, output)

conans/client/manager.py

@@ -2,7 +2,7 @@
 import time
 from collections import OrderedDict
 
-from conans.paths import (CONANFILE, CONANINFO, CONANFILE_TXT, BUILD_INFO)
+from conans.paths import (CONANFILE, CONANINFO, CONANFILE_TXT, BUILD_INFO, CONANENV)
 from conans.client.loader import ConanFileLoader
 from conans.client.export import export_conanfile
 from conans.client.deps_builder import DepsBuilder
@@ -317,31 +317,59 @@ def install(self, reference, current_path, remote=None, options=None, settings=N
         if manifest_manager:
             manifest_manager.print_log()
 
+    def _load_deps_info(self, current_path, conanfile, output):
+        build_info_file = os.path.join(current_path, BUILD_INFO)
+        if os.path.exists(build_info_file):
+            try:
+                deps_cpp_info = DepsCppInfo.loads(load(build_info_file))
+                conanfile.deps_cpp_info = deps_cpp_info
+            except:
+                output.error("Parse error in '%s' file in %s" % (BUILD_INFO, current_path))
+        else:
+            output.warn("%s file not found in %s\nIt is recommended for source, build and package "
+                        "commands\nYou can generate it using 'conan install -g env -g txt'"
+                        % (BUILD_INFO, current_path))
+
+        env_file = os.path.join(current_path, CONANENV)
+        if os.path.exists(env_file):
+            try:
+                deps_env_info = DepsEnvInfo.loads(load(env_file))
+                conanfile.deps_env_info = deps_env_info
+            except:
+                output.error("Parse error in '%s' file in %s" % (CONANENV, current_path))
+        else:
+            output.warn("%s file not found in %s\nIt is recommended for source, build and package "
+                        "commands\nYou can generate it using 'conan install -g env -g txt'"
+                        % (CONANENV, current_path))
+
     def source(self, current_path, reference, force):
         if not isinstance(reference, ConanFileReference):
             output = ScopedOutput("PROJECT", self._user_io.out)
             conan_file_path = os.path.join(reference, CONANFILE)
-            conan_file = self._loader().load_conan(conan_file_path, output, consumer=True)
-
-            env_file = os.path.join(current_path, "conanenv.txt")
-            if os.path.exists(env_file):
-                try:
-                    deps_env_info = DepsEnvInfo.loads(load(env_file))
-                    conan_file.deps_env_info = deps_env_info
-                except:
-                    pass
-            config_source_local(current_path, conan_file, output)
+            conanfile = self._loader().load_conan(conan_file_path, output, consumer=True)
+            self._load_deps_info(current_path, conanfile, output)
+            export_folder = reference
+            config_source_local(export_folder, current_path, conanfile, output)
         else:
             output = ScopedOutput(str(reference), self._user_io.out)
             conan_file_path = self._client_cache.conanfile(reference)
-            conan_file = self._loader().load_conan(conan_file_path, output)
-            src_folder = self._client_cache.source(reference, conan_file.short_paths)
+            conanfile = self._loader().load_conan(conan_file_path, output)
+            self._load_deps_info(current_path, conanfile, output)
+            src_folder = self._client_cache.source(reference, conanfile.short_paths)
             export_folder = self._client_cache.export(reference)
-            config_source(export_folder, src_folder, conan_file, output, force)
+            config_source(export_folder, src_folder, conanfile, output, force)
+
+    def local_package(self, current_path, build_folder):
+        if current_path == build_folder:
+            raise ConanException("Cannot 'conan package' to the build folder. "
+                                 "Please move to another folder and try again")
+        output = ScopedOutput("PROJECT", self._user_io.out)
+        conan_file_path = os.path.join(build_folder, CONANFILE)
+        conanfile = self._loader().load_conan(conan_file_path, output, consumer=True)
+        self._load_deps_info(build_folder, conanfile, output)
+        packager.create_package(conanfile, build_folder, current_path, output, local=True)
 
     def package(self, reference, package_id):
-        assert(isinstance(reference, ConanFileReference))
-
         # Package paths
         conan_file_path = self._client_cache.conanfile(reference)
         if not os.path.exists(conan_file_path):
@@ -373,6 +401,7 @@ def package(self, reference, package_id):
             output.info("Re-packaging %s" % package_reference.package_id)
             loader = self._loader(build_folder)
             conanfile = loader.load_conan(conan_file_path, self._user_io.out)
+            self._load_deps_info(build_folder, conanfile, output)
             rmdir(package_folder)
             packager.create_package(conanfile, build_folder, package_folder, output)
 
@@ -400,21 +429,7 @@ def build(self, conanfile_path, current_path, test=False, filename=None, profile
                                  "requirements and generators from '%s' file"
                                  % (CONANFILE, CONANFILE, CONANFILE_TXT))
         try:
-            build_info_file = os.path.join(current_path, BUILD_INFO)
-            if os.path.exists(build_info_file):
-                try:
-                    deps_cpp_info = DepsCppInfo.loads(load(build_info_file))
-                    conan_file.deps_cpp_info = deps_cpp_info
-                except:
-                    pass
-
-            env_file = os.path.join(current_path, "conanenv.txt")
-            if os.path.exists(env_file):
-                try:
-                    deps_env_info = DepsEnvInfo.loads(load(env_file))
-                    conan_file.deps_env_info = deps_env_info
-                except:
-                    pass
+            self._load_deps_info(current_path, conan_file, output)
 
             os.chdir(current_path)
             conan_file._conanfile_directory = conanfile_path

conans/client/packager.py

@@ -10,7 +10,7 @@
 from conans.client.file_copier import FileCopier
 
 
-def create_package(conanfile, build_folder, package_folder, output):
+def create_package(conanfile, build_folder, package_folder, output, local=False):
     """ copies built artifacts, libs, headers, data, etc from build_folder to
     package folder
     """
@@ -35,12 +35,13 @@ def new_method(pattern, src=""):
         package_output = ScopedOutput("%s package()" % output.scope, output)
         conanfile.copy.report(package_output, warn=True)
     except Exception as e:
-        os.chdir(build_folder)
-        try:
-            rmdir(package_folder)
-        except Exception as e_rm:
-            output.error("Unable to remove package folder %s\n%s" % (package_folder, str(e_rm)))
-            output.warn("**** Please delete it manually ****")
+        if not local:
+            os.chdir(build_folder)
+            try:
+                rmdir(package_folder)
+            except Exception as e_rm:
+                output.error("Unable to remove package folder %s\n%s" % (package_folder, str(e_rm)))
+                output.warn("**** Please delete it manually ****")
 
         msg = format_conanfile_exception(output.scope, "package", e)
         raise ConanException(msg)

conans/client/source.py

@@ -54,12 +54,20 @@ def remove_source(raise_error=True):
             raise ConanException(msg)
 
 
-def config_source_local(current_path, conan_file, output):
+def config_source_local(export_folder, current_path, conan_file, output):
     output.info('Configuring sources in %s' % current_path)
     dirty = os.path.join(current_path, DIRTY_FILE)
     if os.path.exists(dirty):
         output.warn("Your previous source command failed")
 
+    if current_path != export_folder:
+        for item in os.listdir(export_folder):
+            origin = os.path.join(export_folder, item)
+            if os.path.isdir(origin):
+                shutil.copytree(origin, os.path.join(current_path, item))
+            else:
+                shutil.copy2(origin, os.path.join(current_path, item))
+
     save(dirty, "")  # Creation of DIRTY flag
     try:
         conan_file.source()

conans/paths.py

@@ -26,6 +26,7 @@
 BUILD_INFO_XCODE = 'conanbuildinfo.xcconfig'
 BUILD_INFO_YCM = '.ycm_extra_conf.py'
 CONANINFO = "conaninfo.txt"
+CONANENV = "conanenv.txt"
 SYSTEM_REQS = "system_reqs.txt"
 DIRTY_FILE = ".conan_dirty"
 

conans/test/integration/package_command_test.py

@@ -3,15 +3,16 @@
 from conans.model.ref import ConanFileReference, PackageReference
 import os
 from conans.paths import CONANFILE
-from conans.util.files import save
+from conans.util.files import mkdir, load
+from conans.test.utils.test_files import temp_folder
 
 
 class PackageCommandTest(unittest.TestCase):
 
     def package_errors_test(self):
         client = TestClient()
         client.run("package whatever@user/channel", ignore_error=True)
-        self.assertIn("Invalid package recipe reference", client.user_io.out)
+        self.assertIn("Wrong package recipe", client.user_io.out)
 
         client.run("package whatever/1.0@user/channel", ignore_error=True)
         self.assertIn("ERROR: Package recipe 'whatever/1.0@user/channel' does not exist",
@@ -40,9 +41,84 @@ class MyConan(ConanFile):
         self.assertIn("ERROR: MyLib/0.1@lasote/stable: Package binary '1234' folder doesn't exist",
                       client.user_io.out)
 
+    def local_package_test(self):
+        """Use 'conan package' to process locally the package method"""
+        client = TestClient()
+        conanfile_template = """
+from conans import ConanFile
+
+class MyConan(ConanFile):
+    def package(self):
+        self.copy(pattern="*.h", dst="include", src="include")
+"""
+        files = {"include/file.h": "foo",
+                 CONANFILE: conanfile_template}
+
+        client.save(files)
+        client.run("install -g env -g txt")
+        client.run("build")
+        origin_folder = client.current_folder
+        client.current_folder = temp_folder()
+        client.run('package "%s"' % origin_folder)
+        content = load(os.path.join(client.current_folder, "include/file.h"))
+        self.assertEqual(content, "foo")
+
+    def local_package_build_test(self):
+        """Use 'conan package' to process locally the package method"""
+        client = TestClient()
+        conanfile_template = """
+from conans import ConanFile
+
+class MyConan(ConanFile):
+    def package(self):
+        self.copy(pattern="*.h", dst="include", src="include")
+"""
+        files = {"include/file.h": "foo",
+                 CONANFILE: conanfile_template}
+
+        client.save(files)
+        origin_folder = client.current_folder
+        build_folder = os.path.join(client.current_folder, "build")
+        mkdir(build_folder)
+        client.current_folder = build_folder
+        client.run("install .. -g env -g txt")
+        client.run("source ..")
+        client.run("build ..")
+        client.current_folder = temp_folder()
+        client.run('package "%s/build"' % origin_folder)
+        content = load(os.path.join(client.current_folder, "include/file.h"))
+        self.assertEqual(content, "foo")
+
+    def local_flow_test(self):
+        """Use 'conan package' to process locally the package method"""
+        client = TestClient()
+        conanfile_template = """
+from conans import ConanFile
+
+class MyConan(ConanFile):
+    def package(self):
+        self.copy(pattern="*.h", dst="include", src="include")
+"""
+        files = {"include/file.h": "foo",
+                 CONANFILE: conanfile_template}
+
+        client.save(files)
+        origin_folder = client.current_folder
+        client.run("install -g env -g txt")
+        client.run("source")
+        client.run("build")
+        client.run("package .", ignore_error=True)
+        self.assertIn("ERROR: Cannot 'conan package' to the build folder", client.user_io.out)
+        package_folder = os.path.join(origin_folder, "package")
+        mkdir(package_folder)
+        client.current_folder = package_folder
+        client.run('package ..')
+        content = load(os.path.join(client.current_folder, "include/file.h"))
+        self.assertEqual(content, "foo")
+
     def package_test(self):
         """Use 'conan package' command to repackage a generated package (without build it)"""
-        client = TestClient(users={"default": [("lasote", "mypass")]})
+        client = TestClient()
         conanfile_template = """
 from conans import ConanFile