Feature/short paths package (conan-io#563)

* adding short_paths to package folder

* extending the short_paths solution to package folder

conans/client/client_cache.py

@@ -5,9 +5,9 @@
 from conans.model.values import Values
 from conans.client.detect import detect_defaults_settings
 from conans.model.ref import ConanFileReference
-from os.path import isfile
 from conans.model.manifest import FileTreeManifest
-from conans.paths import CONAN_MANIFEST, SimplePaths
+from conans.paths import SimplePaths
+from genericpath import isdir
 
 CONAN_CONF = 'conan.conf'
 CONAN_SETTINGS = "settings.yml"
@@ -84,32 +84,43 @@ def package_paths(self, package_reference):
         return relative_dirs(self.package(package_reference))
 
     def conan_packages(self, conan_reference):
-        """ Returns a list of package_id from a conans """
+        """ Returns a list of package_id from a local cache package folder """
         assert isinstance(conan_reference, ConanFileReference)
         packages_dir = self.packages(conan_reference)
         try:
             packages = [dirname for dirname in os.listdir(packages_dir)
-                        if not isfile(os.path.join(packages_dir, dirname))]
+                        if isdir(os.path.join(packages_dir, dirname))]
         except:  # if there isn't any package folder
             packages = []
         return packages
 
+    def conan_builds(self, conan_reference):
+        """ Returns a list of package ids from a local cache build folder """
+        assert isinstance(conan_reference, ConanFileReference)
+        builds_dir = self.builds(conan_reference)
+        try:
+            builds = [dirname for dirname in os.listdir(builds_dir)
+                      if isdir(os.path.join(builds_dir, dirname))]
+        except:  # if there isn't any package folder
+            builds = []
+        return builds
+
     def load_manifest(self, conan_reference):
         '''conan_id = sha(zip file)'''
         filename = self.digestfile_conanfile(conan_reference)
         return FileTreeManifest.loads(load(filename))
 
     def load_package_manifest(self, package_reference):
         '''conan_id = sha(zip file)'''
-        filename = self.digestfile_package(package_reference)
+        filename = self.digestfile_package(package_reference, short_paths=None)
         return FileTreeManifest.loads(load(filename))
 
     def conan_manifests(self, conan_reference):
         digest_path = self.digestfile_conanfile(conan_reference)
         return self._digests(digest_path)
 
     def package_manifests(self, package_reference):
-        digest_path = self.digestfile_package(package_reference)
+        digest_path = self.digestfile_package(package_reference, short_paths=None)
         return self._digests(digest_path)
 
     def _digests(self, digest_path):

conans/client/export.py

@@ -12,7 +12,8 @@
 from conans.client.file_copier import FileCopier
 
 
-def export_conanfile(output, paths, file_patterns, origin_folder, conan_ref, shorten, keep_source):
+def export_conanfile(output, paths, file_patterns, origin_folder, conan_ref, short_paths,
+                     keep_source):
     destination_folder = paths.export(conan_ref)
 
     previous_digest = _init_export_folder(destination_folder)
@@ -31,7 +32,7 @@ def export_conanfile(output, paths, file_patterns, origin_folder, conan_ref, sho
         output.info('Folder: %s' % destination_folder)
         modified_recipe = True
 
-    source = paths.source(conan_ref, shorten)
+    source = paths.source(conan_ref, short_paths)
     dirty = os.path.join(source, DIRTY_FILE)
     remove = False
     if os.path.exists(dirty):
@@ -44,7 +45,7 @@ def export_conanfile(output, paths, file_patterns, origin_folder, conan_ref, sho
     if remove:
         output.info("Removing 'source' folder, this can take a while for big packages")
         try:
-            rmdir(source, shorten)
+            rmdir(source, short_paths)
         except BaseException as e:
             output.error("Unable to delete source folder. "
                          "Will be marked as dirty for deletion")

conans/client/importer.py

@@ -43,7 +43,8 @@ def _get_folders(self):
                 continue
             package_id = conan_file.info.package_id()
             package_reference = PackageReference(conan_ref, package_id)
-            package_folders[conan_file.name] = self._paths.package(package_reference)
+            short_paths = "check" if conan_file.short_paths else False
+            package_folders[conan_file.name] = self._paths.package(package_reference, short_paths)
         return package_folders
 
     def _get_paths(self, conan_name_pattern):

conans/client/installer.py

@@ -28,7 +28,7 @@ def init_package_info(deps_graph, paths):
         if conan_ref:
             package_id = conan_file.info.package_id()
             package_reference = PackageReference(conan_ref, package_id)
-            package_folder = paths.package(package_reference)
+            package_folder = paths.package(package_reference, conan_file.short_paths)
             conan_file.package_folder = package_folder
             conan_file.cpp_info = CppInfo(package_folder)
             conan_file.env_info = EnvInfo(package_folder)
@@ -73,7 +73,8 @@ def _compute_private_nodes(self, deps_graph, build_mode):
                 package_id = conan_file.info.package_id()
                 package_reference = PackageReference(conan_ref, package_id)
                 build_forced = self._build_forced(conan_ref, build_mode, conan_file)
-                if not self._remote_proxy.get_package(package_reference, build_forced):
+                if not self._remote_proxy.get_package(package_reference, build_forced,
+                                                      short_paths=conan_file.short_paths):
                     break
             else:
                 skippable_nodes.append(private_node)
@@ -148,7 +149,7 @@ def _build_node(self, conan_ref, conan_file, build_mode):
         package_reference = PackageReference(conan_ref, package_id)
 
         conan_ref = package_reference.conan
-        package_folder = self._paths.package(package_reference)
+        package_folder = self._paths.package(package_reference, conan_file.short_paths)
         build_folder = self._paths.build(package_reference, conan_file.short_paths)
         src_folder = self._paths.source(conan_ref, conan_file.short_paths)
         export_folder = self._paths.export(conan_ref)
@@ -162,7 +163,8 @@ def _build_node(self, conan_ref, conan_file, build_mode):
         self._handle_system_requirements(conan_ref, package_reference, conan_file, output)
 
         force_build = self._build_forced(conan_ref, build_mode, conan_file)
-        if self._remote_proxy.get_package(package_reference, force_build):
+        if self._remote_proxy.get_package(package_reference, force_build,
+                                          short_paths=conan_file.short_paths):
             return
 
         # we need and can build? Only if we are forced or build_mode missing and package not exists
@@ -286,12 +288,12 @@ def check_max_path_len(src, files):
             conan_file._conanfile_directory = build_folder
             conan_file.build()
             self._out.writeln("")
-            output.success("Package '%s' built" % os.path.basename(build_folder))
+            output.success("Package '%s' built" % conan_file.info.package_id())
             output.info("Build folder %s" % build_folder)
         except Exception as e:
             os.chdir(src_folder)
             self._out.writeln("")
-            output.error("Package '%s' build failed" % os.path.basename(build_folder))
+            output.error("Package '%s' build failed" % conan_file.info.package_id())
             output.warn("Build folder %s" % build_folder)
             raise ConanException("%s: %s" % (conan_file.name, str(e)))
         finally:

conans/client/manager.py

@@ -305,7 +305,8 @@ def package(self, reference, package_id, only_manifest, package_all):
             packages = [PackageReference(reference, package_id)]
 
         for package_reference in packages:
-            package_folder = self._client_cache.package(package_reference)
+            # The package already exist, we can use short_paths if they were defined
+            package_folder = self._client_cache.package(package_reference, short_paths=None)
             # Will read current conaninfo with specified options and load conanfile with them
             if not only_manifest:
                 self._user_io.out.info("Packaging %s" % package_reference.package_id)

conans/client/manifest_manager.py

@@ -72,5 +72,5 @@ def check_package(self, package_reference, remote):
         remote = "local cache" if not remote else "%s:%s" % (remote.name, remote.url)
         self._match_manifests(read_manifest, expected_manifest, package_reference)
 
-        path = self._paths.digestfile_package(package_reference)
+        path = self._paths.digestfile_package(package_reference, short_paths=None)
         self._check(package_reference, read_manifest, remote, path)

conans/client/proxy.py

@@ -28,12 +28,12 @@ def __init__(self, client_cache, user_io, remote_manager, remote_name,
     def registry(self):
         return self._registry
 
-    def get_package(self, package_reference, force_build):
+    def get_package(self, package_reference, force_build, short_paths):
         """ obtain a package, either from disk or retrieve from remotes if necessary
         and not necessary to build
         """
         output = ScopedOutput(str(package_reference.conan), self._out)
-        package_folder = self._client_cache.package(package_reference)
+        package_folder = self._client_cache.package(package_reference, short_paths=short_paths)
 
         # Check current package status
         if path_exists(package_folder, self._client_cache.store):

conans/client/remote_manager.py

@@ -39,7 +39,8 @@ def upload_conan(self, conan_reference, remote):
     def upload_package(self, package_reference, remote):
         """Will upload the package to the first remote"""
         t1 = time.time()
-        basedir = self._client_cache.package(package_reference)
+        # existing package, will use short paths if defined
+        basedir = self._client_cache.package(package_reference, short_paths=None)
         rel_files = self._client_cache.package_paths(package_reference)
 
         self._output.rewrite_line("Checking package integrity...")

conans/client/remover.py

@@ -1,43 +1,23 @@
 import os
-import platform
 
 from conans.errors import ConanException
 from conans.util.log import logger
 from conans.model.ref import PackageReference
 from conans.paths import SYSTEM_REQS
-from conans.util.files import rmdir, load
+from conans.util.files import rmdir
 from conans.model.ref import ConanFileReference
 
 
 class DiskRemover(object):
     def __init__(self, paths):
         self._paths = paths
 
-    def _remove_source_short_paths(self, path, conan_ref):
-        if platform.system() != "Windows":
-            return
-
-        link = os.path.join(path, "source/.conan_link")
-        if os.path.exists(link):
-            self._remove(load(link), conan_ref)
-
-    def _remove_builds_short_paths(self, path, conan_ref):
-        if platform.system() != "Windows":
-            return
-
-        build = os.path.join(path, "build")
-        if os.path.exists(build):
-            for f in os.listdir(build):
-                link = os.path.join(build, f, ".conan_link")
-                if os.path.exists(link):
-                    self._remove(load(link), conan_ref)
-
     def _remove(self, path, conan_ref, msg=""):
         try:
             logger.debug("Removing folder %s" % path)
-            rmdir(path)
+            rmdir(path, short_paths=True)
         except OSError as e:
-            raise ConanException("Unable to remove %s %s\n\t%s" % (repr(conan_ref), msg, str(e)))
+            raise ConanException("%s: Unable to remove %s\n\t%s" % (repr(conan_ref), msg, str(e)))
 
     def _remove_file(self, path, conan_ref, msg=""):
         try:
@@ -48,29 +28,31 @@ def _remove_file(self, path, conan_ref, msg=""):
             raise ConanException("Unable to remove %s %s\n\t%s" % (repr(conan_ref), msg, str(e)))
 
     def remove(self, conan_ref):
-        path = self._paths.conan(conan_ref)
-        self._remove_source_short_paths(path, conan_ref)
-        self._remove_builds_short_paths(path, conan_ref)
-        self._remove(path, conan_ref)
+        self.remove_src(conan_ref)
+        self.remove_builds(conan_ref)
+        self.remove_packages(conan_ref)
+        self._remove(self._paths.conan(conan_ref), conan_ref)
 
     def remove_src(self, conan_ref):
-        path = self._paths.conan(conan_ref)
-        self._remove_source_short_paths(path, conan_ref)
         self._remove(self._paths.source(conan_ref), conan_ref, "src folder")
 
     def remove_builds(self, conan_ref, ids=None):
         if not ids:
-            path = self._paths.conan(conan_ref)
-            self._remove_builds_short_paths(path, conan_ref)
-            self._remove(self._paths.builds(conan_ref), conan_ref, "builds")
+            path = self._paths.builds(conan_ref)
+            for build in self._paths.conan_builds(conan_ref):
+                self._remove(os.path.join(path, build), conan_ref, "build folder:%s" % build)
+            self._remove(path, conan_ref, "builds")
         else:
             for id_ in ids:
                 self._remove(self._paths.build(PackageReference(conan_ref, id_)), conan_ref,
                              "package:%s" % id_)
 
     def remove_packages(self, conan_ref, ids_filter=None):
         if not ids_filter:  # Remove all
-            self._remove(self._paths.packages(conan_ref), conan_ref, "packages")
+            path = self._paths.packages(conan_ref)
+            for package in self._paths.conan_packages(conan_ref):
+                self._remove(os.path.join(path, package), conan_ref, "package folder:%s" % package)
+            self._remove(path, conan_ref, "packages")
             self._remove_file(self._paths.system_reqs(conan_ref), conan_ref, SYSTEM_REQS)
         else:
             for id_ in ids_filter:  # remove just the specified packages

conans/paths.py

@@ -4,6 +4,7 @@
 from os.path import join, normpath
 import platform
 import tempfile
+from conans.errors import ConanException
 
 
 EXPORT_FOLDER = "export"
@@ -53,12 +54,23 @@ def conan_expand_user(path):
     return os.path.expanduser(path)
 
 
-def shortener(path, shorten):
-    if not shorten:
+def shortener(path, short_paths):
+    """ short_paths is 4-state:
+    False: Never shorten the path
+    True: Always shorten the path, create link if not existing
+    None: Use shorten path only if already exists, not create
+    Other: Integrity check. Consumer knows it should be short, but it isn't
+    """
+    if short_paths is False:
         return path
     link = os.path.join(path, ".conan_link")
     if os.path.exists(link):
         return load(link)
+    elif short_paths is None:
+        return path
+    elif short_paths is not True:
+        raise ConanException("This path should be short, but it isn't: %s\n"
+                             "Try to remove these packages and re-build them" % path)
 
     drive = os.path.splitdrive(path)[0]
     short_path = drive + "/.conan"
@@ -97,10 +109,10 @@ def export(self, conan_reference):
         assert isinstance(conan_reference, ConanFileReference)
         return normpath(join(self.conan(conan_reference), EXPORT_FOLDER))
 
-    def source(self, conan_reference, shorten=False):
+    def source(self, conan_reference, short_paths=False):
         assert isinstance(conan_reference, ConanFileReference)
         p = normpath(join(self.conan(conan_reference), SRC_FOLDER))
-        return self._shortener(p, shorten)
+        return self._shortener(p, short_paths)
 
     def conanfile(self, conan_reference):
         export = self.export(conan_reference)
@@ -110,19 +122,19 @@ def digestfile_conanfile(self, conan_reference):
         export = self.export(conan_reference)
         return normpath(join(export, CONAN_MANIFEST))
 
-    def digestfile_package(self, package_reference):
+    def digestfile_package(self, package_reference, short_paths=False):
         assert isinstance(package_reference, PackageReference)
-        return normpath(join(self.package(package_reference), CONAN_MANIFEST))
+        return normpath(join(self.package(package_reference, short_paths), CONAN_MANIFEST))
 
     def builds(self, conan_reference):
         assert isinstance(conan_reference, ConanFileReference)
         return normpath(join(self.conan(conan_reference), BUILD_FOLDER))
 
-    def build(self, package_reference, shorten=False):
+    def build(self, package_reference, short_paths=False):
         assert isinstance(package_reference, PackageReference)
         p = normpath(join(self.conan(package_reference.conan), BUILD_FOLDER,
                           package_reference.package_id))
-        return self._shortener(p, shorten)
+        return self._shortener(p, short_paths)
 
     def system_reqs(self, conan_reference):
         assert isinstance(conan_reference, ConanFileReference)
@@ -137,8 +149,8 @@ def packages(self, conan_reference):
         assert isinstance(conan_reference, ConanFileReference)
         return normpath(join(self.conan(conan_reference), PACKAGES_FOLDER))
 
-    def package(self, package_reference):
+    def package(self, package_reference, short_paths=False):
         assert isinstance(package_reference, PackageReference)
-        return normpath(join(self.conan(package_reference.conan), PACKAGES_FOLDER,
-                             package_reference.package_id))
-
+        p = normpath(join(self.conan(package_reference.conan), PACKAGES_FOLDER,
+                          package_reference.package_id))
+        return self._shortener(p, short_paths)

conans/search.py

@@ -108,7 +108,9 @@ def search_packages(self, reference, query):
         for package_id in subdirs:
             try:
                 package_reference = PackageReference(reference, package_id)
-                info_path = self._adapter.join_paths(self._paths.package(package_reference), CONANINFO)
+                info_path = self._adapter.join_paths(self._paths.package(package_reference,
+                                                                         short_paths=None),
+                                                     CONANINFO)
                 if not self._adapter.path_exists(info_path, self._paths.store):
                     raise NotFoundException("")
                 conan_info_content = self._adapter.load(info_path)
@@ -170,4 +172,3 @@ def get_properties_from_query(query):
                 logger.error(exc)
                 raise ConanException("Invalid package query: %s" % query)
     return properties
-

conans/test/download_test.py

@@ -67,7 +67,7 @@ def complete_test(self):
         installer = ConanProxy(client2.paths, client2.user_io, client2.remote_manager, "default")
 
         installer.get_recipe(conan_ref)
-        installer.get_package(package_ref, force_build=False)
+        installer.get_package(package_ref, force_build=False, short_paths=False)
         # Check that the output is done in order
         lines = [line.strip() for line in str(client2.user_io.out).splitlines()
                  if line.startswith("Downloading")]