download refactor (conan-io#2932)

* download refactor

* more refactors

* minor changes to manifest test

conans/client/cmd/download.py

@@ -0,0 +1,17 @@
+from conans.client.loader_parse import load_conanfile_class
+from conans.model.ref import PackageReference
+import os
+
+
+def download_binaries(reference, package_ids, client_cache, remote_manager, remote, output, recorder):
+    conanfile_path = client_cache.conanfile(reference)
+    if not os.path.exists(conanfile_path):
+        raise Exception("Download recipe first")
+    conanfile = load_conanfile_class(conanfile_path)
+    short_paths = conanfile.short_paths
+
+    for package_id in package_ids:
+        package_ref = PackageReference(reference, package_id)
+        package_folder = client_cache.package(package_ref, short_paths=short_paths)
+        output.info("Downloading %s" % str(package_ref))
+        remote_manager.get_package(package_ref, package_folder, remote, output, recorder)

conans/client/installer.py

@@ -4,7 +4,8 @@
 import platform
 
 from conans.client import tools
-from conans.client.recorder.action_recorder import INSTALL_ERROR_MISSING_BUILD_FOLDER, INSTALL_ERROR_BUILDING
+from conans.client.recorder.action_recorder import INSTALL_ERROR_MISSING_BUILD_FOLDER, INSTALL_ERROR_BUILDING,\
+    INSTALL_ERROR_MISSING
 from conans.model.conan_file import get_env_context_manager
 from conans.model.env_info import EnvInfo
 from conans.model.user_info import UserInfo
@@ -25,8 +26,7 @@
 
 from conans.util.tracer import log_package_built
 from conans.client.tools.env import pythonpath
-from conans.client.package_installer import raise_package_not_found_error,\
-    get_package
+from conans.client.package_installer import get_package
 
 
 def build_id(conan_file):
@@ -215,6 +215,24 @@ def call_system_requirements(conanfile, output):
         raise ConanException("Error in system requirements")
 
 
+def raise_package_not_found_error(conan_file, conan_ref, package_id, out, recorder, remote_url):
+    settings_text = ", ".join(conan_file.info.full_settings.dumps().splitlines())
+    options_text = ", ".join(conan_file.info.full_options.dumps().splitlines())
+
+    msg = '''Can't find a '%s' package for the specified options and settings:
+- Settings: %s
+- Options: %s
+- Package ID: %s
+''' % (conan_ref, settings_text, options_text, package_id)
+    out.warn(msg)
+    recorder.package_install_error(PackageReference(conan_ref, package_id),
+                                   INSTALL_ERROR_MISSING, msg, remote=remote_url)
+    raise ConanException('''Missing prebuilt package for '%s'
+Try to build it from sources with "--build %s"
+Or read "http://docs.conan.io/en/latest/faq/troubleshooting.html#error-missing-prebuilt-package"
+''' % (conan_ref, conan_ref.name))
+
+
 class ConanInstaller(object):
     """ main responsible of retrieving binary packages or building them from source
     locally in case they are not found in remotes

conans/client/manager.py

@@ -28,6 +28,7 @@
 from conans.util.log import logger
 from conans.client.loader_parse import load_conanfile_class
 from conans.client.graph.build_mode import BuildMode
+from conans.client.cmd.download import download_binaries
 
 
 class ConanManager(object):
@@ -145,14 +146,15 @@ def download(self, reference, package_ids, remote_name, recipe):
         @param only_recipe: download only the recipe
         """
         assert(isinstance(reference, ConanFileReference))
-        remote_proxy = self.get_proxy(remote_name=remote_name)
-        remote, _ = remote_proxy._get_remote()
+        output = ScopedOutput(str(reference), self._user_io.out)
+        remote = self._registry.remote(remote_name) if remote_name else self._registry.default_remote
         package = self._remote_manager.search_recipes(remote, reference, None)
         if not package:  # Search the reference first, and raise if it doesn't exist
             raise ConanException("'%s' not found in remote" % str(reference))
 
         # First of all download package recipe
-        remote_proxy.get_recipe(reference, check_updates=True, update=True)
+        self._remote_manager.get_recipe(reference, remote)
+        self._registry.set_ref(reference, remote)
 
         if recipe:
             return
@@ -164,16 +166,18 @@ def download(self, reference, package_ids, remote_name, recipe):
                                 conanfile, reference)
 
         if package_ids:
-            remote_proxy.download_packages(reference, package_ids)
+            download_binaries(reference, package_ids, self._client_cache, self._remote_manager,
+                              remote, output, self._recorder)
         else:
-            self._user_io.out.info("Getting the complete package list "
-                                   "from '%s'..." % str(reference))
+            output.info("Getting the complete package list "
+                        "from '%s'..." % str(reference))
             packages_props = self._remote_manager.search_packages(remote, reference, None)
             if not packages_props:
                 output = ScopedOutput(str(reference), self._user_io.out)
                 output.warn("No remote binary packages found in remote")
             else:
-                remote_proxy.download_packages(reference, list(packages_props.keys()))
+                download_binaries(reference, list(packages_props.keys()), self._client_cache,
+                                  self._remote_manager, remote, output, self._recorder)
 
     @staticmethod
     def _inject_require(conanfile, inject_require):

conans/client/package_installer.py

@@ -1,67 +1,33 @@
 import os
 
-from conans.client.recorder.action_recorder import INSTALL_ERROR_MISSING
-from conans.errors import (ConanException, NotFoundException, NoRemoteAvailable)
-from conans.model.ref import PackageReference
-from conans.util.files import rmdir, make_read_only
+from conans.errors import NotFoundException, NoRemoteAvailable
+from conans.util.files import rmdir
 from conans.util.tracer import log_package_got_from_local_cache
 from conans.model.manifest import FileTreeManifest
-from conans.util.env_reader import get_env
-
-
-def raise_package_not_found_error(conan_file, conan_ref, package_id, out, recorder, remote_url):
-    settings_text = ", ".join(conan_file.info.full_settings.dumps().splitlines())
-    options_text = ", ".join(conan_file.info.full_options.dumps().splitlines())
-
-    msg = '''Can't find a '%s' package for the specified options and settings:
-- Settings: %s
-- Options: %s
-- Package ID: %s
-''' % (conan_ref, settings_text, options_text, package_id)
-    out.warn(msg)
-    recorder.package_install_error(PackageReference(conan_ref, package_id),
-                                   INSTALL_ERROR_MISSING, msg, remote=remote_url)
-    raise ConanException('''Missing prebuilt package for '%s'
-Try to build it from sources with "--build %s"
-Or read "http://docs.conan.io/en/latest/faq/troubleshooting.html#error-missing-prebuilt-package"
-''' % (conan_ref, conan_ref.name))
 
 
 def get_package(conanfile, package_ref, package_folder, output, recorder, proxy, update):
     # TODO: This access to proxy attributes has to be improved
     remote_manager = proxy._remote_manager
     registry = proxy.registry
-    try:
-        if update:
-            _remove_if_outdated(package_folder, package_ref, proxy, output)
-        local_package = os.path.exists(package_folder)
-        if local_package:
-            output.success('Already installed!')
-            log_package_got_from_local_cache(package_ref)
-            recorder.package_fetched_from_cache(package_ref)
-            return False
-        else:
-            remote = registry.get_ref(package_ref.conan)
-            # remote will be defined, as package availability has been checked from installer
-            remote_manager.get_package(conanfile, package_ref, package_folder, remote, output, recorder)
-            if get_env("CONAN_READ_ONLY_CACHE", False):
-                make_read_only(package_folder)
-            recorder.package_downloaded(package_ref, remote.url)
-            return True
-    except BaseException as e:
-        output.error("Exception while getting package: %s" % str(package_ref.package_id))
-        output.error("Exception: %s %s" % (type(e), str(e)))
-        _clean_package(package_folder, output)
-        raise
-
-
-def _clean_package(package_folder, output):
-    try:
-        output.warn("Trying to remove package folder: %s" % package_folder)
-        rmdir(package_folder)
-    except OSError as e:
-        raise ConanException("%s\n\nCouldn't remove folder '%s', might be busy or open. Close any app "
-                             "using it, and retry" % (str(e), package_folder))
+    if update:
+        _remove_if_outdated(package_folder, package_ref, proxy, output)
+    local_package = os.path.exists(package_folder)
+    if local_package:
+        output.success('Already installed!')
+        log_package_got_from_local_cache(package_ref)
+        recorder.package_fetched_from_cache(package_ref)
+        return False
+    else:
+        remote = registry.get_ref(package_ref.conan)
+        # remote will be defined, as package availability has been checked from installer
+        try:
+            remote_manager.get_package(package_ref, package_folder, remote, output, recorder)
+        except NotFoundException:
+            from conans.client.installer import raise_package_not_found_error
+            raise_package_not_found_error(conanfile, package_ref.conan, package_ref.package_id,
+                                          output, recorder, remote.url)
+        return True
 
 
 def _remove_if_outdated(package_folder, package_ref, proxy, output):

conans/client/proxy.py

@@ -235,21 +235,3 @@ def search_remotes(self, pattern=None, ignorecase=True):
             search_result = self._remote_manager.search_recipes(remote, pattern, ignorecase)
             if search_result:
                 return search_result
-
-    def download_packages(self, reference, package_ids):
-        assert(isinstance(package_ids, list))
-        remote, _ = self._get_remote(reference)
-        conanfile_path = self._client_cache.conanfile(reference)
-        if not os.path.exists(conanfile_path):
-            raise Exception("Download recipe first")
-        conanfile = load_conanfile_class(conanfile_path)
-        # FIXME: This is a hack to provide a info object in case it fails and raise_package_not_found_error doesnt fail
-        conanfile.info = ConanInfo.loads("")
-        short_paths = conanfile.short_paths
-        self._registry.set_ref(reference, remote)
-        output = ScopedOutput(str(reference), self._out)
-        for package_id in package_ids:
-            package_ref = PackageReference(reference, package_id)
-            package_folder = self._client_cache.package(package_ref, short_paths=short_paths)
-            self._out.info("Downloading %s" % str(package_ref))
-            self._remote_manager.get_package(conanfile, package_ref, package_folder, remote, output, self._recorder)

conans/client/remote_manager.py

@@ -10,7 +10,8 @@
 from conans.model.manifest import gather_files
 from conans.paths import PACKAGE_TGZ_NAME, CONANINFO, CONAN_MANIFEST, CONANFILE, EXPORT_TGZ_NAME, \
     rm_conandir, EXPORT_SOURCES_TGZ_NAME, EXPORT_SOURCES_DIR_OLD
-from conans.util.files import gzopen_without_timestamps, is_dirty
+from conans.util.files import gzopen_without_timestamps, is_dirty,\
+    make_read_only
 from conans.util.files import tar_extract, rmdir, exception_message_safe, mkdir
 from conans.util.files import touch_folder
 from conans.util.log import logger
@@ -20,8 +21,8 @@
                                 log_uncompressed_file, log_compressed_files, log_recipe_download,
                                 log_package_download)
 from conans.client.source import merge_directories
-from conans.client.package_installer import raise_package_not_found_error
 import stat
+from conans.util.env_reader import get_env
 
 
 class RemoteManager(object):
@@ -215,25 +216,35 @@ def filter_function(urls):
             rmdir(c_src_path)
         touch_folder(export_sources_folder)
 
-    def get_package(self, conanfile, package_reference, dest_folder, remote, output, recorder):
+    def get_package(self, package_reference, dest_folder, remote, output, recorder):
         package_id = package_reference.package_id
         output.info("Retrieving package %s from remote '%s' " % (package_id, remote.name))
         rm_conandir(dest_folder)  # Remove first the destination folder
         t1 = time.time()
         try:
             urls = self._call_remote(remote, "get_package_urls", package_reference)
             zipped_files = self._call_remote(remote, "download_files_to_folder", urls, dest_folder)
-        except NotFoundException as e:
-            output.warn('Binary for %s not in remote: %s' % (package_id, str(e)))
-            raise_package_not_found_error(conanfile, package_reference.conan,
-                                          package_id, output, recorder, remote.url)
-        else:
             duration = time.time() - t1
             log_package_download(package_reference, duration, remote, zipped_files)
             unzip_and_get_files(zipped_files, dest_folder, PACKAGE_TGZ_NAME)
             # Issue #214 https://github.com/conan-io/conan/issues/214
             touch_folder(dest_folder)
+            if get_env("CONAN_READ_ONLY_CACHE", False):
+                make_read_only(dest_folder)
             output.success('Package installed %s' % package_id)
+            recorder.package_downloaded(package_reference, remote.url)
+        except NotFoundException:
+            raise NotFoundException("Package binary '%s' not found in '%s'" % (package_reference, remote.name))
+        except BaseException as e:
+            output.error("Exception while getting package: %s" % str(package_reference.package_id))
+            output.error("Exception: %s %s" % (type(e), str(e)))
+            try:
+                output.warn("Trying to remove package folder: %s" % dest_folder)
+                rmdir(dest_folder)
+            except OSError as e:
+                raise ConanException("%s\n\nCouldn't remove folder '%s', might be busy or open. Close any app "
+                                     "using it, and retry" % (str(e), dest_folder))
+            raise
 
     def search_recipes(self, remote, pattern=None, ignorecase=True):
         """

conans/test/command/download_test.py

@@ -152,5 +152,4 @@ class Pkg(ConanFile):
 
         error = client.run("download pkg/0.1@lasote/stable -p=wrong", ignore_error=True)
         self.assertTrue(error)
-        self.assertIn("ERROR: Missing prebuilt package for 'pkg/0.1@lasote/stable'", client.out)
-        self.assertIn("Package ID: wrong", client.out)
+        self.assertIn("ERROR: Package binary 'pkg/0.1@lasote/stable:wrong' not found in 'default'", client.out)

conans/test/integration/manifest_validation_test.py

@@ -153,7 +153,7 @@ def remote_capture_verify_manifest_test(self):
         self.client.run("remove Hello* -f")
         files = {"conanfile.txt": "[requires]\nHello/0.1@lasote/stable"}
         self.client.save(files, clean_first=True)
-        self._capture_verify_manifest(".", remote="default:")
+        self._capture_verify_manifest(".", remote="default")
 
     def _failed_verify(self, reference, remote="local cache"):
         self.client.run("install %s --build missing --manifests" % str(reference))
@@ -283,7 +283,7 @@ def test_corrupted_package(self):
                                                    "5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9")
         package_path = self.client.paths.package(package_reference)
         file_path = os.path.join(package_path, "conaninfo.txt")
-        save(file_path, load(file_path) + "RANDOM STRING")
+        save(file_path, load(file_path) + "  ")
 
         self.client.run("install %s --build missing --manifests" % str(self.reference),
                         ignore_error=True)