import os import queue import sys import warnings from collections import defaultdict from tempfile import NamedTemporaryFile from pipenv import environments, exceptions from pipenv.patched.pip._internal.exceptions import PipError from pipenv.routines.lock import do_lock from pipenv.utils import console, err, fileutils from pipenv.utils.dependencies import ( expansive_install_req_from_line, get_lockfile_section_using_pipfile_category, install_req_from_pipfile, ) from pipenv.utils.indexes import get_source_list from pipenv.utils.internet import download_file, is_valid_url from pipenv.utils.pip import ( get_trusted_hosts, pip_install_deps, ) from pipenv.utils.pipfile import ensure_pipfile from pipenv.utils.project import ensure_project from pipenv.utils.requirements import add_index_to_pipfile, import_requirements from pipenv.utils.shell import temp_environ from pipenv.utils.virtualenv import cleanup_virtualenv, do_create_virtualenv def do_install( project, packages=False, editable_packages=False, index=False, dev=False, python=False, pypi_mirror=None, system=False, ignore_pipfile=False, requirementstxt=False, pre=False, deploy=False, site_packages=None, extra_pip_args=None, categories=None, skip_lock=False, ): requirements_directory = fileutils.create_tracked_tempdir( suffix="-requirements", prefix="pipenv-" ) warnings.filterwarnings("default", category=ResourceWarning) packages = packages if packages else [] editable_packages = editable_packages if editable_packages else [] package_args = [p for p in packages if p] + [p for p in editable_packages if p] skip_requirements = False # Don't search for requirements.txt files if the user provides one if requirementstxt or package_args or project.pipfile_exists: skip_requirements = True # Ensure that virtualenv is available and pipfile are available ensure_project( project, python=python, system=system, warn=True, deploy=deploy, skip_requirements=skip_requirements, pypi_mirror=pypi_mirror, site_packages=site_packages, categories=categories, ) do_install_validations( project, package_args, requirements_directory, dev=dev, system=system, ignore_pipfile=ignore_pipfile, requirementstxt=requirementstxt, pre=pre, deploy=deploy, categories=categories, skip_lock=skip_lock, ) # Install all dependencies, if none was provided. # This basically ensures that we have a pipfile and lockfile, then it locks and # installs from the lockfile new_packages = [] if not packages and not editable_packages: if pre: project.update_settings({"allow_prereleases": pre}) do_init( project, allow_global=system, ignore_pipfile=ignore_pipfile, system=system, deploy=deploy, pre=pre, requirements_dir=requirements_directory, pypi_mirror=pypi_mirror, extra_pip_args=extra_pip_args, categories=categories, skip_lock=skip_lock, ) # This is for if the user passed in dependencies; handle with the update routine else: pkg_list = packages + [f"-e {pkg}" for pkg in editable_packages] if not system and not project.virtualenv_exists: do_init( project, system=system, allow_global=system, requirements_dir=requirements_directory, deploy=deploy, pypi_mirror=pypi_mirror, extra_pip_args=extra_pip_args, categories=categories, skip_lock=skip_lock, packages=packages, editable_packages=editable_packages, ) for pkg_line in pkg_list: console.print( f"Installing {pkg_line}...", style="bold green", ) # pip install: with temp_environ(), console.status( "Installing...", spinner=project.s.PIPENV_SPINNER ) as st: if not system: os.environ["PIP_USER"] = "0" if "PYTHONHOME" in os.environ: del os.environ["PYTHONHOME"] st.console.print(f"Resolving {pkg_line}...", markup=False) try: pkg_requirement, _ = expansive_install_req_from_line( pkg_line, expand_env=True ) except ValueError as e: err.print(f"[red]WARNING[/red]: {e}") err.print( environments.PIPENV_SPINNER_FAIL_TEXT.format( "Installation Failed" ) ) sys.exit(1) st.update(f"Installing {pkg_requirement.name}...") if categories: pipfile_sections = "" for c in categories: pipfile_sections += f"[{c}]" elif dev: pipfile_sections = "[dev-packages]" else: pipfile_sections = "[packages]" # Add the package to the Pipfile. if index: source = project.get_index_by_name(index) default_index = project.get_default_index()["name"] if not source: index_name = add_index_to_pipfile(project, index) if index_name != default_index: pkg_requirement.index = index_name elif source["name"] != default_index: pkg_requirement.index = source["name"] try: if categories: for category in categories: added, cat, normalized_name = project.add_package_to_pipfile( pkg_requirement, pkg_line, dev, category ) if added: new_packages.append((normalized_name, cat)) st.console.print( f"[bold]Added [green]{normalized_name}[/green][/bold] to Pipfile's " f"[yellow]\\{pipfile_sections}[/yellow] ..." ) else: added, cat, normalized_name = project.add_package_to_pipfile( pkg_requirement, pkg_line, dev ) if added: new_packages.append((normalized_name, cat)) st.console.print( f"[bold]Added [green]{normalized_name}[/green][/bold] to Pipfile's " f"[yellow]\\{pipfile_sections}[/yellow] ..." ) except ValueError: import traceback err.print(f"[bold][red]Error:[/red][/bold] {traceback.format_exc()}") err.print( environments.PIPENV_SPINNER_FAIL_TEXT.format( "Failed adding package to Pipfile" ) ) # ok has a nice v in front, should do something similar with rich st.console.print( environments.PIPENV_SPINNER_OK_TEXT.format("Installation Succeeded") ) # Update project settings with pre-release preference. if pre: project.update_settings({"allow_prereleases": pre}) try: do_init( project, system=system, allow_global=system, requirements_dir=requirements_directory, deploy=deploy, pypi_mirror=pypi_mirror, extra_pip_args=extra_pip_args, categories=categories, skip_lock=skip_lock, ) do_install_dependencies( project, dev=dev, allow_global=system, requirements_dir=requirements_directory, pypi_mirror=pypi_mirror, extra_pip_args=extra_pip_args, categories=categories, skip_lock=skip_lock, ) except Exception as e: # If we fail to install, remove the package from the Pipfile. for pkg_name, category in new_packages: project.remove_package_from_pipfile(pkg_name, category) raise e sys.exit(0) def do_install_validations( project, package_args, requirements_directory, dev=False, system=False, ignore_pipfile=False, requirementstxt=False, pre=False, deploy=False, categories=None, skip_lock=False, ): # Don't attempt to install develop and default packages if Pipfile is missing if not project.pipfile_exists and not (package_args or dev): if not (ignore_pipfile or deploy): raise exceptions.PipfileNotFound(project.path_to("Pipfile")) elif ((skip_lock and deploy) or ignore_pipfile) and not project.lockfile_exists: raise exceptions.LockfileNotFound(project.path_to("Pipfile.lock")) # Load the --pre settings from the Pipfile. if not pre: pre = project.settings.get("allow_prereleases") remote = requirementstxt and is_valid_url(requirementstxt) if "default" in categories: raise exceptions.PipenvUsageError( message="Cannot install to category `default`-- did you mean `packages`?" ) if "develop" in categories: raise exceptions.PipenvUsageError( message="Cannot install to category `develop`-- did you mean `dev-packages`?" ) # Warn and exit if --system is used without a pipfile. if (system and package_args) and not project.s.PIPENV_VIRTUALENV: raise exceptions.SystemUsageError # Automatically use an activated virtualenv. if project.s.PIPENV_USE_SYSTEM: system = True if system: project.s.PIPENV_USE_SYSTEM = True os.environ["PIPENV_USE_SYSTEM"] = "1" # Check if the file is remote or not if remote: err.print( "Remote requirements file provided! Downloading...", style="bold", ) fd = NamedTemporaryFile( prefix="pipenv-", suffix="-requirement.txt", dir=requirements_directory ) temp_reqs = fd.name requirements_url = requirementstxt # Download requirements file try: download_file(requirements_url, temp_reqs, project.s.PIPENV_MAX_RETRIES) except OSError: fd.close() os.unlink(temp_reqs) err.print( f"Unable to find requirements file at {requirements_url}.", style="red", ) sys.exit(1) finally: fd.close() # Replace the url with the temporary requirements file requirementstxt = temp_reqs if requirementstxt: error, traceback = None, None err.print( "Requirements file provided! Importing into Pipfile...", style="bold", ) try: import_requirements( project, r=project.path_to(requirementstxt), dev=dev, categories=categories, ) except (UnicodeDecodeError, PipError) as e: # Don't print the temp file path if remote since it will be deleted. req_path = project.path_to(requirementstxt) error = f"Unexpected syntax in {req_path}. Are you sure this is a requirements.txt style file?" traceback = e except AssertionError as e: error = ( "Requirements file doesn't appear to exist. Please ensure the file exists in your " "project directory or you provided the correct path." ) traceback = e finally: if error and traceback: console.print(error, style="red") err.print(str(traceback), style="yellow") sys.exit(1) def do_install_dependencies( project, dev=False, bare=False, allow_global=False, ignore_hashes=False, requirements_dir=None, pypi_mirror=None, extra_pip_args=None, categories=None, skip_lock=False, ): """ Executes the installation functionality. """ procs = queue.Queue(maxsize=1) if not categories: if dev: categories = ["packages", "dev-packages"] else: categories = ["packages"] for category in categories: lockfile = None pipfile = None if skip_lock: ignore_hashes = True if not bare: console.print("Installing dependencies from Pipfile...", style="bold") pipfile = project.get_pipfile_section(category) else: lockfile = project.get_or_create_lockfile(categories=categories) if not bare: console.print( f"Installing dependencies from Pipfile.lock " f"({lockfile['_meta'].get('hash', {}).get('sha256')[-6:]})...", style="bold", ) if skip_lock: deps_list = [] for req_name, pipfile_entry in pipfile.items(): install_req, markers, req_line = install_req_from_pipfile( req_name, pipfile_entry ) req_line = f"{req_line}; {markers}" if markers else f"{req_line}" deps_list.append( ( install_req, req_line, ) ) else: deps_list = list( lockfile.get_requirements(dev=dev, only=False, categories=[category]) ) editable_or_vcs_deps = [ (dep, pip_line) for dep, pip_line in deps_list if (dep.link and dep.editable) ] normal_deps = [ (dep, pip_line) for dep, pip_line in deps_list if not (dep.link and dep.editable) ] install_kwargs = { "no_deps": not skip_lock, "ignore_hashes": ignore_hashes, "allow_global": allow_global, "pypi_mirror": pypi_mirror, "sequential_deps": editable_or_vcs_deps, "extra_pip_args": extra_pip_args, } if skip_lock: lockfile_section = pipfile else: lockfile_category = get_lockfile_section_using_pipfile_category(category) lockfile_section = lockfile[lockfile_category] batch_install( project, normal_deps, lockfile_section, procs, requirements_dir, **install_kwargs, ) if not procs.empty(): _cleanup_procs(project, procs) def batch_install_iteration( project, deps_to_install, sources, procs, requirements_dir, no_deps=True, ignore_hashes=False, allow_global=False, extra_pip_args=None, ): with temp_environ(): if not allow_global: os.environ["PIP_USER"] = "0" if "PYTHONHOME" in os.environ: del os.environ["PYTHONHOME"] if "GIT_CONFIG" in os.environ: del os.environ["GIT_CONFIG"] cmds = pip_install_deps( project, deps=deps_to_install, sources=sources, allow_global=allow_global, ignore_hashes=ignore_hashes, no_deps=no_deps, requirements_dir=requirements_dir, use_pep517=True, extra_pip_args=extra_pip_args, ) for c in cmds: procs.put(c) _cleanup_procs(project, procs) def batch_install( project, deps_list, lockfile_section, procs, requirements_dir, no_deps=True, ignore_hashes=False, allow_global=False, pypi_mirror=None, sequential_deps=None, extra_pip_args=None, ): if sequential_deps is None: sequential_deps = [] deps_to_install = deps_list[:] deps_to_install.extend(sequential_deps) deps_to_install = [ (dep, pip_line) for dep, pip_line in deps_to_install if not project.environment.is_satisfied(dep) ] search_all_sources = project.settings.get("install_search_all_sources", False) sources = get_source_list( project, index=None, extra_indexes=None, trusted_hosts=get_trusted_hosts(), pypi_mirror=pypi_mirror, ) if search_all_sources: dependencies = [pip_line for _, pip_line in deps_to_install] batch_install_iteration( project, dependencies, sources, procs, requirements_dir, no_deps=no_deps, ignore_hashes=ignore_hashes, allow_global=allow_global, extra_pip_args=extra_pip_args, ) else: # Sort the dependencies out by index -- include editable/vcs in the default group deps_by_index = defaultdict(list) for dependency, pip_line in deps_to_install: index = project.sources_default["name"] if dependency.name and dependency.name in lockfile_section: entry = lockfile_section[dependency.name] if isinstance(entry, dict) and "index" in entry: index = entry["index"] deps_by_index[index].append(pip_line) # Treat each index as its own pip install phase for index_name, dependencies in deps_by_index.items(): try: install_source = next(filter(lambda s: s["name"] == index_name, sources)) batch_install_iteration( project, dependencies, [install_source], procs, requirements_dir, no_deps=no_deps, ignore_hashes=ignore_hashes, allow_global=allow_global, extra_pip_args=extra_pip_args, ) except StopIteration: # noqa: PERF203 console.print( f"Unable to find {index_name} in sources, please check dependencies: {dependencies}", style="bold red", ) sys.exit(1) def _cleanup_procs(project, procs): while not procs.empty(): c = procs.get() try: out, err = c.communicate() except AttributeError: out, err = c.stdout, c.stderr failed = c.returncode != 0 if project.s.is_verbose(): console.print(out.strip() or err.strip(), style="yellow") # The Installation failed... if failed: # The Installation failed... # We echo both c.stdout and c.stderr because pip returns error details on out. err = err.strip().splitlines() if err else [] out = out.strip().splitlines() if out else [] err_lines = [line for message in [out, err] for line in message] deps = getattr(c, "deps", {}).copy() # Return the subprocess' return code. raise exceptions.InstallError(deps, extra=err_lines) def do_init( project, allow_global=False, ignore_pipfile=False, system=False, deploy=False, pre=False, requirements_dir=None, pypi_mirror=None, extra_pip_args=None, categories=None, skip_lock=False, packages=None, editable_packages=None, ): from pipenv.routines.update import do_update python = None if project.s.PIPENV_PYTHON is not None: python = project.s.PIPENV_PYTHON elif project.s.PIPENV_DEFAULT_PYTHON_VERSION is not None: python = project.s.PIPENV_DEFAULT_PYTHON_VERSION if categories is None: categories = [] if not system and not project.s.PIPENV_USE_SYSTEM and not project.virtualenv_exists: try: do_create_virtualenv(project, python=python, pypi_mirror=pypi_mirror) except KeyboardInterrupt: cleanup_virtualenv(project, bare=False) sys.exit(1) # Ensure the Pipfile exists. if not deploy: ensure_pipfile(project, system=system) if not requirements_dir: requirements_dir = fileutils.create_tracked_tempdir( suffix="-requirements", prefix="pipenv-" ) # Write out the lockfile if it doesn't exist, but not if the Pipfile is being ignored if (project.lockfile_exists and not ignore_pipfile) and not skip_lock: old_hash = project.get_lockfile_hash() new_hash = project.calculate_pipfile_hash() if new_hash != old_hash: if deploy: console.print( f"Your Pipfile.lock ({old_hash[-6:]}) is out of date. Expected: ({new_hash[-6:]}).", style="red", ) raise exceptions.DeployException if (system or allow_global) and not (project.s.PIPENV_VIRTUALENV): err.print( f"Pipfile.lock ({old_hash[-6:]}) out of date, but installation uses --system so" f"re-building lockfile must happen in isolation." f" Please rebuild lockfile in a virtualenv. Continuing anyway...", style="yellow", ) else: if old_hash: msg = "Pipfile.lock ({0}) out of date: run `pipfile lock` to update to ({1})..." else: msg = "Pipfile.lock is corrupt, replaced with ({1})..." err.print( msg.format(old_hash[-6:], new_hash[-6:]), style="bold yellow", ) do_update( project, pre=pre, system=system, pypi_mirror=pypi_mirror, categories=categories, packages=packages, editable_packages=editable_packages, ) # Write out the lockfile if it doesn't exist. if not project.lockfile_exists: # Unless we're in a virtualenv not managed by pipenv, abort if we're # using the system's python. if ( (system or allow_global) and not (project.s.PIPENV_VIRTUALENV) and skip_lock is False ): raise exceptions.PipenvOptionsError( "--system", "--system is intended to be used for Pipfile installation, " "not installation of specific packages. Aborting.\n" "See also: --deploy flag.", ) else: err.print( "Pipfile.lock not found, creating...", style="bold", ) do_lock( project, system=system, pre=pre, write=True, pypi_mirror=pypi_mirror, categories=categories, ) # Hint the user what to do to activate the virtualenv. if not allow_global and not deploy and "PIPENV_ACTIVE" not in os.environ: console.print( "To activate this project's virtualenv, run [yellow]pipenv shell[/yellow].\n" "Alternatively, run a command inside the virtualenv with [yellow]pipenv run[/yellow]." )