import os import re from typing import Tuple from urllib.parse import quote, unquote from pipenv.patched.pip._internal.network.session import PipSession from pipenv.patched.pip._internal.req import parse_requirements from pipenv.patched.pip._internal.req.constructors import ( install_req_from_parsed_requirement, ) from pipenv.patched.pip._internal.utils.misc import _transform_url, split_auth_from_netloc from pipenv.utils.constants import VCS_LIST from pipenv.utils.indexes import parse_indexes from pipenv.utils.internet import get_host_and_port from pipenv.utils.pip import get_trusted_hosts def redact_netloc(netloc: str) -> Tuple[str]: """ Replace the sensitive data in a netloc with "****", if it exists, unless it's an environment variable. For example: - "user:pass@example.com" returns "user:****@example.com" - "accesstoken@example.com" returns "****@example.com" - "${ENV_VAR}:pass@example.com" returns "${ENV_VAR}:****@example.com" if ${ENV_VAR} is an environment variable """ netloc, (user, password) = split_auth_from_netloc(netloc) if user is None: return (netloc,) if password is None: # Check if user is an environment variable if not re.match(r"\$\{\w+\}", user): # If not, redact the user user = "****" password = "" else: # Check if password is an environment variable if not re.match(r"\$\{\w+\}", password): # If not, redact the password password = ":****" else: # If it is, leave it as is password = ":" + password user = quote(user) return (f"{user}{password}@{netloc}",) def redact_auth_from_url(url: str) -> str: """Replace the password in a given url with ****.""" return _transform_url(url, redact_netloc)[0] def normalize_name(pkg) -> str: """Given a package name, return its normalized, non-canonicalized form.""" return pkg.replace("_", "-").lower() def import_requirements(project, r=None, dev=False, categories=None): # Parse requirements.txt file with Pip's parser. # Pip requires a `PipSession` which is a subclass of requests.Session. # Since we're not making any network calls, it's initialized to nothing. if r: assert os.path.isfile(r) # Default path, if none is provided. if r is None: r = project.requirements_location with open(r) as f: contents = f.read() if categories is None: categories = [] indexes = [] trusted_hosts = [] # Find and add extra indexes. for line in contents.split("\n"): index, extra_index, trusted_host, _ = parse_indexes(line.strip(), strict=True) if index: indexes = [index] if extra_index: indexes.append(extra_index) if trusted_host: trusted_hosts.append(get_host_and_port(trusted_host)) for f in parse_requirements(r, session=PipSession()): package = install_req_from_parsed_requirement(f) if package.name not in BAD_PACKAGES: if package.link is not None: if package.editable: package_string = f"-e {package.link}" else: package_string = unquote( redact_auth_from_url(package.original_link.url) ) if categories: for category in categories: project.add_package_to_pipfile( package, package_string, dev=dev, category=category ) else: project.add_package_to_pipfile(package, package_string, dev=dev) else: package_string = str(package.req) if package.markers: package_string += f" ; {package.markers}" if categories: for category in categories: project.add_package_to_pipfile( package, package_string, dev=dev, category=category ) else: project.add_package_to_pipfile(package, package_string, dev=dev) indexes = sorted(set(indexes)) trusted_hosts = sorted(set(trusted_hosts)) for index in indexes: add_index_to_pipfile(project, index, trusted_hosts) project.recase_pipfile() def add_index_to_pipfile(project, index, trusted_hosts=None): # don't require HTTPS for trusted hosts (see: https://pip.pypa.io/en/stable/cli/pip/#cmdoption-trusted-host) if trusted_hosts is None: trusted_hosts = get_trusted_hosts() host_and_port = get_host_and_port(index) require_valid_https = not any( v in trusted_hosts for v in ( host_and_port, host_and_port.partition(":")[ 0 ], # also check if hostname without port is in trusted_hosts ) ) index_name = project.add_index_to_pipfile(index, verify_ssl=require_valid_https) return index_name BAD_PACKAGES = ( "distribute", "pip", "pkg-resources", "setuptools", "wheel", ) def requirement_from_lockfile( package_name, package_info, include_hashes=True, include_markers=True ): from pipenv.utils.dependencies import is_editable_path, is_star, normalize_vcs_url # Handle string requirements if isinstance(package_info, str): if package_info and not is_star(package_info): return f"{package_name}=={package_info}" else: return package_name markers = ( "; {}".format(package_info["markers"]) if include_markers and "markers" in package_info and package_info["markers"] else "" ) os_markers = ( "; {}".format(package_info["os_markers"]) if include_markers and "os_markers" in package_info and package_info["os_markers"] else "" ) # Handling vcs repositories for vcs in VCS_LIST: if vcs in package_info: vcs_url = package_info[vcs] ref = package_info.get("ref", "") # We have to handle the fact that some vcs urls have a ref in them # and some have a netloc with a username and password in them, and some have both vcs_url, fallback_ref = normalize_vcs_url(vcs_url) if not ref: ref = fallback_ref extras = ( "[{}]".format(",".join(package_info.get("extras", []))) if "extras" in package_info else "" ) subdirectory = package_info.get("subdirectory", "") include_vcs = "" if f"{vcs}+" in vcs_url else f"{vcs}+" egg_fragment = "" if "#egg=" in vcs_url else f"#egg={package_name}" ref_str = "" if not ref or f"@{ref}" in vcs_url else f"@{ref}" if ( is_editable_path(vcs_url) or "file://" in vcs_url or package_info.get("editable", False) ): pip_line = f"-e {include_vcs}{vcs_url}{ref_str}{egg_fragment}{extras}" pip_line += f"&subdirectory={subdirectory}" if subdirectory else "" else: pip_line = f"{package_name}{extras}@ {include_vcs}{vcs_url}{ref_str}" pip_line += f"#subdirectory={subdirectory}" if subdirectory else "" return pip_line # Handling file-sourced packages for k in ["file", "path"]: line = [] if k in package_info: path = package_info[k] if is_editable_path(path): line.append("-e") line.append(f"{package_info[k]}") if os_markers: line.append(os_markers) if markers: line.append(markers) pip_line = " ".join(line) return pip_line # Handling packages from standard pypi like indexes version = package_info.get("version", "").replace("==", "") hashes = ( f" --hash={' --hash='.join(package_info['hashes'])}" if include_hashes and "hashes" in package_info else "" ) extras = ( "[{}]".format(",".join(package_info.get("extras", []))) if "extras" in package_info else "" ) pip_line = f"{package_name}{extras}=={version}{os_markers}{markers}{hashes}" return pip_line def requirements_from_lockfile(deps, include_hashes=True, include_markers=True): pip_packages = [] for package_name, package_info in deps.items(): pip_package = requirement_from_lockfile( package_name, package_info, include_hashes, include_markers ) # Append to the list pip_packages.append(pip_package) # pip_packages contains the pip-installable lines return pip_packages