# -*- coding: utf-8 -*- from __future__ import unicode_literals, absolute_import import os from collections import OrderedDict import re import sys from io import StringIO from configparser import ConfigParser, NoOptionError from pathlib import PurePath from .errors import MalformedDependencyFileError from .regex import HASH_REGEX from .dependencies import DependencyFile, Dependency from pipenv.vendor.packaging.requirements import Requirement as PackagingRequirement,\ InvalidRequirement from . import filetypes from pipenv.vendor.packaging.specifiers import SpecifierSet from pipenv.vendor.packaging.version import Version, InvalidVersion import json if sys.version_info >= (3, 11): import tomllib else: import pipenv.vendor.tomli as tomllib # this is a backport from setuptools 26.1 def setuptools_parse_requirements_backport(strs): # pragma: no cover # Copyright (C) 2016 Jason R Coombs # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files # (the "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, # distribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the # following conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. # IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """Yield ``Requirement`` objects for each specification in `strs` `strs` must be a string, or a (possibly-nested) iterable thereof. """ # create a steppable iterator, so we can handle \-continuations def yield_lines(strs): """Yield non-empty/non-comment lines of a string or sequence""" if isinstance(strs, str): for s in strs.splitlines(): s = s.strip() # skip blank lines/comments if s and not s.startswith('#'): yield s else: for ss in strs: for s in yield_lines(ss): yield s lines = iter(yield_lines(strs)) for line in lines: # Drop comments -- a hash without a space may be in a URL. if ' #' in line: line = line[:line.find(' #')] # If there is a line continuation, drop it, and append the next line. if line.endswith('\\'): line = line[:-2].strip() line += next(lines) yield PackagingRequirement(line) class RequirementsTXTLineParser(object): """ """ @classmethod def parse(cls, line): """ :param line: :return: """ try: # setuptools requires a space before the comment. # If this isn't the case, add it. if "\t#" in line: parsed, = setuptools_parse_requirements_backport( line.replace("\t#", "\t #")) else: parsed, = setuptools_parse_requirements_backport(line) except InvalidRequirement: return None dep = Dependency( name=parsed.name, specs=parsed.specifier, line=line, extras=parsed.extras, dependency_type=filetypes.requirements_txt ) return dep class Parser(object): """ """ def __init__(self, obj, resolve=False): """ :param obj: """ self.obj = obj self._lines = None self.resolve = resolve def iter_lines(self, lineno=0): """ :param lineno: :return: """ for line in self.lines[lineno:]: yield line @property def lines(self): """ :return: """ if self._lines is None: self._lines = self.obj.content.splitlines() return self._lines @property def is_marked_file(self): """ :return: """ for n, line in enumerate(self.iter_lines()): for marker in self.obj.file_marker: if marker in line: return True if n >= 2: break return False def is_marked_line(self, line): """ :param line: :return: """ for marker in self.obj.line_marker: if marker in line: return True return False @classmethod def parse_hashes(cls, line): """ :param line: :return: """ hashes = [] for match in re.finditer(HASH_REGEX, line): hashes.append(line[match.start():match.end()]) return re.sub(HASH_REGEX, "", line).strip(), hashes @classmethod def parse_index_server(cls, line): """ :param line: :return: """ groups = re.split(pattern=r"[=\s]+", string=line.strip(), maxsplit=100) if len(groups) >= 2: return groups[1] if groups[1].endswith("/") else groups[1] + "/" return None @classmethod def resolve_file(cls, file_path, line): """ :param file_path: :param line: :return: """ line = line.replace("-r ", "").replace("--requirement ", "") normalized_path = PurePath(file_path) if " #" in line: line = line.split("#")[0].strip() return str(normalized_path.parent.joinpath(line)) class RequirementsTXTParser(Parser): """ """ def parse(self): """ Parses a requirements.txt-like file """ index_server = None for num, line in enumerate(self.iter_lines()): line = line.rstrip() if not line: continue if line.startswith('#'): # comments are lines that start with # only continue if line.startswith('-i') or \ line.startswith('--index-url') or \ line.startswith('--extra-index-url'): # this file is using a private index server, try to parse it index_server = self.parse_index_server(line) continue elif self.obj.path and \ (line.startswith('-r') or line.startswith('--requirement')): req_file_path = self.resolve_file(self.obj.path, line) if self.resolve and os.path.exists(req_file_path): with open(req_file_path, 'r') as f: content = f.read() dep_file = DependencyFile( content=content, path=req_file_path, resolve=True ) dep_file.parse() self.obj.resolved_files.append(dep_file) else: self.obj.resolved_files.append(req_file_path) elif line.startswith('-f') or line.startswith('--find-links') or \ line.startswith('--no-index') or \ line.startswith('--allow-external') or \ line.startswith('--allow-unverified') or \ line.startswith('-Z') or \ line.startswith('--always-unzip'): continue elif self.is_marked_line(line): continue else: try: parseable_line = line # multiline requirements are not parseable if "\\" in line: parseable_line = line.replace("\\", "") for next_line in self.iter_lines(num + 1): parseable_line += next_line.strip().replace("\\", "") line += "\n" + next_line if "\\" in next_line: continue break # ignore multiline requirements if they are marked if self.is_marked_line(parseable_line): continue hashes = [] if "--hash" in parseable_line: parseable_line, hashes = Parser.parse_hashes( parseable_line) req = RequirementsTXTLineParser.parse(parseable_line) if req: req.hashes = hashes req.index_server = index_server # replace the requirements line with the 'real' line req.line = line self.obj.dependencies.append(req) except ValueError: continue class ToxINIParser(Parser): """ """ def parse(self): """ :return: """ parser = ConfigParser() parser.readfp(StringIO(self.obj.content)) for section in parser.sections(): try: content = parser.get(section=section, option="deps") for n, line in enumerate(content.splitlines()): if self.is_marked_line(line): continue if line: req = RequirementsTXTLineParser.parse(line) if req: req.dependency_type = self.obj.file_type self.obj.dependencies.append(req) except NoOptionError: pass class CondaYMLParser(Parser): """ """ def parse(self): """ :return: """ import yaml try: data = yaml.safe_load(self.obj.content) if data and 'dependencies' in data and \ isinstance(data['dependencies'], list): for dep in data['dependencies']: if isinstance(dep, dict) and 'pip' in dep: for n, line in enumerate(dep['pip']): if self.is_marked_line(line): continue req = RequirementsTXTLineParser.parse(line) if req: req.dependency_type = self.obj.file_type self.obj.dependencies.append(req) except yaml.YAMLError: pass class PipfileParser(Parser): def parse(self): """ Parse a Pipfile (as seen in pipenv) :return: """ try: data = tomllib.loads(self.obj.content) if data: for package_type in ['packages', 'dev-packages']: if package_type in data: for name, specs in data[package_type].items(): # skip on VCS dependencies if not isinstance(specs, str): continue if specs == '*': specs = '' self.obj.dependencies.append( Dependency( name=name, specs=SpecifierSet(specs), dependency_type=filetypes.pipfile, line=''.join([name, specs]), section=package_type ) ) except (tomllib.TOMLDecodeError, IndexError): pass class PipfileLockParser(Parser): def parse(self): """ Parse a Pipfile.lock (as seen in pipenv) :return: """ try: data = json.loads(self.obj.content, object_pairs_hook=OrderedDict) if data: for package_type in ['default', 'develop']: if package_type in data: for name, meta in data[package_type].items(): # skip VCS dependencies if 'version' not in meta: continue specs = meta['version'] hashes = meta['hashes'] self.obj.dependencies.append( Dependency( name=name, specs=SpecifierSet(specs), dependency_type=filetypes.pipfile_lock, hashes=hashes, line=''.join([name, specs]), section=package_type ) ) except ValueError as e: raise MalformedDependencyFileError(info=str(e)) class SetupCfgParser(Parser): def parse(self): parser = ConfigParser() parser.readfp(StringIO(self.obj.content)) for section in parser.values(): if section.name == 'options': options = 'install_requires', 'setup_requires', 'test_require' for name in options: content = section.get(name) if not content: continue self._parse_content(content) elif section.name == 'options.extras_require': for content in section.values(): self._parse_content(content) def _parse_content(self, content): for n, line in enumerate(content.splitlines()): if self.is_marked_line(line): continue if line: req = RequirementsTXTLineParser.parse(line) if req: req.dependency_type = self.obj.file_type self.obj.dependencies.append(req) class PoetryLockParser(Parser): def parse(self): """ Parse a poetry.lock """ try: data = tomllib.loads(self.obj.content) pkg_key = 'package' if data: try: dependencies = data[pkg_key] except KeyError: raise KeyError( "Poetry lock file is missing the package section") for dep in dependencies: try: name = dep['name'] spec = "=={version}".format( version=Version(dep['version'])) section = dep['category'] except KeyError: raise KeyError("Malformed poetry lock file") except InvalidVersion: continue self.obj.dependencies.append( Dependency( name=name, specs=SpecifierSet(spec), dependency_type=filetypes.poetry_lock, line=''.join([name, spec]), section=section ) ) except (tomllib.TOMLDecodeError, IndexError) as e: raise MalformedDependencyFileError(info=str(e)) def parse(content, file_type=None, path=None, sha=None, marker=((), ()), parser=None, resolve=False): """ :param content: :param file_type: :param path: :param sha: :param marker: :param parser: :return: """ dep_file = DependencyFile( content=content, path=path, sha=sha, marker=marker, file_type=file_type, parser=parser, resolve=resolve ) return dep_file.parse()