"""

[0] Fix | Delete

Requirements file parsing

[1] Fix | Delete

"""

[2] Fix | Delete

[3] Fix | Delete

from __future__ import absolute_import

[4] Fix | Delete

[5] Fix | Delete

import os

[6] Fix | Delete

import re

[7] Fix | Delete

import shlex

[8] Fix | Delete

import sys

[9] Fix | Delete

import optparse

[10] Fix | Delete

import warnings

[11] Fix | Delete

[12] Fix | Delete

from pip._vendor.six.moves.urllib import parse as urllib_parse

[13] Fix | Delete

from pip._vendor.six.moves import filterfalse

[14] Fix | Delete

[15] Fix | Delete

import pip

[16] Fix | Delete

from pip.download import get_file_content

[17] Fix | Delete

from pip.req.req_install import InstallRequirement

[18] Fix | Delete

from pip.exceptions import (RequirementsFileParseError)

[19] Fix | Delete

from pip.utils.deprecation import RemovedInPip10Warning

[20] Fix | Delete

from pip import cmdoptions

[21] Fix | Delete

[22] Fix | Delete

__all__ = ['parse_requirements']

[23] Fix | Delete

[24] Fix | Delete

SCHEME_RE = re.compile(r'^(http|https|file):', re.I)

[25] Fix | Delete

COMMENT_RE = re.compile(r'(^|\s)+#.*$')

[26] Fix | Delete

[27] Fix | Delete

SUPPORTED_OPTIONS = [

[28] Fix | Delete

cmdoptions.constraints,

[29] Fix | Delete

cmdoptions.editable,

[30] Fix | Delete

cmdoptions.requirements,

[31] Fix | Delete

cmdoptions.no_index,

[32] Fix | Delete

cmdoptions.index_url,

[33] Fix | Delete

cmdoptions.find_links,

[34] Fix | Delete

cmdoptions.extra_index_url,

[35] Fix | Delete

cmdoptions.allow_external,

[36] Fix | Delete

cmdoptions.allow_all_external,

[37] Fix | Delete

cmdoptions.no_allow_external,

[38] Fix | Delete

cmdoptions.allow_unsafe,

[39] Fix | Delete

cmdoptions.no_allow_unsafe,

[40] Fix | Delete

cmdoptions.use_wheel,

[41] Fix | Delete

cmdoptions.no_use_wheel,

[42] Fix | Delete

cmdoptions.always_unzip,

[43] Fix | Delete

cmdoptions.no_binary,

[44] Fix | Delete

cmdoptions.only_binary,

[45] Fix | Delete

cmdoptions.pre,

[46] Fix | Delete

cmdoptions.process_dependency_links,

[47] Fix | Delete

cmdoptions.trusted_host,

[48] Fix | Delete

cmdoptions.require_hashes,

[49] Fix | Delete

]

[50] Fix | Delete

[51] Fix | Delete

# options to be passed to requirements

[52] Fix | Delete

SUPPORTED_OPTIONS_REQ = [

[53] Fix | Delete

cmdoptions.install_options,

[54] Fix | Delete

cmdoptions.global_options,

[55] Fix | Delete

cmdoptions.hash,

[56] Fix | Delete

]

[57] Fix | Delete

[58] Fix | Delete

# the 'dest' string values

[59] Fix | Delete

SUPPORTED_OPTIONS_REQ_DEST = [o().dest for o in SUPPORTED_OPTIONS_REQ]

[60] Fix | Delete

[61] Fix | Delete

[62] Fix | Delete

def parse_requirements(filename, finder=None, comes_from=None, options=None,

[63] Fix | Delete

session=None, constraint=False, wheel_cache=None):

[64] Fix | Delete

"""Parse a requirements file and yield InstallRequirement instances.

[65] Fix | Delete

[66] Fix | Delete

:param filename: Path or url of requirements file.

[67] Fix | Delete

:param finder: Instance of pip.index.PackageFinder.

[68] Fix | Delete

:param comes_from: Origin description of requirements.

[69] Fix | Delete

:param options: cli options.

[70] Fix | Delete

:param session: Instance of pip.download.PipSession.

[71] Fix | Delete

:param constraint: If true, parsing a constraint file rather than

[72] Fix | Delete

requirements file.

[73] Fix | Delete

:param wheel_cache: Instance of pip.wheel.WheelCache

[74] Fix | Delete

"""

[75] Fix | Delete

if session is None:

[76] Fix | Delete

raise TypeError(

[77] Fix | Delete

"parse_requirements() missing 1 required keyword argument: "

[78] Fix | Delete

"'session'"

[79] Fix | Delete

)

[80] Fix | Delete

[81] Fix | Delete

_, content = get_file_content(

[82] Fix | Delete

filename, comes_from=comes_from, session=session

[83] Fix | Delete

)

[84] Fix | Delete

[85] Fix | Delete

lines_enum = preprocess(content, options)

[86] Fix | Delete

[87] Fix | Delete

for line_number, line in lines_enum:

[88] Fix | Delete

req_iter = process_line(line, filename, line_number, finder,

[89] Fix | Delete

comes_from, options, session, wheel_cache,

[90] Fix | Delete

constraint=constraint)

[91] Fix | Delete

for req in req_iter:

[92] Fix | Delete

yield req

[93] Fix | Delete

[94] Fix | Delete

[95] Fix | Delete

def preprocess(content, options):

[96] Fix | Delete

"""Split, filter, and join lines, and return a line iterator

[97] Fix | Delete

[98] Fix | Delete

:param content: the content of the requirements file

[99] Fix | Delete

:param options: cli options

[100] Fix | Delete

"""

[101] Fix | Delete

lines_enum = enumerate(content.splitlines(), start=1)

[102] Fix | Delete

lines_enum = join_lines(lines_enum)

[103] Fix | Delete

lines_enum = ignore_comments(lines_enum)

[104] Fix | Delete

lines_enum = skip_regex(lines_enum, options)

[105] Fix | Delete

return lines_enum

[106] Fix | Delete

[107] Fix | Delete

[108] Fix | Delete

def process_line(line, filename, line_number, finder=None, comes_from=None,

[109] Fix | Delete

options=None, session=None, wheel_cache=None,

[110] Fix | Delete

constraint=False):

[111] Fix | Delete

"""Process a single requirements line; This can result in creating/yielding

[112] Fix | Delete

requirements, or updating the finder.

[113] Fix | Delete

[114] Fix | Delete

For lines that contain requirements, the only options that have an effect

[115] Fix | Delete

are from SUPPORTED_OPTIONS_REQ, and they are scoped to the

[116] Fix | Delete

requirement. Other options from SUPPORTED_OPTIONS may be present, but are

[117] Fix | Delete

ignored.

[118] Fix | Delete

[119] Fix | Delete

For lines that do not contain requirements, the only options that have an

[120] Fix | Delete

effect are from SUPPORTED_OPTIONS. Options from SUPPORTED_OPTIONS_REQ may

[121] Fix | Delete

be present, but are ignored. These lines may contain multiple options

[122] Fix | Delete

(although our docs imply only one is supported), and all our parsed and

[123] Fix | Delete

affect the finder.

[124] Fix | Delete

[125] Fix | Delete

:param constraint: If True, parsing a constraints file.

[126] Fix | Delete

:param options: OptionParser options that we may update

[127] Fix | Delete

"""

[128] Fix | Delete

parser = build_parser()

[129] Fix | Delete

defaults = parser.get_default_values()

[130] Fix | Delete

defaults.index_url = None

[131] Fix | Delete

if finder:

[132] Fix | Delete

# `finder.format_control` will be updated during parsing

[133] Fix | Delete

defaults.format_control = finder.format_control

[134] Fix | Delete

args_str, options_str = break_args_options(line)

[135] Fix | Delete

if sys.version_info < (2, 7, 3):

[136] Fix | Delete

# Prior to 2.7.3, shlex cannot deal with unicode entries

[137] Fix | Delete

options_str = options_str.encode('utf8')

[138] Fix | Delete

opts, _ = parser.parse_args(shlex.split(options_str), defaults)

[139] Fix | Delete

[140] Fix | Delete

# preserve for the nested code path

[141] Fix | Delete

line_comes_from = '%s %s (line %s)' % (

[142] Fix | Delete

'-c' if constraint else '-r', filename, line_number)

[143] Fix | Delete

[144] Fix | Delete

# yield a line requirement

[145] Fix | Delete

if args_str:

[146] Fix | Delete

isolated = options.isolated_mode if options else False

[147] Fix | Delete

if options:

[148] Fix | Delete

cmdoptions.check_install_build_global(options, opts)

[149] Fix | Delete

# get the options that apply to requirements

[150] Fix | Delete

req_options = {}

[151] Fix | Delete

for dest in SUPPORTED_OPTIONS_REQ_DEST:

[152] Fix | Delete

if dest in opts.__dict__ and opts.__dict__[dest]:

[153] Fix | Delete

req_options[dest] = opts.__dict__[dest]

[154] Fix | Delete

yield InstallRequirement.from_line(

[155] Fix | Delete

args_str, line_comes_from, constraint=constraint,

[156] Fix | Delete

isolated=isolated, options=req_options, wheel_cache=wheel_cache

[157] Fix | Delete

)

[158] Fix | Delete

[159] Fix | Delete

# yield an editable requirement

[160] Fix | Delete

elif opts.editables:

[161] Fix | Delete

isolated = options.isolated_mode if options else False

[162] Fix | Delete

default_vcs = options.default_vcs if options else None

[163] Fix | Delete

yield InstallRequirement.from_editable(

[164] Fix | Delete

opts.editables[0], comes_from=line_comes_from,

[165] Fix | Delete

constraint=constraint, default_vcs=default_vcs, isolated=isolated,

[166] Fix | Delete

wheel_cache=wheel_cache

[167] Fix | Delete

)

[168] Fix | Delete

[169] Fix | Delete

# parse a nested requirements file

[170] Fix | Delete

elif opts.requirements or opts.constraints:

[171] Fix | Delete

if opts.requirements:

[172] Fix | Delete

req_path = opts.requirements[0]

[173] Fix | Delete

nested_constraint = False

[174] Fix | Delete

else:

[175] Fix | Delete

req_path = opts.constraints[0]

[176] Fix | Delete

nested_constraint = True

[177] Fix | Delete

# original file is over http

[178] Fix | Delete

if SCHEME_RE.search(filename):

[179] Fix | Delete

# do a url join so relative paths work

[180] Fix | Delete

req_path = urllib_parse.urljoin(filename, req_path)

[181] Fix | Delete

# original file and nested file are paths

[182] Fix | Delete

elif not SCHEME_RE.search(req_path):

[183] Fix | Delete

# do a join so relative paths work

[184] Fix | Delete

req_path = os.path.join(os.path.dirname(filename), req_path)

[185] Fix | Delete

# TODO: Why not use `comes_from='-r {} (line {})'` here as well?

[186] Fix | Delete

parser = parse_requirements(

[187] Fix | Delete

req_path, finder, comes_from, options, session,

[188] Fix | Delete

constraint=nested_constraint, wheel_cache=wheel_cache

[189] Fix | Delete

)

[190] Fix | Delete

for req in parser:

[191] Fix | Delete

yield req

[192] Fix | Delete

[193] Fix | Delete

# percolate hash-checking option upward

[194] Fix | Delete

elif opts.require_hashes:

[195] Fix | Delete

options.require_hashes = opts.require_hashes

[196] Fix | Delete

[197] Fix | Delete

# set finder options

[198] Fix | Delete

elif finder:

[199] Fix | Delete

if opts.allow_external:

[200] Fix | Delete

warnings.warn(

[201] Fix | Delete

"--allow-external has been deprecated and will be removed in "

[202] Fix | Delete

"the future. Due to changes in the repository protocol, it no "

[203] Fix | Delete

"longer has any effect.",

[204] Fix | Delete

RemovedInPip10Warning,

[205] Fix | Delete

)

[206] Fix | Delete

[207] Fix | Delete

if opts.allow_all_external:

[208] Fix | Delete

warnings.warn(

[209] Fix | Delete

"--allow-all-external has been deprecated and will be removed "

[210] Fix | Delete

"in the future. Due to changes in the repository protocol, it "

[211] Fix | Delete

"no longer has any effect.",

[212] Fix | Delete

RemovedInPip10Warning,

[213] Fix | Delete

)

[214] Fix | Delete

[215] Fix | Delete

if opts.allow_unverified:

[216] Fix | Delete

warnings.warn(

[217] Fix | Delete

"--allow-unverified has been deprecated and will be removed "

[218] Fix | Delete

"in the future. Due to changes in the repository protocol, it "

[219] Fix | Delete

"no longer has any effect.",

[220] Fix | Delete

RemovedInPip10Warning,

[221] Fix | Delete

)

[222] Fix | Delete

[223] Fix | Delete

if opts.index_url:

[224] Fix | Delete

finder.index_urls = [opts.index_url]

[225] Fix | Delete

if opts.use_wheel is False:

[226] Fix | Delete

finder.use_wheel = False

[227] Fix | Delete

pip.index.fmt_ctl_no_use_wheel(finder.format_control)

[228] Fix | Delete

if opts.no_index is True:

[229] Fix | Delete

finder.index_urls = []

[230] Fix | Delete

if opts.extra_index_urls:

[231] Fix | Delete

finder.index_urls.extend(opts.extra_index_urls)

[232] Fix | Delete

if opts.find_links:

[233] Fix | Delete

# FIXME: it would be nice to keep track of the source

[234] Fix | Delete

# of the find_links: support a find-links local path

[235] Fix | Delete

# relative to a requirements file.

[236] Fix | Delete

value = opts.find_links[0]

[237] Fix | Delete

req_dir = os.path.dirname(os.path.abspath(filename))

[238] Fix | Delete

relative_to_reqs_file = os.path.join(req_dir, value)

[239] Fix | Delete

if os.path.exists(relative_to_reqs_file):

[240] Fix | Delete

value = relative_to_reqs_file

[241] Fix | Delete

finder.find_links.append(value)

[242] Fix | Delete

if opts.pre:

[243] Fix | Delete

finder.allow_all_prereleases = True

[244] Fix | Delete

if opts.process_dependency_links:

[245] Fix | Delete

finder.process_dependency_links = True

[246] Fix | Delete

if opts.trusted_hosts:

[247] Fix | Delete

finder.secure_origins.extend(

[248] Fix | Delete

("*", host, "*") for host in opts.trusted_hosts)

[249] Fix | Delete

[250] Fix | Delete

[251] Fix | Delete

def break_args_options(line):

[252] Fix | Delete

"""Break up the line into an args and options string. We only want to shlex

[253] Fix | Delete

(and then optparse) the options, not the args. args can contain markers

[254] Fix | Delete

which are corrupted by shlex.

[255] Fix | Delete

"""

[256] Fix | Delete

tokens = line.split(' ')

[257] Fix | Delete

args = []

[258] Fix | Delete

options = tokens[:]

[259] Fix | Delete

for token in tokens:

[260] Fix | Delete

if token.startswith('-') or token.startswith('--'):

[261] Fix | Delete

break

[262] Fix | Delete

else:

[263] Fix | Delete

args.append(token)

[264] Fix | Delete

options.pop(0)

[265] Fix | Delete

return ' '.join(args), ' '.join(options)

[266] Fix | Delete

[267] Fix | Delete

[268] Fix | Delete

def build_parser():

[269] Fix | Delete

"""

[270] Fix | Delete

Return a parser for parsing requirement lines

[271] Fix | Delete

"""

[272] Fix | Delete

parser = optparse.OptionParser(add_help_option=False)

[273] Fix | Delete

[274] Fix | Delete

option_factories = SUPPORTED_OPTIONS + SUPPORTED_OPTIONS_REQ

[275] Fix | Delete

for option_factory in option_factories:

[276] Fix | Delete

option = option_factory()

[277] Fix | Delete

parser.add_option(option)

[278] Fix | Delete

[279] Fix | Delete

# By default optparse sys.exits on parsing errors. We want to wrap

[280] Fix | Delete

# that in our own exception.

[281] Fix | Delete

def parser_exit(self, msg):

[282] Fix | Delete

raise RequirementsFileParseError(msg)

[283] Fix | Delete

parser.exit = parser_exit

[284] Fix | Delete

[285] Fix | Delete

return parser

[286] Fix | Delete

[287] Fix | Delete

[288] Fix | Delete

def join_lines(lines_enum):

[289] Fix | Delete

"""Joins a line ending in '\' with the previous line (except when following

[290] Fix | Delete

comments). The joined line takes on the index of the first line.

[291] Fix | Delete

"""

[292] Fix | Delete

primary_line_number = None

[293] Fix | Delete

new_line = []

[294] Fix | Delete

for line_number, line in lines_enum:

[295] Fix | Delete

if not line.endswith('\\') or COMMENT_RE.match(line):

[296] Fix | Delete

if COMMENT_RE.match(line):

[297] Fix | Delete

# this ensures comments are always matched later

[298] Fix | Delete

line = ' ' + line

[299] Fix | Delete

if new_line:

[300] Fix | Delete

new_line.append(line)

[301] Fix | Delete

yield primary_line_number, ''.join(new_line)

[302] Fix | Delete

new_line = []

[303] Fix | Delete

else:

[304] Fix | Delete

yield line_number, line

[305] Fix | Delete

else:

[306] Fix | Delete

if not new_line:

[307] Fix | Delete

primary_line_number = line_number

[308] Fix | Delete

new_line.append(line.strip('\\'))

[309] Fix | Delete

[310] Fix | Delete

# last line contains \

[311] Fix | Delete

if new_line:

[312] Fix | Delete

yield primary_line_number, ''.join(new_line)

[313] Fix | Delete

[314] Fix | Delete

# TODO: handle space after '\'.

[315] Fix | Delete

[316] Fix | Delete

[317] Fix | Delete

def ignore_comments(lines_enum):

[318] Fix | Delete

"""

[319] Fix | Delete

Strips comments and filter empty lines.

[320] Fix | Delete

"""

[321] Fix | Delete

for line_number, line in lines_enum:

[322] Fix | Delete

line = COMMENT_RE.sub('', line)

[323] Fix | Delete

line = line.strip()

[324] Fix | Delete

if line:

[325] Fix | Delete

yield line_number, line

[326] Fix | Delete

[327] Fix | Delete

[328] Fix | Delete

def skip_regex(lines_enum, options):

[329] Fix | Delete

"""

[330] Fix | Delete

Skip lines that match '--skip-requirements-regex' pattern

[331] Fix | Delete

[332] Fix | Delete

Note: the regex pattern is only built once

[333] Fix | Delete

"""

[334] Fix | Delete

skip_regex = options.skip_requirements_regex if options else None

[335] Fix | Delete

if skip_regex:

[336] Fix | Delete

pattern = re.compile(skip_regex)

[337] Fix | Delete

lines_enum = filterfalse(

[338] Fix | Delete

lambda e: pattern.search(e[1]),

[339] Fix | Delete

lines_enum)

[340] Fix | Delete

return lines_enum

[341] Fix | Delete

[342] Fix | Delete