common.py 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612
  1. # Copyright (c) 2012 Google Inc. All rights reserved.
  2. # Use of this source code is governed by a BSD-style license that can be
  3. # found in the LICENSE file.
  4. from __future__ import with_statement
  5. import collections
  6. import errno
  7. import filecmp
  8. import os.path
  9. import re
  10. import tempfile
  11. import sys
  12. # A minimal memoizing decorator. It'll blow up if the args aren't immutable,
  13. # among other "problems".
  14. class memoize(object):
  15. def __init__(self, func):
  16. self.func = func
  17. self.cache = {}
  18. def __call__(self, *args):
  19. try:
  20. return self.cache[args]
  21. except KeyError:
  22. result = self.func(*args)
  23. self.cache[args] = result
  24. return result
  25. class GypError(Exception):
  26. """Error class representing an error, which is to be presented
  27. to the user. The main entry point will catch and display this.
  28. """
  29. pass
  30. def ExceptionAppend(e, msg):
  31. """Append a message to the given exception's message."""
  32. if not e.args:
  33. e.args = (msg,)
  34. elif len(e.args) == 1:
  35. e.args = (str(e.args[0]) + ' ' + msg,)
  36. else:
  37. e.args = (str(e.args[0]) + ' ' + msg,) + e.args[1:]
  38. def FindQualifiedTargets(target, qualified_list):
  39. """
  40. Given a list of qualified targets, return the qualified targets for the
  41. specified |target|.
  42. """
  43. return [t for t in qualified_list if ParseQualifiedTarget(t)[1] == target]
  44. def ParseQualifiedTarget(target):
  45. # Splits a qualified target into a build file, target name and toolset.
  46. # NOTE: rsplit is used to disambiguate the Windows drive letter separator.
  47. target_split = target.rsplit(':', 1)
  48. if len(target_split) == 2:
  49. [build_file, target] = target_split
  50. else:
  51. build_file = None
  52. target_split = target.rsplit('#', 1)
  53. if len(target_split) == 2:
  54. [target, toolset] = target_split
  55. else:
  56. toolset = None
  57. return [build_file, target, toolset]
  58. def ResolveTarget(build_file, target, toolset):
  59. # This function resolves a target into a canonical form:
  60. # - a fully defined build file, either absolute or relative to the current
  61. # directory
  62. # - a target name
  63. # - a toolset
  64. #
  65. # build_file is the file relative to which 'target' is defined.
  66. # target is the qualified target.
  67. # toolset is the default toolset for that target.
  68. [parsed_build_file, target, parsed_toolset] = ParseQualifiedTarget(target)
  69. if parsed_build_file:
  70. if build_file:
  71. # If a relative path, parsed_build_file is relative to the directory
  72. # containing build_file. If build_file is not in the current directory,
  73. # parsed_build_file is not a usable path as-is. Resolve it by
  74. # interpreting it as relative to build_file. If parsed_build_file is
  75. # absolute, it is usable as a path regardless of the current directory,
  76. # and os.path.join will return it as-is.
  77. build_file = os.path.normpath(os.path.join(os.path.dirname(build_file),
  78. parsed_build_file))
  79. # Further (to handle cases like ../cwd), make it relative to cwd)
  80. if not os.path.isabs(build_file):
  81. build_file = RelativePath(build_file, '.')
  82. else:
  83. build_file = parsed_build_file
  84. if parsed_toolset:
  85. toolset = parsed_toolset
  86. return [build_file, target, toolset]
  87. def BuildFile(fully_qualified_target):
  88. # Extracts the build file from the fully qualified target.
  89. return ParseQualifiedTarget(fully_qualified_target)[0]
  90. def GetEnvironFallback(var_list, default):
  91. """Look up a key in the environment, with fallback to secondary keys
  92. and finally falling back to a default value."""
  93. for var in var_list:
  94. if var in os.environ:
  95. return os.environ[var]
  96. return default
  97. def QualifiedTarget(build_file, target, toolset):
  98. # "Qualified" means the file that a target was defined in and the target
  99. # name, separated by a colon, suffixed by a # and the toolset name:
  100. # /path/to/file.gyp:target_name#toolset
  101. fully_qualified = build_file + ':' + target
  102. if toolset:
  103. fully_qualified = fully_qualified + '#' + toolset
  104. return fully_qualified
  105. @memoize
  106. def RelativePath(path, relative_to, follow_path_symlink=True):
  107. # Assuming both |path| and |relative_to| are relative to the current
  108. # directory, returns a relative path that identifies path relative to
  109. # relative_to.
  110. # If |follow_symlink_path| is true (default) and |path| is a symlink, then
  111. # this method returns a path to the real file represented by |path|. If it is
  112. # false, this method returns a path to the symlink. If |path| is not a
  113. # symlink, this option has no effect.
  114. # Convert to normalized (and therefore absolute paths).
  115. if follow_path_symlink:
  116. path = os.path.realpath(path)
  117. else:
  118. path = os.path.abspath(path)
  119. relative_to = os.path.realpath(relative_to)
  120. # On Windows, we can't create a relative path to a different drive, so just
  121. # use the absolute path.
  122. if sys.platform == 'win32':
  123. if (os.path.splitdrive(path)[0].lower() !=
  124. os.path.splitdrive(relative_to)[0].lower()):
  125. return path
  126. # Split the paths into components.
  127. path_split = path.split(os.path.sep)
  128. relative_to_split = relative_to.split(os.path.sep)
  129. # Determine how much of the prefix the two paths share.
  130. prefix_len = len(os.path.commonprefix([path_split, relative_to_split]))
  131. # Put enough ".." components to back up out of relative_to to the common
  132. # prefix, and then append the part of path_split after the common prefix.
  133. relative_split = [os.path.pardir] * (len(relative_to_split) - prefix_len) + \
  134. path_split[prefix_len:]
  135. if len(relative_split) == 0:
  136. # The paths were the same.
  137. return ''
  138. # Turn it back into a string and we're done.
  139. return os.path.join(*relative_split)
  140. @memoize
  141. def InvertRelativePath(path, toplevel_dir=None):
  142. """Given a path like foo/bar that is relative to toplevel_dir, return
  143. the inverse relative path back to the toplevel_dir.
  144. E.g. os.path.normpath(os.path.join(path, InvertRelativePath(path)))
  145. should always produce the empty string, unless the path contains symlinks.
  146. """
  147. if not path:
  148. return path
  149. toplevel_dir = '.' if toplevel_dir is None else toplevel_dir
  150. return RelativePath(toplevel_dir, os.path.join(toplevel_dir, path))
  151. def FixIfRelativePath(path, relative_to):
  152. # Like RelativePath but returns |path| unchanged if it is absolute.
  153. if os.path.isabs(path):
  154. return path
  155. return RelativePath(path, relative_to)
  156. def UnrelativePath(path, relative_to):
  157. # Assuming that |relative_to| is relative to the current directory, and |path|
  158. # is a path relative to the dirname of |relative_to|, returns a path that
  159. # identifies |path| relative to the current directory.
  160. rel_dir = os.path.dirname(relative_to)
  161. return os.path.normpath(os.path.join(rel_dir, path))
  162. # re objects used by EncodePOSIXShellArgument. See IEEE 1003.1 XCU.2.2 at
  163. # http://www.opengroup.org/onlinepubs/009695399/utilities/xcu_chap02.html#tag_02_02
  164. # and the documentation for various shells.
  165. # _quote is a pattern that should match any argument that needs to be quoted
  166. # with double-quotes by EncodePOSIXShellArgument. It matches the following
  167. # characters appearing anywhere in an argument:
  168. # \t, \n, space parameter separators
  169. # # comments
  170. # $ expansions (quoted to always expand within one argument)
  171. # % called out by IEEE 1003.1 XCU.2.2
  172. # & job control
  173. # ' quoting
  174. # (, ) subshell execution
  175. # *, ?, [ pathname expansion
  176. # ; command delimiter
  177. # <, >, | redirection
  178. # = assignment
  179. # {, } brace expansion (bash)
  180. # ~ tilde expansion
  181. # It also matches the empty string, because "" (or '') is the only way to
  182. # represent an empty string literal argument to a POSIX shell.
  183. #
  184. # This does not match the characters in _escape, because those need to be
  185. # backslash-escaped regardless of whether they appear in a double-quoted
  186. # string.
  187. _quote = re.compile('[\t\n #$%&\'()*;<=>?[{|}~]|^$')
  188. # _escape is a pattern that should match any character that needs to be
  189. # escaped with a backslash, whether or not the argument matched the _quote
  190. # pattern. _escape is used with re.sub to backslash anything in _escape's
  191. # first match group, hence the (parentheses) in the regular expression.
  192. #
  193. # _escape matches the following characters appearing anywhere in an argument:
  194. # " to prevent POSIX shells from interpreting this character for quoting
  195. # \ to prevent POSIX shells from interpreting this character for escaping
  196. # ` to prevent POSIX shells from interpreting this character for command
  197. # substitution
  198. # Missing from this list is $, because the desired behavior of
  199. # EncodePOSIXShellArgument is to permit parameter (variable) expansion.
  200. #
  201. # Also missing from this list is !, which bash will interpret as the history
  202. # expansion character when history is enabled. bash does not enable history
  203. # by default in non-interactive shells, so this is not thought to be a problem.
  204. # ! was omitted from this list because bash interprets "\!" as a literal string
  205. # including the backslash character (avoiding history expansion but retaining
  206. # the backslash), which would not be correct for argument encoding. Handling
  207. # this case properly would also be problematic because bash allows the history
  208. # character to be changed with the histchars shell variable. Fortunately,
  209. # as history is not enabled in non-interactive shells and
  210. # EncodePOSIXShellArgument is only expected to encode for non-interactive
  211. # shells, there is no room for error here by ignoring !.
  212. _escape = re.compile(r'(["\\`])')
  213. def EncodePOSIXShellArgument(argument):
  214. """Encodes |argument| suitably for consumption by POSIX shells.
  215. argument may be quoted and escaped as necessary to ensure that POSIX shells
  216. treat the returned value as a literal representing the argument passed to
  217. this function. Parameter (variable) expansions beginning with $ are allowed
  218. to remain intact without escaping the $, to allow the argument to contain
  219. references to variables to be expanded by the shell.
  220. """
  221. if not isinstance(argument, str):
  222. argument = str(argument)
  223. if _quote.search(argument):
  224. quote = '"'
  225. else:
  226. quote = ''
  227. encoded = quote + re.sub(_escape, r'\\\1', argument) + quote
  228. return encoded
  229. def EncodePOSIXShellList(list):
  230. """Encodes |list| suitably for consumption by POSIX shells.
  231. Returns EncodePOSIXShellArgument for each item in list, and joins them
  232. together using the space character as an argument separator.
  233. """
  234. encoded_arguments = []
  235. for argument in list:
  236. encoded_arguments.append(EncodePOSIXShellArgument(argument))
  237. return ' '.join(encoded_arguments)
  238. def DeepDependencyTargets(target_dicts, roots):
  239. """Returns the recursive list of target dependencies."""
  240. dependencies = set()
  241. pending = set(roots)
  242. while pending:
  243. # Pluck out one.
  244. r = pending.pop()
  245. # Skip if visited already.
  246. if r in dependencies:
  247. continue
  248. # Add it.
  249. dependencies.add(r)
  250. # Add its children.
  251. spec = target_dicts[r]
  252. pending.update(set(spec.get('dependencies', [])))
  253. pending.update(set(spec.get('dependencies_original', [])))
  254. return list(dependencies - set(roots))
  255. def BuildFileTargets(target_list, build_file):
  256. """From a target_list, returns the subset from the specified build_file.
  257. """
  258. return [p for p in target_list if BuildFile(p) == build_file]
  259. def AllTargets(target_list, target_dicts, build_file):
  260. """Returns all targets (direct and dependencies) for the specified build_file.
  261. """
  262. bftargets = BuildFileTargets(target_list, build_file)
  263. deptargets = DeepDependencyTargets(target_dicts, bftargets)
  264. return bftargets + deptargets
  265. def WriteOnDiff(filename):
  266. """Write to a file only if the new contents differ.
  267. Arguments:
  268. filename: name of the file to potentially write to.
  269. Returns:
  270. A file like object which will write to temporary file and only overwrite
  271. the target if it differs (on close).
  272. """
  273. class Writer(object):
  274. """Wrapper around file which only covers the target if it differs."""
  275. def __init__(self):
  276. # Pick temporary file.
  277. tmp_fd, self.tmp_path = tempfile.mkstemp(
  278. suffix='.tmp',
  279. prefix=os.path.split(filename)[1] + '.gyp.',
  280. dir=os.path.split(filename)[0])
  281. try:
  282. self.tmp_file = os.fdopen(tmp_fd, 'wb')
  283. except Exception:
  284. # Don't leave turds behind.
  285. os.unlink(self.tmp_path)
  286. raise
  287. def __getattr__(self, attrname):
  288. # Delegate everything else to self.tmp_file
  289. return getattr(self.tmp_file, attrname)
  290. def close(self):
  291. try:
  292. # Close tmp file.
  293. self.tmp_file.close()
  294. # Determine if different.
  295. same = False
  296. try:
  297. same = filecmp.cmp(self.tmp_path, filename, False)
  298. except OSError, e:
  299. if e.errno != errno.ENOENT:
  300. raise
  301. if same:
  302. # The new file is identical to the old one, just get rid of the new
  303. # one.
  304. os.unlink(self.tmp_path)
  305. else:
  306. # The new file is different from the old one, or there is no old one.
  307. # Rename the new file to the permanent name.
  308. #
  309. # tempfile.mkstemp uses an overly restrictive mode, resulting in a
  310. # file that can only be read by the owner, regardless of the umask.
  311. # There's no reason to not respect the umask here, which means that
  312. # an extra hoop is required to fetch it and reset the new file's mode.
  313. #
  314. # No way to get the umask without setting a new one? Set a safe one
  315. # and then set it back to the old value.
  316. umask = os.umask(077)
  317. os.umask(umask)
  318. os.chmod(self.tmp_path, 0666 & ~umask)
  319. if sys.platform == 'win32' and os.path.exists(filename):
  320. # NOTE: on windows (but not cygwin) rename will not replace an
  321. # existing file, so it must be preceded with a remove. Sadly there
  322. # is no way to make the switch atomic.
  323. os.remove(filename)
  324. os.rename(self.tmp_path, filename)
  325. except Exception:
  326. # Don't leave turds behind.
  327. os.unlink(self.tmp_path)
  328. raise
  329. return Writer()
  330. def EnsureDirExists(path):
  331. """Make sure the directory for |path| exists."""
  332. try:
  333. os.makedirs(os.path.dirname(path))
  334. except OSError:
  335. pass
  336. def GetFlavor(params):
  337. """Returns |params.flavor| if it's set, the system's default flavor else."""
  338. flavors = {
  339. 'cygwin': 'win',
  340. 'win32': 'win',
  341. 'darwin': 'mac',
  342. }
  343. if 'flavor' in params:
  344. return params['flavor']
  345. if sys.platform in flavors:
  346. return flavors[sys.platform]
  347. if sys.platform.startswith('sunos'):
  348. return 'solaris'
  349. if sys.platform.startswith('freebsd'):
  350. return 'freebsd'
  351. if sys.platform.startswith('openbsd'):
  352. return 'openbsd'
  353. if sys.platform.startswith('netbsd'):
  354. return 'netbsd'
  355. if sys.platform.startswith('aix'):
  356. return 'aix'
  357. if sys.platform.startswith('zos'):
  358. return 'zos'
  359. if sys.platform.startswith('os390'):
  360. return 'zos'
  361. return 'linux'
  362. def CopyTool(flavor, out_path):
  363. """Finds (flock|mac|win)_tool.gyp in the gyp directory and copies it
  364. to |out_path|."""
  365. # aix and solaris just need flock emulation. mac and win use more complicated
  366. # support scripts.
  367. prefix = {
  368. 'aix': 'flock',
  369. 'solaris': 'flock',
  370. 'mac': 'mac',
  371. 'win': 'win'
  372. }.get(flavor, None)
  373. if not prefix:
  374. return
  375. # Slurp input file.
  376. source_path = os.path.join(
  377. os.path.dirname(os.path.abspath(__file__)), '%s_tool.py' % prefix)
  378. with open(source_path) as source_file:
  379. source = source_file.readlines()
  380. # Add header and write it out.
  381. tool_path = os.path.join(out_path, 'gyp-%s-tool' % prefix)
  382. with open(tool_path, 'w') as tool_file:
  383. tool_file.write(
  384. ''.join([source[0], '# Generated by gyp. Do not edit.\n'] + source[1:]))
  385. # Make file executable.
  386. os.chmod(tool_path, 0755)
  387. # From Alex Martelli,
  388. # http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/52560
  389. # ASPN: Python Cookbook: Remove duplicates from a sequence
  390. # First comment, dated 2001/10/13.
  391. # (Also in the printed Python Cookbook.)
  392. def uniquer(seq, idfun=None):
  393. if idfun is None:
  394. idfun = lambda x: x
  395. seen = {}
  396. result = []
  397. for item in seq:
  398. marker = idfun(item)
  399. if marker in seen: continue
  400. seen[marker] = 1
  401. result.append(item)
  402. return result
  403. # Based on http://code.activestate.com/recipes/576694/.
  404. class OrderedSet(collections.MutableSet):
  405. def __init__(self, iterable=None):
  406. self.end = end = []
  407. end += [None, end, end] # sentinel node for doubly linked list
  408. self.map = {} # key --> [key, prev, next]
  409. if iterable is not None:
  410. self |= iterable
  411. def __len__(self):
  412. return len(self.map)
  413. def __contains__(self, key):
  414. return key in self.map
  415. def add(self, key):
  416. if key not in self.map:
  417. end = self.end
  418. curr = end[1]
  419. curr[2] = end[1] = self.map[key] = [key, curr, end]
  420. def discard(self, key):
  421. if key in self.map:
  422. key, prev_item, next_item = self.map.pop(key)
  423. prev_item[2] = next_item
  424. next_item[1] = prev_item
  425. def __iter__(self):
  426. end = self.end
  427. curr = end[2]
  428. while curr is not end:
  429. yield curr[0]
  430. curr = curr[2]
  431. def __reversed__(self):
  432. end = self.end
  433. curr = end[1]
  434. while curr is not end:
  435. yield curr[0]
  436. curr = curr[1]
  437. # The second argument is an addition that causes a pylint warning.
  438. def pop(self, last=True): # pylint: disable=W0221
  439. if not self:
  440. raise KeyError('set is empty')
  441. key = self.end[1][0] if last else self.end[2][0]
  442. self.discard(key)
  443. return key
  444. def __repr__(self):
  445. if not self:
  446. return '%s()' % (self.__class__.__name__,)
  447. return '%s(%r)' % (self.__class__.__name__, list(self))
  448. def __eq__(self, other):
  449. if isinstance(other, OrderedSet):
  450. return len(self) == len(other) and list(self) == list(other)
  451. return set(self) == set(other)
  452. # Extensions to the recipe.
  453. def update(self, iterable):
  454. for i in iterable:
  455. if i not in self:
  456. self.add(i)
  457. class CycleError(Exception):
  458. """An exception raised when an unexpected cycle is detected."""
  459. def __init__(self, nodes):
  460. self.nodes = nodes
  461. def __str__(self):
  462. return 'CycleError: cycle involving: ' + str(self.nodes)
  463. def TopologicallySorted(graph, get_edges):
  464. r"""Topologically sort based on a user provided edge definition.
  465. Args:
  466. graph: A list of node names.
  467. get_edges: A function mapping from node name to a hashable collection
  468. of node names which this node has outgoing edges to.
  469. Returns:
  470. A list containing all of the node in graph in topological order.
  471. It is assumed that calling get_edges once for each node and caching is
  472. cheaper than repeatedly calling get_edges.
  473. Raises:
  474. CycleError in the event of a cycle.
  475. Example:
  476. graph = {'a': '$(b) $(c)', 'b': 'hi', 'c': '$(b)'}
  477. def GetEdges(node):
  478. return re.findall(r'\$\(([^))]\)', graph[node])
  479. print TopologicallySorted(graph.keys(), GetEdges)
  480. ==>
  481. ['a', 'c', b']
  482. """
  483. get_edges = memoize(get_edges)
  484. visited = set()
  485. visiting = set()
  486. ordered_nodes = []
  487. def Visit(node):
  488. if node in visiting:
  489. raise CycleError(visiting)
  490. if node in visited:
  491. return
  492. visited.add(node)
  493. visiting.add(node)
  494. for neighbor in get_edges(node):
  495. Visit(neighbor)
  496. visiting.remove(node)
  497. ordered_nodes.insert(0, node)
  498. for node in sorted(graph):
  499. Visit(node)
  500. return ordered_nodes
  501. def CrossCompileRequested():
  502. # TODO: figure out how to not build extra host objects in the
  503. # non-cross-compile case when this is enabled, and enable unconditionally.
  504. return (os.environ.get('GYP_CROSSCOMPILE') or
  505. os.environ.get('AR_host') or
  506. os.environ.get('CC_host') or
  507. os.environ.get('CXX_host') or
  508. os.environ.get('AR_target') or
  509. os.environ.get('CC_target') or
  510. os.environ.get('CXX_target'))