import sys import os import subprocess import re import ast import tempfile from buildtools import unify_requirements from buildtools.version import executable from buildtools.version import WIN from buildtools.helper import print_exception_indented def call(args, cwd=None): print("Calling " + " ".join(args)) sys.stdout.flush() sys.stderr.flush() subprocess.check_call(args, cwd=cwd) class Venv(object): def __init__(self, path, version): self.environ = {} self.path = path exe_ext = [ "Scripts", "python.exe" ] if WIN else [ "bin", "python" ] self.venv_executable = os.path.join(self.path, *exe_ext) self._lib = None self.pip_upgraded = False self.empty_folder = tempfile.mkdtemp(prefix="empty", dir=os.environ["LGTM_WORKSPACE"]) self.version = version def create(self): if self.version < 3: venv = ["-m", "virtualenv", "--never-download"] else: venv = ["-m", "venv"] call(executable(self.version) + venv + [self.path], cwd=self.empty_folder) def upgrade_pip(self): 'Make sure that pip has been upgraded to latest version' if self.pip_upgraded: return self.pip([ "install", "--upgrade", "pip"]) self.pip_upgraded = True def pip(self, args): call([self.venv_executable, "-m", "pip"] + args, cwd=self.empty_folder) @property def lib(self): if self._lib is None: try: tools = os.path.join(os.environ['SEMMLE_DIST'], "tools") get_venv_lib = os.path.join(tools, "get_venv_lib.py") if os.path.exists(self.venv_executable): python_executable = [self.venv_executable] else: python_executable = executable(self.version) args = python_executable + [get_venv_lib] print("Calling " + " ".join(args)) sys.stdout.flush() sys.stderr.flush() self._lib = subprocess.check_output(args) if sys.version_info >= (3,): self._lib = str(self._lib, sys.getfilesystemencoding()) self._lib = self._lib.rstrip("\r\n") except: lib_ext = ["Lib"] if WIN else [ "lib" ] self._lib = os.path.join(self.path, *lib_ext) print('Error trying to run get_venv_lib (this is Python {})'.format(sys.version[:5])) print_exception_indented() return self._lib def venv_path(): return os.path.join(os.environ["LGTM_WORKSPACE"], "venv") def system_packages(version): output = subprocess.check_output(executable(version) + [ "-c", "import sys; print(sys.path)"]) if sys.version_info >= (3,): output = str(output, sys.getfilesystemencoding()) paths = ast.literal_eval(output.strip()) return [ path for path in paths if ("dist-packages" in path or "site-packages" in path) ] REQUIREMENTS_TAG = "LGTM_PYTHON_SETUP_REQUIREMENTS" EXCLUDE_REQUIREMENTS_TAG = "LGTM_PYTHON_SETUP_EXCLUDE_REQUIREMENTS" def main(version, root, requirement_files): # We import `auto_install` here, as it has a dependency on the `packaging` # module. For the CodeQL CLI (where we do not install any packages) we never # run the `main` function, and so there is no need to always import this # dependency. from buildtools import auto_install print("version, root, requirement_files", version, root, requirement_files) venv = Venv(venv_path(), version) venv.create() if REQUIREMENTS_TAG in os.environ: if not auto_install.install(os.environ[REQUIREMENTS_TAG], venv): sys.exit(1) requirements_from_setup = os.path.join(os.environ["LGTM_WORKSPACE"], "setup_requirements.txt") args = [ venv.venv_executable, os.path.join(os.environ["SEMMLE_DIST"], "tools", "convert_setup.py"), root, requirements_from_setup] + system_packages(version) print("Calling " + " ".join(args)) sys.stdout.flush() sys.stderr.flush() #We don't care if this fails, we only care if `requirements_from_setup` was created. subprocess.call(args) if os.path.exists(requirements_from_setup): requirement_files = [ requirements_from_setup ] + requirement_files[1:] print("Requirement files: " + str(requirement_files)) requirements = unify_requirements.gather(requirement_files) if EXCLUDE_REQUIREMENTS_TAG in os.environ: excludes = os.environ[EXCLUDE_REQUIREMENTS_TAG].splitlines() print("Excluding ", excludes) regex = re.compile("|".join(exclude + r'\b' for exclude in excludes)) requirements = [ req for req in requirements if not regex.match(req) ] err = 0 if auto_install.install(requirements, venv) else 1 sys.exit(err) def get_library(version): return Venv(venv_path(), version).lib if __name__ == "__main__": version, root, requirement_files = sys.argv[1], sys.argv[2], sys.argv[3:] version = int(version) main(version, root, requirement_files)