Skip to content

Commit 85207a0

Browse files
AndreMirasAndreMiras
authored
Merge pull request kivy#1862 from JonasT/fix_leftover_constraint_file
Fix various setup.py processing bugs
2 parents 80e4f05 + 6ae074e commit 85207a0

File tree

4 files changed

+131
-106
lines changed

4 files changed

+131
-106
lines changed

‎pythonforandroid/build.py

Lines changed: 123 additions & 103 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
11
from __future__ import print_function
22

3-
from os.path import (join, realpath, dirname, expanduser, exists,
4-
split, isdir)
3+
from os.path import (
4+
abspath, join, realpath, dirname, expanduser, exists,
5+
split, isdir
6+
)
57
from os import environ
68
import copy
79
import os
@@ -590,6 +592,120 @@ def project_has_setup_py(project_dir):
590592
return False
591593

592594

595+
def run_setuppy_install(ctx, project_dir, env=None):
596+
if env is None:
597+
env = dict()
598+
599+
with current_directory(project_dir):
600+
info('got setup.py or similar, running project install. ' +
601+
'(disable this behavior with --ignore-setup-py)')
602+
603+
# Compute & output the constraints we will use:
604+
info('Contents that will be used for constraints.txt:')
605+
constraints = subprocess.check_output([
606+
join(
607+
ctx.build_dir, "venv", "bin", "pip"
608+
),
609+
"freeze"
610+
], env=copy.copy(env))
611+
try:
612+
constraints = constraints.decode("utf-8", "replace")
613+
except AttributeError:
614+
pass
615+
info(constraints)
616+
617+
# Make sure all packages found are fixed in version
618+
# by writing a constraint file, to avoid recipes being
619+
# upgraded & reinstalled:
620+
with open('._tmp_p4a_recipe_constraints.txt', 'wb') as fileh:
621+
fileh.write(constraints.encode("utf-8", "replace"))
622+
try:
623+
624+
info('Populating venv\'s site-packages with '
625+
'ctx.get_site_packages_dir()...')
626+
627+
# Copy dist contents into site-packages for discovery.
628+
# Why this is needed:
629+
# --target is somewhat evil and messes with discovery of
630+
# packages in PYTHONPATH if that also includes the target
631+
# folder. So we need to use the regular virtualenv
632+
# site-packages folder instead.
633+
# Reference:
634+
# https://github.com/pypa/pip/issues/6223
635+
ctx_site_packages_dir = os.path.normpath(
636+
os.path.abspath(ctx.get_site_packages_dir())
637+
)
638+
venv_site_packages_dir = os.path.normpath(os.path.join(
639+
ctx.build_dir, "venv", "lib", [
640+
f for f in os.listdir(os.path.join(
641+
ctx.build_dir, "venv", "lib"
642+
)) if f.startswith("python")
643+
][0], "site-packages"
644+
))
645+
copied_over_contents = []
646+
for f in os.listdir(ctx_site_packages_dir):
647+
full_path = os.path.join(ctx_site_packages_dir, f)
648+
if not os.path.exists(os.path.join(
649+
venv_site_packages_dir, f
650+
)):
651+
if os.path.isdir(full_path):
652+
shutil.copytree(full_path, os.path.join(
653+
venv_site_packages_dir, f
654+
))
655+
else:
656+
shutil.copy2(full_path, os.path.join(
657+
venv_site_packages_dir, f
658+
))
659+
copied_over_contents.append(f)
660+
661+
# Get listing of virtualenv's site-packages, to see the
662+
# newly added things afterwards & copy them back into
663+
# the distribution folder / build context site-packages:
664+
previous_venv_contents = os.listdir(
665+
venv_site_packages_dir
666+
)
667+
668+
# Actually run setup.py:
669+
info('Launching package install...')
670+
shprint(sh.bash, '-c', (
671+
"'" + join(
672+
ctx.build_dir, "venv", "bin", "pip"
673+
).replace("'", "'\"'\"'") + "' " +
674+
"install -c ._tmp_p4a_recipe_constraints.txt -v ."
675+
).format(ctx.get_site_packages_dir().
676+
replace("'", "'\"'\"'")),
677+
_env=copy.copy(env))
678+
679+
# Go over all new additions and copy them back:
680+
info('Copying additions resulting from setup.py back '
681+
'into ctx.get_site_packages_dir()...')
682+
new_venv_additions = []
683+
for f in (set(os.listdir(venv_site_packages_dir)) -
684+
set(previous_venv_contents)):
685+
new_venv_additions.append(f)
686+
full_path = os.path.join(venv_site_packages_dir, f)
687+
if os.path.isdir(full_path):
688+
shutil.copytree(full_path, os.path.join(
689+
ctx_site_packages_dir, f
690+
))
691+
else:
692+
shutil.copy2(full_path, os.path.join(
693+
ctx_site_packages_dir, f
694+
))
695+
696+
# Undo all the changes we did to the venv-site packages:
697+
info('Reverting additions to '
698+
'virtualenv\'s site-packages...')
699+
for f in set(copied_over_contents + new_venv_additions):
700+
full_path = os.path.join(venv_site_packages_dir, f)
701+
if os.path.isdir(full_path):
702+
shutil.rmtree(full_path)
703+
else:
704+
os.remove(full_path)
705+
finally:
706+
os.remove("._tmp_p4a_recipe_constraints.txt")
707+
708+
593709
def run_pymodules_install(ctx, modules, project_dir=None,
594710
ignore_setup_py=False):
595711
""" This function will take care of all non-recipe things, by:
@@ -605,6 +721,10 @@ def run_pymodules_install(ctx, modules, project_dir=None,
605721
info('*** PYTHON PACKAGE / PROJECT INSTALL STAGE ***')
606722
modules = list(filter(ctx.not_has_package, modules))
607723

724+
# We change current working directory later, so this
725+
# has to be an absolute path:
726+
project_dir = abspath(project_dir)
727+
608728
# Bail out if no python deps and no setup.py to process:
609729
if not modules and (
610730
ignore_setup_py or
@@ -697,107 +817,7 @@ def run_pymodules_install(ctx, modules, project_dir=None,
697817
if project_dir is not None and (
698818
project_has_setup_py(project_dir) and not ignore_setup_py
699819
):
700-
with current_directory(project_dir):
701-
info('got setup.py or similar, running project install. ' +
702-
'(disable this behavior with --ignore-setup-py)')
703-
704-
# Compute & output the constraints we will use:
705-
info('Contents that will be used for constraints.txt:')
706-
constraints = subprocess.check_output([
707-
join(
708-
ctx.build_dir, "venv", "bin", "pip"
709-
),
710-
"freeze"
711-
], env=copy.copy(env))
712-
try:
713-
constraints = constraints.decode("utf-8", "replace")
714-
except AttributeError:
715-
pass
716-
info(constraints)
717-
718-
# Make sure all packages found are fixed in version
719-
# by writing a constraint file, to avoid recipes being
720-
# upgraded & reinstalled:
721-
with open('constraints.txt', 'wb') as fileh:
722-
fileh.write(constraints.encode("utf-8", "replace"))
723-
724-
info('Populating venv\'s site-packages with '
725-
'ctx.get_site_packages_dir()...')
726-
727-
# Copy dist contents into site-packages for discovery.
728-
# Why this is needed:
729-
# --target is somewhat evil and messes with discovery of
730-
# packages in PYTHONPATH if that also includes the target
731-
# folder. So we need to use the regular virtualenv
732-
# site-packages folder instead.
733-
# Reference:
734-
# https://github.com/pypa/pip/issues/6223
735-
ctx_site_packages_dir = os.path.normpath(
736-
os.path.abspath(ctx.get_site_packages_dir())
737-
)
738-
venv_site_packages_dir = os.path.normpath(os.path.join(
739-
ctx.build_dir, "venv", "lib", [
740-
f for f in os.listdir(os.path.join(
741-
ctx.build_dir, "venv", "lib"
742-
)) if f.startswith("python")
743-
][0], "site-packages"
744-
))
745-
copied_over_contents = []
746-
for f in os.listdir(ctx_site_packages_dir):
747-
full_path = os.path.join(ctx_site_packages_dir, f)
748-
if not os.path.exists(os.path.join(
749-
venv_site_packages_dir, f
750-
)):
751-
if os.path.isdir(full_path):
752-
shutil.copytree(full_path, os.path.join(
753-
venv_site_packages_dir, f
754-
))
755-
else:
756-
shutil.copy2(full_path, os.path.join(
757-
venv_site_packages_dir, f
758-
))
759-
copied_over_contents.append(f)
760-
761-
# Get listing of virtualenv's site-packages, to see the
762-
# newly added things afterwards & copy them back into
763-
# the distribution folder / build context site-packages:
764-
previous_venv_contents = os.listdir(venv_site_packages_dir)
765-
766-
# Actually run setup.py:
767-
info('Launching package install...')
768-
shprint(sh.bash, '-c', (
769-
"'" + join(
770-
ctx.build_dir, "venv", "bin", "pip"
771-
).replace("'", "'\"'\"'") + "' " +
772-
"install -c constraints.txt -v ."
773-
).format(ctx.get_site_packages_dir().replace("'", "'\"'\"'")),
774-
_env=copy.copy(env))
775-
776-
# Go over all new additions and copy them back:
777-
info('Copying additions resulting from setup.py back ' +
778-
'into ctx.get_site_packages_dir()...')
779-
new_venv_additions = []
780-
for f in (set(os.listdir(venv_site_packages_dir)) -
781-
set(previous_venv_contents)):
782-
new_venv_additions.append(f)
783-
full_path = os.path.join(venv_site_packages_dir, f)
784-
if os.path.isdir(full_path):
785-
shutil.copytree(full_path, os.path.join(
786-
ctx_site_packages_dir, f
787-
))
788-
else:
789-
shutil.copy2(full_path, os.path.join(
790-
ctx_site_packages_dir, f
791-
))
792-
793-
# Undo all the changes we did to the venv-site packages:
794-
info('Reverting additions to virtualenv\'s site-packages...')
795-
for f in set(copied_over_contents + new_venv_additions):
796-
full_path = os.path.join(venv_site_packages_dir, f)
797-
if os.path.isdir(full_path):
798-
shutil.rmtree(full_path)
799-
else:
800-
os.remove(full_path)
820+
run_setuppy_install(ctx, project_dir, env)
801821
elif not ignore_setup_py:
802822
info("No setup.py found in project directory: " +
803823
str(project_dir)

‎pythonforandroid/pythonpackage.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -559,7 +559,7 @@ def parse_as_folder_reference(dep):
559559
# Check if this is either not an url, or a file URL:
560560
if dep.startswith(("/", "file://")) or (
561561
dep.find("/") > 0 and
562-
dep.find("://") < 0):
562+
dep.find("://") < 0) or (dep in ["", "."]):
563563
if dep.startswith("file://"):
564564
dep = urlunquote(urlparse(dep).path)
565565
return dep
@@ -689,7 +689,7 @@ def get_package_dependencies(package,
689689
for package_dep in current_queue:
690690
new_reqs = set()
691691
if verbose:
692-
print("get_package_dependencies: resolving dependecy "
692+
print("get_package_dependencies: resolving dependency "
693693
"to package name: ".format(package_dep))
694694
package = get_package_name(package_dep)
695695
if package.lower() in packages_processed:

‎pythonforandroid/toolchain.py

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -631,7 +631,10 @@ def add_parser(subparsers, *args, **kwargs):
631631
args.requirements += u",".join(dependencies)
632632
except ValueError:
633633
# Not a python package, apparently.
634-
pass
634+
warning(
635+
"Processing failed, is this project a valid "
636+
"package? Will continue WITHOUT setup.py deps."
637+
)
635638

636639
# Parse --requirements argument list:
637640
for requirement in split_argument_list(args.requirements):

‎tests/test_pythonpackage_basic.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -187,6 +187,8 @@ def test_is_filesystem_path():
187187
assert not is_filesystem_path("test @ bla")
188188
assert is_filesystem_path("/abc/c@d")
189189
assert not is_filesystem_path("https://user:pw@host/")
190+
assert is_filesystem_path(".")
191+
assert is_filesystem_path("")
190192

191193

192194
def test_parse_as_folder_reference():

0 commit comments

Comments
 (0)