Skip to content

Commit 0f9facf

Browse files
scbeddscbedd
committed
successfully pulled the tox_specific items into ox_harness.py. time to start getting the pooling working
1 parent 1455d01 commit 0f9facf

File tree

4 files changed

+108
-130
lines changed

4 files changed

+108
-130
lines changed

‎.gitignore

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,8 @@ TestResults/
3333
test-junit-*.xml
3434
pylint-*.out.txt
3535
coverage-*.xml
36+
stderr.txt
37+
stdout.txt
3638

3739
# tox environment folders
3840
.tox/

‎scripts/devops_tasks/common_tasks.py

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@
1212
from pathlib import Path
1313
from subprocess import check_call, CalledProcessError
1414
import os
15+
import errno
16+
import shutil
1517
import sys
1618
import logging
1719

@@ -28,6 +30,12 @@
2830
"azure-cognitiveservices-vision-contentmoderator" # same issue with this one
2931
]
3032

33+
MANAGEMENT_PACKAGE_IDENTIFIERS = [
34+
"mgmt",
35+
"azure-cognitiveservices",
36+
"azure-servicefabric",
37+
"azure-nspkg",
38+
]
3139

3240
def cleanup_folder(target_folder):
3341
for file in os.listdir(target_folder):
@@ -38,6 +46,16 @@ def cleanup_folder(target_folder):
3846
except Exception as e:
3947
logging.error(e)
4048

49+
# helper functions
50+
def clean_coverage(coverage_dir):
51+
try:
52+
os.mkdir(coverage_dir)
53+
except OSError as e:
54+
if e.errno == errno.EEXIST:
55+
logging.info("Coverage dir already exists. Cleaning.")
56+
cleanup_folder(coverage_dir)
57+
else:
58+
raise
4159

4260
# this function is where a glob string gets translated to a list of packages
4361
# It is called by both BUILD (package) and TEST. In the future, this function will be the central location

‎scripts/devops_tasks/setup_execute_tests.py

Lines changed: 6 additions & 125 deletions
Original file line numberDiff line numberDiff line change
@@ -14,41 +14,20 @@
1414
from pathlib import Path
1515
import os
1616
import errno
17-
import glob
1817
import shutil
18+
import glob
1919
import logging
2020
import pdb
21-
from common_tasks import process_glob_string, run_check_call, cleanup_folder
22-
from ToxWorkItem import ToxWorkItem
23-
from simple_threadpool import execute_parallel_workload
24-
from subprocess import Popen, PIPE, STDOUT
21+
from common_tasks import process_glob_string, run_check_call, cleanup_folder, clean_coverage, MANAGEMENT_PACKAGE_IDENTIFIERS
22+
from tox_harness import prep_and_run_tox
23+
2524

2625
logging.getLogger().setLevel(logging.INFO)
2726

2827
root_dir = os.path.abspath(os.path.join(os.path.abspath(__file__), "..", "..", ".."))
2928
coverage_dir = os.path.join(root_dir, "_coverage/")
3029
dev_setup_script_location = os.path.join(root_dir, "scripts/dev_setup.py")
3130

32-
DEFAULT_TOX_INI_LOCATION = os.path.join(root_dir, "eng/tox/tox.ini")
33-
34-
MANAGEMENT_PACKAGE_IDENTIFIERS = [
35-
"mgmt",
36-
"azure-cognitiveservices",
37-
"azure-servicefabric",
38-
"azure-nspkg",
39-
]
40-
41-
# helper functions
42-
def clean_coverage():
43-
try:
44-
os.mkdir(coverage_dir)
45-
except OSError as e:
46-
if e.errno == errno.EEXIST:
47-
logging.info("Coverage dir already exists. Cleaning.")
48-
cleanup_folder(coverage_dir)
49-
else:
50-
raise
51-
5231
def log_file(file_location, is_error=False):
5332
with open(file_location, 'r') as file:
5433
for line in file:
@@ -79,36 +58,6 @@ def collect_pytest_coverage_files(targeted_packages):
7958

8059
shutil.move(source, dest)
8160

82-
# TODO, dedup this function with collect_pytest
83-
def collect_tox_coverage_files(targeted_packages):
84-
root_coverage_dir = os.path.join(root_dir, "_coverage/")
85-
86-
clean_coverage()
87-
88-
coverage_files = []
89-
# generate coverage files
90-
for package_dir in [package for package in targeted_packages]:
91-
coverage_file = os.path.join(package_dir, ".coverage")
92-
if os.path.isfile(coverage_file):
93-
destination_file = os.path.join(
94-
root_coverage_dir, ".coverage_{}".format(os.path.basename(package_dir))
95-
)
96-
shutil.copyfile(coverage_file, destination_file)
97-
coverage_files.append(destination_file)
98-
99-
logging.info("Visible uncombined .coverage files: {}".format(coverage_files))
100-
101-
if len(coverage_files):
102-
cov_cmd_array = ["coverage", "combine"]
103-
cov_cmd_array.extend(coverage_files)
104-
105-
# merge them with coverage combine and copy to root
106-
run_check_call(cov_cmd_array, os.path.join(root_dir, "_coverage/"))
107-
108-
source = os.path.join(coverage_dir, "./.coverage")
109-
dest = os.path.join(root_dir, ".coverage")
110-
111-
shutil.move(source, dest)
11261

11362
def prep_tests(targeted_packages, python_version):
11463
logging.info("running test setup for {}".format(targeted_packages))
@@ -125,7 +74,7 @@ def prep_tests(targeted_packages, python_version):
12574
def run_tests(targeted_packages, python_version, test_output_location, test_res):
12675
err_results = []
12776

128-
clean_coverage()
77+
clean_coverage(coverage_dir)
12978

13079
# base command array without a targeted package
13180
command_array = [python_version, "-m", "pytest"]
@@ -194,74 +143,6 @@ def run_tests(targeted_packages, python_version, test_output_location, test_res)
194143
if err_results:
195144
exit(1)
196145

197-
def prep_and_run_tox(targeted_packages, tox_env, options_array=[]):
198-
tox_command_tuples = []
199-
200-
for index, package_dir in enumerate(targeted_packages):
201-
destination_tox_ini = os.path.join(package_dir, "tox.ini")
202-
destination_dev_req = os.path.join(package_dir, "dev_requirements.txt")
203-
tox_execution_array = ["tox"]
204-
local_options_array = options_array[:]
205-
206-
# if we are targeting only packages that are management plane, it is a possibility
207-
# that no tests running is an acceptable situation
208-
# we explicitly handle this here.
209-
if all(
210-
map(
211-
lambda x: any(
212-
[pkg_id in x for pkg_id in MANAGEMENT_PACKAGE_IDENTIFIERS]
213-
),
214-
[package_dir],
215-
)
216-
):
217-
local_options_array.append("--suppress-no-test-exit-code")
218-
219-
# if not present, re-use base
220-
if not os.path.exists(destination_tox_ini):
221-
logging.info("No customized tox.ini present, using common eng/tox/tox.ini.")
222-
tox_execution_array.extend(["-c", DEFAULT_TOX_INI_LOCATION])
223-
224-
# handle empty file
225-
if not os.path.exists(destination_dev_req):
226-
logging.info("No dev_requirements present.")
227-
with open(destination_dev_req, "w+") as file:
228-
file.write("-e ../../../tools/azure-sdk-tools")
229-
230-
if tox_env:
231-
tox_execution_array.extend(["-e", tox_env])
232-
233-
if local_options_array:
234-
tox_execution_array.extend(["--"] + local_options_array)
235-
236-
tox_command_tuples.append((tox_execution_array, package_dir))
237-
238-
procs_list = []
239-
240-
for cmd_tuple in tox_command_tuples:
241-
package_dir = cmd_tuple[1]
242-
243-
logging.info(
244-
"Running tox for {}. {} of {}.".format(
245-
package_dir, index, len(targeted_packages)
246-
)
247-
)
248-
249-
with open(os.path.join(package_dir, 'stdout.txt'), 'w') as f_stdout, open(os.path.join(package_dir, 'stderr.txt'), 'w') as f_stderr:
250-
proc = Popen(cmd_tuple[0], stdout=f_stdout, stderr=f_stderr, cwd=package_dir, env=os.environ.copy())
251-
procs_list.append(proc)
252-
253-
failed_test_run = False
254-
255-
for index, proc in enumerate(procs_list):
256-
proc.wait()
257-
258-
if proc.returncode != 0:
259-
logging.error("Package returned with code {}".format(proc.returncode))
260-
261-
# TODO: get a bit smarter here
262-
if not tox_env:
263-
collect_tox_coverage_files(targeted_packages)
264-
265146
def execute_global_install_and_test(
266147
parsed_args, targeted_packages, extended_pytest_args
267148
):
@@ -286,7 +167,7 @@ def execute_tox_harness(parsed_args, targeted_packages, extended_pytest_args):
286167
if args.mark_arg:
287168
extended_pytest_args.extend(["-m", "'{}'".format(args.mark_arg)])
288169

289-
prep_and_run_tox(targeted_packages, args.tox_env, extended_pytest_args)
170+
prep_and_run_tox(targeted_packages, args.tox_env, extended_pytest_args, parsed_args.tparallel)
290171

291172
if __name__ == "__main__":
292173
parser = argparse.ArgumentParser(

‎scripts/devops_tasks/tox_harness.py

Lines changed: 82 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,27 @@
11
import sys
2+
import os
3+
import errno
4+
import shutil
5+
26
if sys.version_info < (3, 0):
37
from Queue import Queue
48
else:
59
from queue import Queue
610
from threading import Thread
711
import multiprocessing
12+
13+
from subprocess import Popen, PIPE, STDOUT
14+
from common_tasks import process_glob_string, run_check_call, cleanup_folder, clean_coverage, MANAGEMENT_PACKAGE_IDENTIFIERS
15+
16+
import logging
17+
logging.getLogger().setLevel(logging.INFO)
18+
819
import pdb
920

21+
root_dir = os.path.abspath(os.path.join(os.path.abspath(__file__), "..", "..", ".."))
22+
coverage_dir = os.path.join(root_dir, "_coverage/")
23+
DEFAULT_TOX_INI_LOCATION = os.path.join(root_dir, "eng/tox/tox.ini")
24+
1025
class ToxWorkItem:
1126
def __init__(self, target_package_path, tox_env, options_array):
1227
self.target_package_path = target_package_path
@@ -51,14 +66,65 @@ def wait_completion(self):
5166
"""Wait for completion of all the tasks in the queue"""
5267
self.tasks.join()
5368

54-
def prep_and_run_tox(targeted_packages, tox_env, options_array=[]):
69+
# TODO, dedup this function with collect_pytest
70+
def collect_tox_coverage_files(targeted_packages):
71+
root_coverage_dir = os.path.join(root_dir, "_coverage/")
72+
73+
clean_coverage(coverage_dir)
74+
75+
coverage_files = []
76+
# generate coverage files
77+
for package_dir in [package for package in targeted_packages]:
78+
coverage_file = os.path.join(package_dir, ".coverage")
79+
if os.path.isfile(coverage_file):
80+
destination_file = os.path.join(
81+
root_coverage_dir, ".coverage_{}".format(os.path.basename(package_dir))
82+
)
83+
shutil.copyfile(coverage_file, destination_file)
84+
coverage_files.append(destination_file)
85+
86+
logging.info("Visible uncombined .coverage files: {}".format(coverage_files))
87+
88+
if len(coverage_files):
89+
cov_cmd_array = ["coverage", "combine"]
90+
cov_cmd_array.extend(coverage_files)
91+
92+
# merge them with coverage combine and copy to root
93+
run_check_call(cov_cmd_array, os.path.join(root_dir, "_coverage/"))
94+
95+
source = os.path.join(coverage_dir, "./.coverage")
96+
dest = os.path.join(root_dir, ".coverage")
97+
98+
shutil.move(source, dest)
99+
100+
def execute_tox_parallel(tox_command_tuples):
101+
procs_list = []
102+
103+
for index, cmd_tuple in enumerate(tox_command_tuples):
104+
package_dir = cmd_tuple[1]
55105

56-
for index, package_dir in enumerate(targeted_packages):
57106
logging.info(
58-
"Running tox for {}. {} of {}.".format(
59-
package_dir, index, len(targeted_packages)
107+
"Starting tox for {}. {} of {}.".format(
108+
os.path.basename(package_dir), index, len(tox_command_tuples)
60109
)
61110
)
111+
112+
with open(os.path.join(package_dir, 'stdout.txt'), 'w') as f_stdout, open(os.path.join(package_dir, 'stderr.txt'), 'w') as f_stderr:
113+
proc = Popen(cmd_tuple[0], stdout=f_stdout, stderr=f_stderr, cwd=package_dir, env=os.environ.copy())
114+
procs_list.append(proc)
115+
116+
failed_test_run = False
117+
118+
for index, proc in enumerate(procs_list):
119+
proc.wait()
120+
121+
if proc.returncode != 0:
122+
logging.error("Package returned with code {}".format(proc.returncode))
123+
124+
def prep_and_run_tox(targeted_packages, tox_env, options_array=[], is_parallel=False):
125+
tox_command_tuples = []
126+
127+
for index, package_dir in enumerate(targeted_packages):
62128
destination_tox_ini = os.path.join(package_dir, "tox.ini")
63129
destination_dev_req = os.path.join(package_dir, "dev_requirements.txt")
64130
tox_execution_array = ["tox"]
@@ -94,7 +160,18 @@ def prep_and_run_tox(targeted_packages, tox_env, options_array=[]):
94160
if local_options_array:
95161
tox_execution_array.extend(["--"] + local_options_array)
96162

97-
run_check_call(tox_execution_array, package_dir)
163+
tox_command_tuples.append((tox_execution_array, package_dir))
164+
165+
if is_parallel:
166+
execute_tox_parallel(tox_command_tuples)
167+
else:
168+
for index, cmd_tuple in enumerate(tox_command_tuples):
169+
logging.info(
170+
"Running tox for {}. {} of {}.".format(
171+
os.path.basename(cmd_tuple[1]), index, len(tox_command_tuples)
172+
)
173+
)
174+
run_check_call(cmd_tuple[0], cmd_tuple[1])
98175

99176
# TODO: get a bit smarter here
100177
if not tox_env:

0 commit comments

Comments
 (0)