Skip to content

Commit db0519c

Browse files
JFoedererJFoederer
authored
Implement get_library_information interface for Remote libraries (robotframework#3802)
This enhances performance of getting information about keywords with big remote libraries considerably. See issue robotframework#3362 for more information.
1 parent e1d602a commit db0519c

File tree

6 files changed

+118
-27
lines changed

6 files changed

+118
-27
lines changed

‎atest/robot/standard_libraries/remote/library_info.robot

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
*** Settings ***
2+
Suite Setup Run Remote Tests library_info.robot libraryinfo.py
3+
Resource remote_resource.robot
4+
5+
*** Test Cases ***
6+
Load large library
7+
Check Test Case ${TESTNAME}

‎atest/testdata/standard_libraries/remote/library_info.robot

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
*** Settings ***
2+
Library Remote http://127.0.0.1:${PORT}
3+
Suite Setup Set Log Level DEBUG
4+
5+
*** Variables ***
6+
${PORT} 8270
7+
8+
*** Test Cases ***
9+
Load large library
10+
${ret}= some keyword
11+
Should be equal ${ret} some
12+
${ret}= keyword 7777
13+
Should be equal ${ret} 7777

‎atest/testdata/standard_libraries/remote/libraryinfo.py

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
import sys
2+
3+
from remoteserver import RemoteServer, keyword
4+
5+
6+
class BulkLoadRemoteServer(RemoteServer):
7+
8+
def _register_functions(self):
9+
"""
10+
Individual get_keyword_* methods are not registered.
11+
This removes the fall back scenario should get_library_information fail.
12+
"""
13+
self.register_function(self.get_library_information)
14+
self.register_function(self.run_keyword)
15+
16+
def get_library_information(self):
17+
info_dict = dict()
18+
for kw in self.get_keyword_names():
19+
info_dict[kw] = dict(args=self.get_keyword_arguments(kw),
20+
tags=self.get_keyword_tags(kw),
21+
doc=self.get_keyword_documentation(kw))
22+
return info_dict
23+
24+
class The10001KeywordsLibrary(object):
25+
26+
def __init__(self):
27+
def count(n): return lambda: "%d"%n
28+
for i in range(10000):
29+
setattr(self, "keyword_%d"%i, count(i))
30+
31+
def some_keyword(self):
32+
return "some"
33+
34+
35+
if __name__ == '__main__':
36+
BulkLoadRemoteServer(The10001KeywordsLibrary(), *sys.argv[1:])

‎atest/testdata/standard_libraries/remote/remoteserver.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -65,11 +65,11 @@ def _format_kwo(self, arg, defaults):
6565

6666
def get_keyword_tags(self, name):
6767
kw = getattr(self.library, name)
68-
return getattr(kw, 'robot_tags', None)
68+
return getattr(kw, 'robot_tags', [])
6969

7070
def get_keyword_documentation(self, name):
7171
kw = getattr(self.library, name)
72-
return inspect.getdoc(kw)
72+
return inspect.getdoc(kw) or ''
7373

7474
def run_keyword(self, name, args, kwargs=None):
7575
try:

‎doc/userguide/src/ExtendingRobotFramework/RemoteLibrary.rst

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -206,6 +206,13 @@ The method, and also the exposed keyword, should return `True`
206206
or `False` depending on whether stopping is allowed or not. That makes it
207207
possible for external tools to know if stopping the server succeeded.
208208

209+
Performance improvements at load-time can be achieved by implementing the
210+
`get_library_information` method. When supported by the remote server, Robot
211+
Framework will utilize this method to load all information in a single call.
212+
Without `get_library_information`, all `get_keyword_*` methods will be invoked
213+
separately for each individual keyword, causing significant delays when loading
214+
large libraries.
215+
209216
The `Python remote server`__ can be used as a reference implementation.
210217

211218
__ https://github.com/robotframework/PythonRemoteServer

‎src/robot/libraries/Remote.py

Lines changed: 53 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
from __future__ import absolute_import
1717

1818
from contextlib import contextmanager
19+
from functools import wraps
1920

2021
try:
2122
import httplib
@@ -64,40 +65,63 @@ def __init__(self, uri='http://127.0.0.1:8270', timeout=None):
6465
timeout = timestr_to_secs(timeout)
6566
self._uri = uri
6667
self._client = XmlRpcRemoteClient(uri, timeout)
68+
self._kw_cache = None
69+
70+
def _cached(info_type=None, default=None):
71+
def decorator(f):
72+
@wraps(f)
73+
def wrapper(self, name=None):
74+
if self._kw_cache:
75+
return self._kw_cache.get(info_type, dict()).get(name, default)
76+
else:
77+
try:
78+
return f(self, name)
79+
except TypeError:
80+
return default
81+
return wrapper
82+
return decorator
83+
84+
def _build_kw_info_cache(self):
85+
"""
86+
Single attempt to build the cache using get_library_information interface
87+
cache structure:
88+
{ "keyword_name" : { 'args':[], 'tags':[], 'doc':"", 'types':[]},
89+
"kw2" : {...}, etc. }
90+
"""
91+
try:
92+
self._kw_cache = self._client.get_library_information()
93+
self._kw_cache['__intro__'] = dict(
94+
doc=self._client.get_keyword_documentation('__intro__'))
95+
self._kw_cache['__init__'] = dict(
96+
doc=self._client.get_keyword_documentation('__init__'))
97+
except TypeError:
98+
pass # Best effort failed, fall back to regular loading
6799

68-
def get_keyword_names(self, attempts=2):
69-
for i in range(attempts):
70-
time.sleep(i)
71-
try:
72-
return self._client.get_keyword_names()
73-
except TypeError as err:
74-
error = err
75-
raise RuntimeError('Connecting remote server at %s failed: %s'
76-
% (self._uri, error))
100+
def get_keyword_names(self):
101+
self._build_kw_info_cache()
102+
if self._kw_cache:
103+
return self._kw_cache.keys()
104+
try:
105+
return self._client.get_keyword_names()
106+
except TypeError as error:
107+
raise RuntimeError('Connecting remote server at %s failed: %s'
108+
% (self._uri, error))
77109

110+
@_cached('args', default=['*args'])
78111
def get_keyword_arguments(self, name):
79-
try:
80-
return self._client.get_keyword_arguments(name)
81-
except TypeError:
82-
return ['*args']
112+
return self._client.get_keyword_arguments(name)
83113

114+
@_cached('types')
84115
def get_keyword_types(self, name):
85-
try:
86-
return self._client.get_keyword_types(name)
87-
except TypeError:
88-
return None
116+
return self._client.get_keyword_types(name)
89117

118+
@_cached('tags')
90119
def get_keyword_tags(self, name):
91-
try:
92-
return self._client.get_keyword_tags(name)
93-
except TypeError:
94-
return None
120+
return self._client.get_keyword_tags(name)
95121

122+
@_cached('doc')
96123
def get_keyword_documentation(self, name):
97-
try:
98-
return self._client.get_keyword_documentation(name)
99-
except TypeError:
100-
return None
124+
return self._client.get_keyword_documentation(name)
101125

102126
def run_keyword(self, name, args, kwargs):
103127
coercer = ArgumentCoercer()
@@ -227,6 +251,10 @@ def _server(self):
227251
finally:
228252
server('close')()
229253

254+
def get_library_information(self):
255+
with self._server as server:
256+
return server.get_library_information()
257+
230258
def get_keyword_names(self):
231259
with self._server as server:
232260
return server.get_keyword_names()

0 commit comments

Comments
 (0)