Skip to content

Commit 84c8927

Browse files
HypeMCHypeMC
committed
[FrameworkBundle][HttpClient] Add ThrottlingHttpClient to limit requests within a timeframe
1 parent 6c44684 commit 84c8927

File tree

13 files changed

+316
-6
lines changed

13 files changed

+316
-6
lines changed

‎src/Symfony/Bundle/FrameworkBundle/CHANGELOG.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ CHANGELOG
99
* Move the Router `cache_dir` to `kernel.build_dir`
1010
* Deprecate the `router.cache_dir` config option
1111
* Add `rate_limiter` tags to rate limiter services
12+
* Add `rate_limiter` option to `http_client.default_options` and `http_client.scoped_clients`
1213

1314
7.0
1415
---

‎src/Symfony/Bundle/FrameworkBundle/DependencyInjection/Configuration.php

Lines changed: 29 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1716,17 +1716,32 @@ private function addHttpClientSection(ArrayNodeDefinition $rootNode, callable $e
17161716
->fixXmlConfig('scoped_client')
17171717
->beforeNormalization()
17181718
->always(function ($config) {
1719-
if (empty($config['scoped_clients']) || !\is_array($config['default_options']['retry_failed'] ?? null)) {
1719+
if (empty($config['scoped_clients'])) {
1720+
return $config;
1721+
}
1722+
1723+
$hasDefaultRateLimiter = isset($config['default_options']['rate_limiter']);
1724+
$hasDefaultRetryFailed = \is_array($config['default_options']['retry_failed'] ?? null);
1725+
1726+
if (!$hasDefaultRateLimiter && !$hasDefaultRetryFailed) {
17201727
return $config;
17211728
}
17221729

17231730
foreach ($config['scoped_clients'] as &$scopedConfig) {
1724-
if (!isset($scopedConfig['retry_failed']) || true === $scopedConfig['retry_failed']) {
1725-
$scopedConfig['retry_failed'] = $config['default_options']['retry_failed'];
1726-
continue;
1731+
if ($hasDefaultRateLimiter) {
1732+
if (!isset($scopedConfig['rate_limiter']) || true === $scopedConfig['rate_limiter']) {
1733+
$scopedConfig['rate_limiter'] = $config['default_options']['rate_limiter'];
1734+
} elseif (false === $scopedConfig['rate_limiter']) {
1735+
$scopedConfig['rate_limiter'] = null;
1736+
}
17271737
}
1728-
if (\is_array($scopedConfig['retry_failed'])) {
1729-
$scopedConfig['retry_failed'] += $config['default_options']['retry_failed'];
1738+
1739+
if ($hasDefaultRetryFailed) {
1740+
if (!isset($scopedConfig['retry_failed']) || true === $scopedConfig['retry_failed']) {
1741+
$scopedConfig['retry_failed'] = $config['default_options']['retry_failed'];
1742+
} elseif (\is_array($scopedConfig['retry_failed'])) {
1743+
$scopedConfig['retry_failed'] += $config['default_options']['retry_failed'];
1744+
}
17301745
}
17311746
}
17321747

@@ -1831,6 +1846,10 @@ private function addHttpClientSection(ArrayNodeDefinition $rootNode, callable $e
18311846
->normalizeKeys(false)
18321847
->variablePrototype()->end()
18331848
->end()
1849+
->scalarNode('rate_limiter')
1850+
->defaultNull()
1851+
->info('Rate limiter name to use for throttling requests')
1852+
->end()
18341853
->append($this->createHttpClientRetrySection())
18351854
->end()
18361855
->end()
@@ -1979,6 +1998,10 @@ private function addHttpClientSection(ArrayNodeDefinition $rootNode, callable $e
19791998
->normalizeKeys(false)
19801999
->variablePrototype()->end()
19812000
->end()
2001+
->scalarNode('rate_limiter')
2002+
->defaultNull()
2003+
->info('Rate limiter name to use for throttling requests')
2004+
->end()
19822005
->append($this->createHttpClientRetrySection())
19832006
->end()
19842007
->end()

‎src/Symfony/Bundle/FrameworkBundle/DependencyInjection/FrameworkExtension.php

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,7 @@
8585
use Symfony\Component\HttpClient\Retry\GenericRetryStrategy;
8686
use Symfony\Component\HttpClient\RetryableHttpClient;
8787
use Symfony\Component\HttpClient\ScopingHttpClient;
88+
use Symfony\Component\HttpClient\ThrottlingHttpClient;
8889
use Symfony\Component\HttpClient\UriTemplateHttpClient;
8990
use Symfony\Component\HttpFoundation\Request;
9091
use Symfony\Component\HttpKernel\Attribute\AsController;
@@ -345,6 +346,7 @@ public function load(array $configs, ContainerBuilder $container): void
345346
}
346347

347348
if ($this->readConfigEnabled('http_client', $container, $config['http_client'])) {
349+
$this->readConfigEnabled('rate_limiter', $container, $config['rate_limiter']); // makes sure that isInitializedConfigEnabled() will work
348350
$this->registerHttpClientConfiguration($config['http_client'], $container, $loader);
349351
}
350352

@@ -2408,6 +2410,8 @@ private function registerHttpClientConfiguration(array $config, ContainerBuilder
24082410
$loader->load('http_client.php');
24092411

24102412
$options = $config['default_options'] ?? [];
2413+
$rateLimiter = $options['rate_limiter'] ?? null;
2414+
unset($options['rate_limiter']);
24112415
$retryOptions = $options['retry_failed'] ?? ['enabled' => false];
24122416
unset($options['retry_failed']);
24132417
$defaultUriTemplateVars = $options['vars'] ?? [];
@@ -2429,6 +2433,10 @@ private function registerHttpClientConfiguration(array $config, ContainerBuilder
24292433
$container->removeAlias(HttpClient::class);
24302434
}
24312435

2436+
if (null !== $rateLimiter) {
2437+
$this->registerThrottlingHttpClient($rateLimiter, 'http_client', $container);
2438+
}
2439+
24322440
if ($this->readConfigEnabled('http_client.retry_failed', $container, $retryOptions)) {
24332441
$this->registerRetryableHttpClient($retryOptions, 'http_client', $container);
24342442
}
@@ -2450,6 +2458,8 @@ private function registerHttpClientConfiguration(array $config, ContainerBuilder
24502458

24512459
$scope = $scopeConfig['scope'] ?? null;
24522460
unset($scopeConfig['scope']);
2461+
$rateLimiter = $scopeConfig['rate_limiter'] ?? null;
2462+
unset($scopeConfig['rate_limiter']);
24532463
$retryOptions = $scopeConfig['retry_failed'] ?? ['enabled' => false];
24542464
unset($scopeConfig['retry_failed']);
24552465

@@ -2469,6 +2479,10 @@ private function registerHttpClientConfiguration(array $config, ContainerBuilder
24692479
;
24702480
}
24712481

2482+
if (null !== $rateLimiter) {
2483+
$this->registerThrottlingHttpClient($rateLimiter, $name, $container);
2484+
}
2485+
24722486
if ($this->readConfigEnabled('http_client.scoped_clients.'.$name.'.retry_failed', $container, $retryOptions)) {
24732487
$this->registerRetryableHttpClient($retryOptions, $name, $container);
24742488
}
@@ -2506,6 +2520,22 @@ private function registerHttpClientConfiguration(array $config, ContainerBuilder
25062520
}
25072521
}
25082522

2523+
private function registerThrottlingHttpClient(string $rateLimiter, string $name, ContainerBuilder $container): void
2524+
{
2525+
if (!class_exists(ThrottlingHttpClient::class)) {
2526+
throw new LogicException('Rate limiter support cannot be enabled as version 7.1+ of the HttpClient component is required.');
2527+
}
2528+
2529+
if (!$this->isInitializedConfigEnabled('rate_limiter')) {
2530+
throw new LogicException('Rate limiter cannot be used within HttpClient as the RateLimiter component is not enabled.');
2531+
}
2532+
2533+
$container
2534+
->register($name.'.throttling', ThrottlingHttpClient::class)
2535+
->setDecoratedService($name, null, 15) // higher priority than RetryableHttpClient (10)
2536+
->setArguments([new Reference($name.'.throttling.inner'), new Reference('limiter.'.$rateLimiter)]);
2537+
}
2538+
25092539
private function registerRetryableHttpClient(array $options, string $name, ContainerBuilder $container): void
25102540
{
25112541
if (null !== $options['retry_strategy']) {

‎src/Symfony/Bundle/FrameworkBundle/Resources/config/schema/symfony-1.0.xsd

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -661,6 +661,7 @@
661661
<xsd:attribute name="local-pk" type="xsd:string" />
662662
<xsd:attribute name="passphrase" type="xsd:string" />
663663
<xsd:attribute name="ciphers" type="xsd:string" />
664+
<xsd:attribute name="rate-limiter" type="xsd:string" />
664665
</xsd:complexType>
665666

666667
<xsd:complexType name="http_client_scope_options" mixed="true">
@@ -691,6 +692,7 @@
691692
<xsd:attribute name="local-pk" type="xsd:string" />
692693
<xsd:attribute name="passphrase" type="xsd:string" />
693694
<xsd:attribute name="ciphers" type="xsd:string" />
695+
<xsd:attribute name="rate-limiter" type="xsd:string" />
694696
</xsd:complexType>
695697

696698
<xsd:complexType name="fingerprint">

‎src/Symfony/Bundle/FrameworkBundle/Tests/DependencyInjection/ConfigurationTest.php

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -530,6 +530,46 @@ public function testEnabledLockNeedsResources()
530530
]);
531531
}
532532

533+
public function testScopedHttpClientsInheritRateLimiterAndRetryFailedConfiguration()
534+
{
535+
$processor = new Processor();
536+
$configuration = new Configuration(true);
537+
538+
$config = $processor->processConfiguration($configuration, [[
539+
'http_client' => [
540+
'default_options' => ['rate_limiter' => 'default_limiter', 'retry_failed' => ['max_retries' => 77]],
541+
'scoped_clients' => [
542+
'foo' => ['base_uri' => 'http://example.com'],
543+
'bar' => ['base_uri' => 'http://example.com', 'rate_limiter' => true, 'retry_failed' => true],
544+
'baz' => ['base_uri' => 'http://example.com', 'rate_limiter' => false, 'retry_failed' => false],
545+
'qux' => ['base_uri' => 'http://example.com', 'rate_limiter' => 'foo_limiter', 'retry_failed' => ['max_retries' => 88, 'delay' => 999]],
546+
],
547+
],
548+
]]);
549+
550+
$scopedClients = $config['http_client']['scoped_clients'];
551+
552+
$this->assertSame('default_limiter', $scopedClients['foo']['rate_limiter']);
553+
$this->assertTrue($scopedClients['foo']['retry_failed']['enabled']);
554+
$this->assertSame(77, $scopedClients['foo']['retry_failed']['max_retries']);
555+
$this->assertSame(1000, $scopedClients['foo']['retry_failed']['delay']);
556+
557+
$this->assertSame('default_limiter', $scopedClients['bar']['rate_limiter']);
558+
$this->assertTrue($scopedClients['bar']['retry_failed']['enabled']);
559+
$this->assertSame(77, $scopedClients['bar']['retry_failed']['max_retries']);
560+
$this->assertSame(1000, $scopedClients['bar']['retry_failed']['delay']);
561+
562+
$this->assertNull($scopedClients['baz']['rate_limiter']);
563+
$this->assertFalse($scopedClients['baz']['retry_failed']['enabled']);
564+
$this->assertSame(3, $scopedClients['baz']['retry_failed']['max_retries']);
565+
$this->assertSame(1000, $scopedClients['baz']['retry_failed']['delay']);
566+
567+
$this->assertSame('foo_limiter', $scopedClients['qux']['rate_limiter']);
568+
$this->assertTrue($scopedClients['qux']['retry_failed']['enabled']);
569+
$this->assertSame(88, $scopedClients['qux']['retry_failed']['max_retries']);
570+
$this->assertSame(999, $scopedClients['qux']['retry_failed']['delay']);
571+
}
572+
533573
protected static function getBundleDefaultConfig()
534574
{
535575
return [

‎src/Symfony/Bundle/FrameworkBundle/Tests/DependencyInjection/Fixtures/php/http_client_rate_limiter.php

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
<?php
2+
3+
$container->loadFromExtension('framework', [
4+
'annotations' => false,
5+
'http_method_override' => false,
6+
'handle_all_throwables' => true,
7+
'php_errors' => ['log' => true],
8+
'rate_limiter' => [
9+
'foo_limiter' => [
10+
'lock_factory' => null,
11+
'policy' => 'token_bucket',
12+
'limit' => 10,
13+
'rate' => ['interval' => '5 seconds', 'amount' => 10],
14+
],
15+
],
16+
'http_client' => [
17+
'default_options' => [
18+
'rate_limiter' => 'default_limiter',
19+
],
20+
'scoped_clients' => [
21+
'foo' => [
22+
'base_uri' => 'http://example.com',
23+
'rate_limiter' => 'foo_limiter',
24+
],
25+
],
26+
],
27+
]);

‎src/Symfony/Bundle/FrameworkBundle/Tests/DependencyInjection/Fixtures/xml/http_client_rate_limiter.xml

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
<?xml version="1.0" encoding="utf-8" ?>
2+
<container xmlns="http://symfony.com/schema/dic/services"
3+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
4+
xmlns:framework="http://symfony.com/schema/dic/symfony"
5+
xsi:schemaLocation="http://symfony.com/schema/dic/services http://symfony.com/schema/dic/services/services-1.0.xsd
6+
http://symfony.com/schema/dic/symfony http://symfony.com/schema/dic/symfony/symfony-1.0.xsd">
7+
8+
<framework:config http-method-override="false" handle-all-throwables="true">
9+
<framework:annotations enabled="false" />
10+
<framework:php-errors log="true" />
11+
<framework:rate-limiter>
12+
<framework:limiter name="foo_limiter" lock-factory="null" policy="token_bucket" limit="10">
13+
<framework:rate interval="5 seconds" amount="10" />
14+
</framework:limiter>
15+
</framework:rate-limiter>
16+
<framework:http-client>
17+
<framework:default-options rate-limiter="default_limiter" />
18+
<framework:scoped-client name="foo" base-uri="http://example.com" rate-limiter="foo_limiter" />
19+
</framework:http-client>
20+
</framework:config>
21+
</container>

‎src/Symfony/Bundle/FrameworkBundle/Tests/DependencyInjection/Fixtures/yml/http_client_rate_limiter.yml

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
framework:
2+
annotations: false
3+
http_method_override: false
4+
handle_all_throwables: true
5+
php_errors:
6+
log: true
7+
rate_limiter:
8+
foo_limiter:
9+
lock_factory: null
10+
policy: token_bucket
11+
limit: 10
12+
rate: { interval: '5 seconds', amount: 10 }
13+
http_client:
14+
default_options:
15+
rate_limiter: default_limiter
16+
scoped_clients:
17+
foo:
18+
base_uri: http://example.com
19+
rate_limiter: foo_limiter

‎src/Symfony/Bundle/FrameworkBundle/Tests/DependencyInjection/FrameworkExtensionTestCase.php

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@
3838
use Symfony\Component\DependencyInjection\ContainerBuilder;
3939
use Symfony\Component\DependencyInjection\ContainerInterface;
4040
use Symfony\Component\DependencyInjection\Definition;
41+
use Symfony\Component\DependencyInjection\Exception\LogicException;
4142
use Symfony\Component\DependencyInjection\Loader\ClosureLoader;
4243
use Symfony\Component\DependencyInjection\Loader\Configurator\ContainerConfigurator;
4344
use Symfony\Component\DependencyInjection\ParameterBag\EnvPlaceholderParameterBag;
@@ -51,6 +52,7 @@
5152
use Symfony\Component\HttpClient\MockHttpClient;
5253
use Symfony\Component\HttpClient\RetryableHttpClient;
5354
use Symfony\Component\HttpClient\ScopingHttpClient;
55+
use Symfony\Component\HttpClient\ThrottlingHttpClient;
5456
use Symfony\Component\HttpFoundation\IpUtils;
5557
use Symfony\Component\HttpKernel\DependencyInjection\LoggerPass;
5658
use Symfony\Component\HttpKernel\Fragment\FragmentUriGeneratorInterface;
@@ -1986,6 +1988,35 @@ public function testHttpClientFullDefaultOptions()
19861988
$this->assertSame(['foo' => ['bar' => 'baz']], $defaultOptions['extra']);
19871989
}
19881990

1991+
public function testHttpClientRateLimiter()
1992+
{
1993+
if (!class_exists(ThrottlingHttpClient::class)) {
1994+
$this->expectException(LogicException::class);
1995+
}
1996+
1997+
$container = $this->createContainerFromFile('http_client_rate_limiter');
1998+
1999+
$this->assertTrue($container->hasDefinition('http_client.throttling'));
2000+
$definition = $container->getDefinition('http_client.throttling');
2001+
$this->assertSame(ThrottlingHttpClient::class, $definition->getClass());
2002+
$this->assertSame('http_client', $definition->getDecoratedService()[0]);
2003+
$this->assertCount(2, $arguments = $definition->getArguments());
2004+
$this->assertInstanceOf(Reference::class, $arguments[0]);
2005+
$this->assertSame('http_client.throttling.inner', (string) $arguments[0]);
2006+
$this->assertInstanceOf(Reference::class, $arguments[1]);
2007+
$this->assertSame('limiter.default_limiter', (string) $arguments[1]);
2008+
2009+
$this->assertTrue($container->hasDefinition('foo.throttling'));
2010+
$definition = $container->getDefinition('foo.throttling');
2011+
$this->assertSame(ThrottlingHttpClient::class, $definition->getClass());
2012+
$this->assertSame('foo', $definition->getDecoratedService()[0]);
2013+
$this->assertCount(2, $arguments = $definition->getArguments());
2014+
$this->assertInstanceOf(Reference::class, $arguments[0]);
2015+
$this->assertSame('foo.throttling.inner', (string) $arguments[0]);
2016+
$this->assertInstanceOf(Reference::class, $arguments[1]);
2017+
$this->assertSame('limiter.foo_limiter', (string) $arguments[1]);
2018+
}
2019+
19892020
public static function provideMailer(): array
19902021
{
19912022
return [

‎src/Symfony/Component/HttpClient/CHANGELOG.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ CHANGELOG
77
* Add `HttpOptions::setHeader()` to add or replace a single header
88
* Allow mocking `start_time` info in `MockResponse`
99
* Add `MockResponse::fromFile()` and `JsonMockResponse::fromFile()` methods to help using fixtures files
10+
* Add `ThrottlingHttpClient` to enable limiting the number request within a certain period
1011

1112
7.0
1213
---

0 commit comments

Comments
 (0)