Skip to content

feat(HTTPClient): add support for collecting all HTTP headers #11768

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
+122 −30
Open

feat(HTTPClient): add support for collecting all HTTP headers #11768

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
69549c8
feat(http-client): add support for collecting all HTTP headers, close…
JakubAndrysek Aug 28, 2025
a2df766
refactor(CustomHeaders): replace USE_SERIAL with Serial for consistency
JakubAndrysek Aug 29, 2025
3bd54ba
Merge branch 'master' into http-client/collecting-headers
JakubAndrysek Sep 1, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
83 changes: 83 additions & 0 deletions libraries/HTTPClient/examples/CustomHeaders/CustomHeaders.ino
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
#include <Arduino.h>

#include <WiFi.h>
#include <HTTPClient.h>

// Enable or disable collecting all headers
#define COLLECT_ALL_HEADERS true

void setup() {

Serial.begin(115200);

Serial.println();
Serial.println();
Serial.println();

for (uint8_t t = 4; t > 0; t--) {
Serial.printf("[SETUP] WAIT %d...\n", t);
Serial.flush();
delay(1000);
}

WiFi.begin("SSID", "PASSWORD");

while (WiFi.status() != WL_CONNECTED) {
delay(500);
Serial.print(".");
}
Serial.println();
Serial.println("Connected to WiFi: " + WiFi.SSID());
}

void loop() {

HTTPClient http;

Serial.print("[HTTP] Preparing HTTP request...\n");
// This page will return the headers we want to test + some others
http.begin("https://httpbingo.org/response-headers?x-custom-header=value:42");

#if COLLECT_ALL_HEADERS
// Collect all headers
http.collectAllHeaders();
#else
// Collect specific headers, only that one will be stored
const char *headerKeys[] = {"x-custom-header"};
const size_t headerKeysCount = sizeof(headerKeys) / sizeof(headerKeys[0]);
http.collectHeaders(headerKeys, headerKeysCount);
#endif

Serial.print("[HTTP] Sending HTTP GET request...\n");
// start connection and send HTTP header
int httpCode = http.GET();

// httpCode will be negative on error
if (httpCode > 0) {
// HTTP header has been send and Server response header has been handled
Serial.printf("[HTTP] GET response code: %d\n", httpCode);

Serial.println("[HTTP] Headers collected:");
for (size_t i = 0; i < http.headers(); i++) {
Serial.printf("[HTTP] - '%s': '%s'\n", http.headerName(i).c_str(), http.header(i).c_str());
}

Serial.println("[HTTP] Has header 'x-custom-header'? " + String(http.hasHeader("x-custom-header")) + " (expected true)");
Serial.printf("[HTTP] x-custom-header: '%s' (expected 'value:42')\n", http.header("x-custom-header").c_str());
Serial.printf("[HTTP] non-existing-header: '%s' (expected empty string)\n", http.header("non-existing-header").c_str());

#if COLLECT_ALL_HEADERS
// Server response with multiple headers, one of them is 'server'
Serial.println("[HTTP] Has header 'server'? " + String(http.hasHeader("server")) + " (expected true)");
Serial.printf("[HTTP] server: '%s'\n", http.header("server").c_str());
#endif

} else {
Serial.printf("[HTTP] GET... failed, error: %s\n", http.errorToString(httpCode).c_str());
}

http.end();

Serial.println("[HTTP] end connection\n\n");
delay(5000);
}
6 changes: 6 additions & 0 deletions libraries/HTTPClient/examples/CustomHeaders/ci.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
{
"requires_any": [
"CONFIG_SOC_WIFI_SUPPORTED=y",
"CONFIG_ESP_WIFI_REMOTE_ENABLED=y"
]
}
56 changes: 29 additions & 27 deletions libraries/HTTPClient/src/HTTPClient.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,9 +90,6 @@ HTTPClient::~HTTPClient() {
if (_client) {
_client->stop();
}
if (_currentHeaders) {
delete[] _currentHeaders;
}
if (_tcpDeprecated) {
_tcpDeprecated.reset(nullptr);
}
Expand Down Expand Up @@ -564,9 +561,12 @@ int HTTPClient::sendRequest(const char *type, uint8_t *payload, size_t size) {
bool redirect = false;
uint16_t redirectCount = 0;
do {
// wipe out any existing headers from previous request
for (size_t i = 0; i < _headerKeysCount; i++) {
if (_currentHeaders[i].value.length() > 0) {
// wipe out any existing headers from previous request, but preserve the keys if collecting specific headers
if (_collectAllHeaders) {
_currentHeaders.clear();
} else {
// Only clear values, keep the keys for specific header collection
for (size_t i = 0; i < _currentHeaders.size(); ++i) {
_currentHeaders[i].value.clear();
}
}
Expand Down Expand Up @@ -1015,19 +1015,24 @@ void HTTPClient::addHeader(const String &name, const String &value, bool first,
}
}

void HTTPClient::collectAllHeaders(bool collectAll) {
_collectAllHeaders = collectAll;
}

void HTTPClient::collectHeaders(const char *headerKeys[], const size_t headerKeysCount) {
_headerKeysCount = headerKeysCount;
if (_currentHeaders) {
delete[] _currentHeaders;
if (_collectAllHeaders) {
log_w("collectHeaders is ignored when collectAllHeaders is set");
return;
}
_currentHeaders = new RequestArgument[_headerKeysCount];
for (size_t i = 0; i < _headerKeysCount; i++) {
_currentHeaders.clear();
_currentHeaders.resize(headerKeysCount);
for (size_t i = 0; i < headerKeysCount; i++) {
_currentHeaders[i].key = headerKeys[i];
}
}

String HTTPClient::header(const char *name) {
for (size_t i = 0; i < _headerKeysCount; ++i) {
for (size_t i = 0; i < _currentHeaders.size(); ++i) {
if (_currentHeaders[i].key.equalsIgnoreCase(name)) {
return _currentHeaders[i].value;
}
Expand All @@ -1036,25 +1041,25 @@ String HTTPClient::header(const char *name) {
}

String HTTPClient::header(size_t i) {
if (i < _headerKeysCount) {
if (i < _currentHeaders.size()) {
return _currentHeaders[i].value;
}
return String();
}

String HTTPClient::headerName(size_t i) {
if (i < _headerKeysCount) {
if (i < _currentHeaders.size()) {
return _currentHeaders[i].key;
}
return String();
}

int HTTPClient::headers() {
return _headerKeysCount;
return _currentHeaders.size();
}

bool HTTPClient::hasHeader(const char *name) {
for (size_t i = 0; i < _headerKeysCount; ++i) {
for (size_t i = 0; i < _currentHeaders.size(); ++i) {
if ((_currentHeaders[i].key.equalsIgnoreCase(name)) && (_currentHeaders[i].value.length() > 0)) {
return true;
}
Expand Down Expand Up @@ -1238,17 +1243,14 @@ int HTTPClient::handleHeaderResponse() {
setCookie(date, headerValue);
}

for (size_t i = 0; i < _headerKeysCount; i++) {
if (_currentHeaders[i].key.equalsIgnoreCase(headerName)) {
// Uncomment the following lines if you need to add support for multiple headers with the same key:
// if (!_currentHeaders[i].value.isEmpty()) {
// // Existing value, append this one with a comma
// _currentHeaders[i].value += ',';
// _currentHeaders[i].value += headerValue;
// } else {
_currentHeaders[i].value = headerValue;
// }
break; // We found a match, stop looking
if (_collectAllHeaders && headerName.length() > 0) {
_currentHeaders.emplace_back(headerName, headerValue);
} else {
for (size_t i = 0; i < _currentHeaders.size(); ++i) {
if (_currentHeaders[i].key.equalsIgnoreCase(headerName)) {
_currentHeaders[i].value = headerValue;
break; // We found a match, stop looking
}
}
}
}
Expand Down
7 changes: 4 additions & 3 deletions libraries/HTTPClient/src/HTTPClient.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@
#include <NetworkClientSecure.h>
#endif // HTTPCLIENT_NOSECURE

/// Cookie jar support
/// Cookie jar and header support
#include <vector>

#define HTTPCLIENT_DEFAULT_TCP_TIMEOUT (5000)
Expand Down Expand Up @@ -238,6 +238,7 @@ class HTTPClient {
void addHeader(const String &name, const String &value, bool first = false, bool replace = true);

/// Response handling
void collectAllHeaders(bool collectAll = true);
void collectHeaders(const char *headerKeys[], const size_t headerKeysCount);
String header(const char *name); // get request header value by name
String header(size_t i); // get request header value by number
Expand Down Expand Up @@ -294,6 +295,7 @@ class HTTPClient {
uint16_t _tcpTimeout = HTTPCLIENT_DEFAULT_TCP_TIMEOUT;
bool _useHTTP10 = false;
bool _secure = false;
bool _collectAllHeaders = false;

String _uri;
String _protocol;
Expand All @@ -304,8 +306,7 @@ class HTTPClient {
String _acceptEncoding = "identity;q=1,chunked;q=0.1,*;q=0";

/// Response handling
RequestArgument *_currentHeaders = nullptr;
size_t _headerKeysCount = 0;
std::vector<RequestArgument> _currentHeaders;
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why, vector probably needs more bytes.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You are right, the vector probably needs more bytes, but since I do not know the exact size and I am reading until the _client has some data, it is a lot easier to just use a vector and let it grow as needed
The previous implementation used only a fixed-size array, but after the update, I need to account for dynamic resizing to allow storing multiple headers. It is possible to do it with a resize operation of a static vector, but I think it is more convenient to use a dynamic vector for this purpose.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fully understand, over time every part of the core does need more resources. One day we will have a core which can not be used for anything more than just "blink".


int _returnCode = 0;
int _size = -1;
Expand Down
Loading