I am making a GET request via HTTPS using HTTP 1.1 thru an Ethernet connection (no WiFi involved here).
My code is below.
For very small files (25B), the file is successfully downloaded. But is not downloaded when the file size increases (20K or 2MB), the connection is closed quite quickly.
So, my question is: how can I make a download of a "large" file?
My code runs on an ESP32.
#include <EthernetUdp.h>
#include <EthernetLarge.h>
#include <SSLClient.h>
// Choose the analog pin to get semi-random data from for SSL
// Pick a pin that's not connected or attached to a randomish voltage source
const int rand_pin = A5;
// Initialize the SSL client library
// We input an EthernetClient, our trust anchors, and the analog pin
EthernetClient base_client;
SSLClient client(base_client, TAs, (size_t)TAs_NUM, rand_pin);
void NanoEthernet::httpsGet(
char const *hostname, char const *path,
byte *buffer, size_t bufferCapacity,
size_t &bodyLength, bool &bufferOverflows,
void (*downloadChunkCallback)(byte * buffer, size_t bufferReadCount, size_t contentLength)
) {
log("Time is: %d %d %d %02d:%02d:%02d", day(), month(), year(), hour(), minute(), second());
char const * subModule = "httpsGet: ";
log("%sbegin %s%s…",subModule, hostname, path);
// === if you get a connection, report back via serial:
if (!client.connect(hostname, 443)) {
// if you didn't get a connection to the server:
error("%sconnection failed", subModule);
return false;
}
success("%sConnected!",subModule);
// Make a HTTP request:
client.printf("GET %s HTTP/1.1", path); client.println();
client.printf("Host: %s", hostname); client.println();
// client.println("Connection: keep-alive"); client.println();
// client.println("Keep-Alive: timeout=5, max=100"); client.println();
//client.println("Connection: close");
client.println();
//while(client.connected() && !client.available()) delay(1); //waits for data
// success("%sSent query headers (connected:%d)",
// subModule,
// client.connected()
// );
/// Body counter
int bodyCounter = 0;
/// Chunk counter
int chunkReadCount = 0;
/// Whether to read response header?
bool readResponseHeader = true;
byte *p = buffer;
int contentLength = 0;
const unsigned timeoutNoData = 10*1000; // 10 seconds
unsigned long long lastTimeNoData = millis();
unsigned waitForConnection = 10;
while (true) {
// log("%sResponse: connected: %d | readResponseHeader: %d",
// subModule,
// client.connected(),
// readResponseHeader
// );
if (!client.connected()) {
delay(1);
error("%sNo more connected, aborting", subModule);
if (waitForConnection-- <= 0)
continue;
else
break;
}
if (!client.available()) {
delay(1);
// == Still reading the header then continue, otherwise hanle timeout
if (!readResponseHeader) {
if (millis() - lastTimeNoData >= timeoutNoData) {
if (contentLength==0) {
log("%sNo more data are available after %d seconds timeout",
subModule,
timeoutNoData/1000
);
}
else {
error("%sNo more data are available timeout", subModule);
}
// == Stopping the connection
break;
}
}
//log(">.");
continue;
}
lastTimeNoData = millis();
// = Skip response header
if (readResponseHeader) {
const String header = client.readStringUntil('\n'); // End of line is "\r\n"
log("%sResponse headers: %s", subModule, header.c_str());
const int nextData = client.peek();
// End of stream?
if (nextData==-1) {
error("%sEnd of stream (received data -1) while reading headers");
break;
}
const char nextChar = char(nextData);
//log("Response headers: => next char is %d 0x%02x (%c)", nextChar, nextChar, nextChar);
// == End of headers? (empty line)
if (nextChar == '\r') {
// Actually read the end of line
client.read(); // eat \r
client.read(); // eat \n
//log("End of response header, starting to read response body…");
readResponseHeader=false;
}
else {
char const *headerString = "Content-Length: ";
const int headerStringLength = 16;
if (header.startsWith(headerString)) {
const String sContentLength = header.substring(headerStringLength);
contentLength = header.substring(headerStringLength).toInt();
//log("Read Content-Length: string: '%s'| int: %d bytes", sContentLength.c_str(), byteCountToRead);
}
}
continue;
}
const int data = client.read(); //gets byte from ethernet buffer
// == End of stream?
if (data==-1) {
log("%sEnd of stream (data received: -1)",subModule);
break;
}
// == Keep the actual byte
const byte b = byte(data);
#if LOG_LEVEL >= LOG_LEVEL_DEBUG
//log(">%02x (%d)", b, bodyCounter);
log(">%02x ", b);
#endif
*p++ = b;
bodyCounter++;
chunkReadCount++;
// = When no chunk callback, overflow means stop reading
if (downloadChunkCallback==NULL) {
// == If buffer overflows, stop
if (bodyCounter >= bufferCapacity) {
bufferOverflows = true;
break;
}
}
// = But when chunk callback has been provided, pass it the
// chunk, rewind the buffer and continue
else {
// == If buffer overflows, stop
if (chunkReadCount >= bufferCapacity) {
downloadChunkCallback(buffer, chunkReadCount, contentLength);
// Now rewind the chunk to the satrt of the buffer prior to
// continue reading the reponsse body
chunkReadCount = 0;
p = buffer;
}
}
// == We read everything?
if (contentLength>0) {
if (bodyCounter>=contentLength) {
log("%sEnd of stream (received ontentLength bytes)", subModule);
break;
}
}
}
// = When chunk callback, pass it the potential remaining chunk
if(downloadChunkCallback!=NULL
&& chunkReadCount>0) {
downloadChunkCallback(buffer, chunkReadCount, contentLength);
}
// == Write the length we read
bodyLength = bodyCounter;
#if LOG_LEVEL >= LOG_LEVEL_DEBUG
log("");
log("%s: did the get and read %d bytes", subModule, bodyCounter);
#endif
client.stop();
return true;
}
Content-Lengthheader is not the only way to determine the end of a response. For instance, you are not handlingTransfer-Encoding: chunkedat all. What do the response headers actually look like when a download "succeeds" vs "fails"? Is there a reason why you are usingSSLClientto implement HTTPS manually, instead of using an actual HTTPS client library?available(), thenconnected(), because there can be still buffered data. github.com/OPEnSLab-OSU/SSLClient/issues/43