From dd0ddaba819866eaa4a46feecc728a95df624ee8 Mon Sep 17 00:00:00 2001 From: Thomas Watson Date: Tue, 24 Jan 2023 12:55:08 -0600 Subject: [PATCH 01/23] change EnviroDIY URL to Memphis server and update copyright For future testing and development --- src/publishers/EnviroDIYPublisher.cpp | 1 + src/publishers/EnviroDIYPublisher.h | 1 + 2 files changed, 2 insertions(+) diff --git a/src/publishers/EnviroDIYPublisher.cpp b/src/publishers/EnviroDIYPublisher.cpp index 5a6a043d8..cb2cd394b 100644 --- a/src/publishers/EnviroDIYPublisher.cpp +++ b/src/publishers/EnviroDIYPublisher.cpp @@ -4,6 +4,7 @@ * Part of the EnviroDIY ModularSensors library for Arduino. * This library is published under the BSD-3 license. * @author Sara Geleskie Damiano + * @author Thomas Watson * * @brief Implements the EnviroDIYPublisher class. */ diff --git a/src/publishers/EnviroDIYPublisher.h b/src/publishers/EnviroDIYPublisher.h index 025ece3df..a05b95418 100644 --- a/src/publishers/EnviroDIYPublisher.h +++ b/src/publishers/EnviroDIYPublisher.h @@ -4,6 +4,7 @@ * Part of the EnviroDIY ModularSensors library for Arduino. * This library is published under the BSD-3 license. * @author Sara Geleskie Damiano + * @author Thomas Watson * * @brief Contains the EnviroDIYPublisher subclass of dataPublisher for * publishing data to the Monitor My Watershed/EnviroDIY data portal at From cd37b8ff25e3dee8b663c74a3f57b177414a3fef Mon Sep 17 00:00:00 2001 From: Thomas Watson Date: Thu, 9 Feb 2023 21:08:54 -0600 Subject: [PATCH 02/23] add 8K data buffer for testing Device still reports 3K RAM free --- src/publishers/EnviroDIYPublisher.cpp | 1 + src/publishers/EnviroDIYPublisher.h | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/src/publishers/EnviroDIYPublisher.cpp b/src/publishers/EnviroDIYPublisher.cpp index cb2cd394b..c2322e3d7 100644 --- a/src/publishers/EnviroDIYPublisher.cpp +++ b/src/publishers/EnviroDIYPublisher.cpp @@ -11,6 +11,7 @@ #include "EnviroDIYPublisher.h" +char EnviroDIYPublisher::dataBuffer[MS_DATA_BUFFER_SIZE]; // ============================================================================ // Functions for the EnviroDIY data portal receivers. diff --git a/src/publishers/EnviroDIYPublisher.h b/src/publishers/EnviroDIYPublisher.h index a05b95418..b9793f31e 100644 --- a/src/publishers/EnviroDIYPublisher.h +++ b/src/publishers/EnviroDIYPublisher.h @@ -18,6 +18,10 @@ // Debugging Statement // #define MS_ENVIRODIYPUBLISHER_DEBUG +#ifndef MS_DATA_BUFFER_SIZE +#define MS_DATA_BUFFER_SIZE 8192 +#endif + #ifdef MS_ENVIRODIYPUBLISHER_DEBUG #define MS_DEBUGGING_STD "EnviroDIYPublisher" #endif @@ -186,6 +190,9 @@ class EnviroDIYPublisher : public dataPublisher { static const char* timestampTag; ///< The JSON feature timestamp tag /**@}*/ + // queued sensor data buffer + static char dataBuffer[MS_DATA_BUFFER_SIZE]; + private: // Tokens and UUID's for EnviroDIY const char* _registrationToken = nullptr; From 4320fd28c1b5f7a7afe244900160c3a1c6ba9a29 Mon Sep 17 00:00:00 2001 From: Thomas Watson Date: Sat, 4 Feb 2023 14:40:37 -0600 Subject: [PATCH 03/23] dataPublisherBase: remove redundant zero init of TX buffer Saves 750 bytes of flash as the buffer can be placed in .bss to be zero-initialized instead of .data. --- src/dataPublisherBase.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/dataPublisherBase.cpp b/src/dataPublisherBase.cpp index 5429a696f..7f495b916 100644 --- a/src/dataPublisherBase.cpp +++ b/src/dataPublisherBase.cpp @@ -11,9 +11,7 @@ */ #include "dataPublisherBase.h" -char dataPublisher::txBuffer[MS_SEND_BUFFER_SIZE]; -Client* dataPublisher::txBufferOutClient = nullptr; -size_t dataPublisher::txBufferLen; +char dataPublisher::txBuffer[MS_SEND_BUFFER_SIZE]; // Basic chunks of HTTP const char* dataPublisher::getHeader = "GET "; From e275718510aa386aca630e8be2247916beeea79d Mon Sep 17 00:00:00 2001 From: Thomas Watson Date: Thu, 9 Feb 2023 21:23:29 -0600 Subject: [PATCH 04/23] publishers: refactor transmit buffer usage Use cleaner interface and common functions that avoid repeated snprintf and strlen usage to save ~2.5KB of flash and dozens of lines of code. Removes extra \r\n from HTTP requests as a side effect, which were against spec and caused spurious 400 Bad Request status messages from servers. --- src/dataPublisherBase.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/dataPublisherBase.cpp b/src/dataPublisherBase.cpp index 7f495b916..5429a696f 100644 --- a/src/dataPublisherBase.cpp +++ b/src/dataPublisherBase.cpp @@ -11,7 +11,9 @@ */ #include "dataPublisherBase.h" -char dataPublisher::txBuffer[MS_SEND_BUFFER_SIZE]; +char dataPublisher::txBuffer[MS_SEND_BUFFER_SIZE]; +Client* dataPublisher::txBufferOutClient = nullptr; +size_t dataPublisher::txBufferLen; // Basic chunks of HTTP const char* dataPublisher::getHeader = "GET "; From 40b91c5aa36fda8c63bb3247dfb4a9dc46e339ad Mon Sep 17 00:00:00 2001 From: Thomas Watson Date: Thu, 9 Feb 2023 21:30:59 -0600 Subject: [PATCH 05/23] publishers/EnviroDIYPublisher: send data as one element arrays For testing the server component --- src/publishers/EnviroDIYPublisher.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/publishers/EnviroDIYPublisher.cpp b/src/publishers/EnviroDIYPublisher.cpp index c2322e3d7..edcd3da7c 100644 --- a/src/publishers/EnviroDIYPublisher.cpp +++ b/src/publishers/EnviroDIYPublisher.cpp @@ -29,7 +29,7 @@ const char* EnviroDIYPublisher::contentTypeHeader = "\r\nContent-Type: application/json\r\n\r\n"; const char* EnviroDIYPublisher::samplingFeatureTag = "{\"sampling_feature\":\""; -const char* EnviroDIYPublisher::timestampTag = "\",\"timestamp\":\""; +const char* EnviroDIYPublisher::timestampTag = "\",\"timestamp\":[\""; // Constructors @@ -68,13 +68,13 @@ void EnviroDIYPublisher::setToken(const char* registrationToken) { uint16_t EnviroDIYPublisher::calculateJsonSize() { uint16_t jsonLength = 21; // {"sampling_feature":" jsonLength += 36; // sampling feature UUID - jsonLength += 15; // ","timestamp":" + jsonLength += 17; // ","timestamp":["] jsonLength += 25; // markedISO8601Time jsonLength += 2; // ", for (uint8_t i = 0; i < _baseLogger->getArrayVarCount(); i++) { jsonLength += 1; // " jsonLength += 36; // variable UUID - jsonLength += 2; // ": + jsonLength += 4; // ":[] jsonLength += _baseLogger->getValueStringAtI(i).length(); if (i + 1 != _baseLogger->getArrayVarCount()) { jsonLength += 1; // , @@ -147,6 +147,7 @@ int16_t EnviroDIYPublisher::publishData(Client* outClient) { Logger::formatDateTime_ISO8601(Logger::markedLocalEpochTime) .c_str()); txBufferAppend('"'); + txBufferAppend(']'); txBufferAppend(','); for (uint8_t i = 0; i < _baseLogger->getArrayVarCount(); i++) { @@ -154,7 +155,9 @@ int16_t EnviroDIYPublisher::publishData(Client* outClient) { txBufferAppend(_baseLogger->getVarUUIDAtI(i).c_str()); txBufferAppend('"'); txBufferAppend(':'); + txBufferAppend('['); txBufferAppend(_baseLogger->getValueStringAtI(i).c_str()); + txBufferAppend(']'); if (i + 1 != _baseLogger->getArrayVarCount()) { txBufferAppend(','); } else { From 5df9aa6e16af3e32f56000c595e961391a3c4a4c Mon Sep 17 00:00:00 2001 From: Thomas Watson Date: Thu, 9 Feb 2023 21:37:15 -0600 Subject: [PATCH 06/23] publishers/EnviroDIYPublisher: buffer log data and send in batches Serialize timestamps and variable values to the data buffer, then unserialize and format into JSON arrays. Offers considerable data and power savings. --- src/LoggerBase.cpp | 9 +++ src/LoggerBase.h | 18 +++++ src/VariableBase.cpp | 11 ++- src/VariableBase.h | 8 +++ src/publishers/EnviroDIYPublisher.cpp | 97 +++++++++++++++++++++++---- src/publishers/EnviroDIYPublisher.h | 5 ++ 6 files changed, 134 insertions(+), 14 deletions(-) diff --git a/src/LoggerBase.cpp b/src/LoggerBase.cpp index 7b36fd4f1..36b6f33e2 100644 --- a/src/LoggerBase.cpp +++ b/src/LoggerBase.cpp @@ -265,11 +265,20 @@ String Logger::getVarCodeAtI(uint8_t position_i) { String Logger::getVarUUIDAtI(uint8_t position_i) { return _internalArray->arrayOfVars[position_i]->getVarUUID(); } +// This returns the current value of the variable as a float +float Logger::getValueAtI(uint8_t position_i) { + return _internalArray->arrayOfVars[position_i]->getValue(); +} // This returns the current value of the variable as a string with the // correct number of significant figures String Logger::getValueStringAtI(uint8_t position_i) { return _internalArray->arrayOfVars[position_i]->getValueString(); } +// This returns a particular value of the variable as a string with the +// correct number of significant figures +String Logger::formatValueStringAtI(uint8_t position_i, float value) { + return _internalArray->arrayOfVars[position_i]->formatValueString(value); +} // ===================================================================== // diff --git a/src/LoggerBase.h b/src/LoggerBase.h index 8940a282b..96a1f7d33 100644 --- a/src/LoggerBase.h +++ b/src/LoggerBase.h @@ -526,6 +526,14 @@ class Logger { * @return **String** The variable UUID */ String getVarUUIDAtI(uint8_t position_i); + /** + * @brief Get the most recent value of the variable at the given position in + * the internal variable array object. + * + * @param position_i The position of the variable in the array. + * @return **float** The value of the variable as a float. + */ + float getValueAtI(uint8_t position_i); /** * @brief Get the most recent value of the variable at the given position in * the internal variable array object. @@ -535,6 +543,16 @@ class Logger { * number of significant figures. */ String getValueStringAtI(uint8_t position_i); + /** + * @brief Get the string representing a particular value of the variable at + * the given position in the internal variable array object. + * + * @param position_i The position of the variable in the array. + * @param value The value to format. + * @return **String** The given value as a string with the correct number of + * significant figures. + */ + String formatValueStringAtI(uint8_t position_i, float value); protected: /** diff --git a/src/VariableBase.cpp b/src/VariableBase.cpp index 4fc8cc0ab..977e7e143 100644 --- a/src/VariableBase.cpp +++ b/src/VariableBase.cpp @@ -255,11 +255,18 @@ float Variable::getValue(bool updateValue) { // This returns the current value of the variable as a string // with the correct number of significant figures String Variable::getValueString(bool updateValue) { + return formatValueString(getValue(updateValue)); +} + + +// This returns a particular value of the variable as a string +// with the correct number of significant figures +String Variable::formatValueString(float value) { // Need this because otherwise get extra spaces in strings from int if (_decimalResolution == 0) { - auto val = static_cast(getValue(updateValue)); + auto val = static_cast(value); return String(val); } else { - return String(getValue(updateValue), _decimalResolution); + return String(value, _decimalResolution); } } diff --git a/src/VariableBase.h b/src/VariableBase.h index 35c413acd..b72c5fb19 100644 --- a/src/VariableBase.h +++ b/src/VariableBase.h @@ -356,6 +356,14 @@ class Variable { * @return **String** The current value of the variable */ String getValueString(bool updateValue = false); + /** + * @brief Get a particular value of the variable as a string with the + * correct decimal resolution + * + * @param value value to format + * @return **String** The formatted value of the variable + */ + String formatValueString(float value); /** * @brief Pointer to the parent sensor diff --git a/src/publishers/EnviroDIYPublisher.cpp b/src/publishers/EnviroDIYPublisher.cpp index edcd3da7c..4e0c46990 100644 --- a/src/publishers/EnviroDIYPublisher.cpp +++ b/src/publishers/EnviroDIYPublisher.cpp @@ -12,6 +12,7 @@ #include "EnviroDIYPublisher.h" char EnviroDIYPublisher::dataBuffer[MS_DATA_BUFFER_SIZE]; +int EnviroDIYPublisher::dataBufferNumRecords; // ============================================================================ // Functions for the EnviroDIY data portal receivers. @@ -29,7 +30,7 @@ const char* EnviroDIYPublisher::contentTypeHeader = "\r\nContent-Type: application/json\r\n\r\n"; const char* EnviroDIYPublisher::samplingFeatureTag = "{\"sampling_feature\":\""; -const char* EnviroDIYPublisher::timestampTag = "\",\"timestamp\":[\""; +const char* EnviroDIYPublisher::timestampTag = "\",\"timestamp\":["; // Constructors @@ -66,16 +67,34 @@ void EnviroDIYPublisher::setToken(const char* registrationToken) { // Calculates how long the JSON will be uint16_t EnviroDIYPublisher::calculateJsonSize() { - uint16_t jsonLength = 21; // {"sampling_feature":" - jsonLength += 36; // sampling feature UUID - jsonLength += 17; // ","timestamp":["] - jsonLength += 25; // markedISO8601Time - jsonLength += 2; // ", + char* ptr = (char*)&dataBuffer; + size_t recordSize = sizeof(uint32_t) + + sizeof(float) * _baseLogger->getArrayVarCount(); + + uint16_t jsonLength = strlen(samplingFeatureTag); + jsonLength += 36; // sampling feature UUID + jsonLength += strlen(timestampTag); + // markedISO8601Time + quotes and commas + jsonLength += dataBufferNumRecords * (25 + 2) + dataBufferNumRecords - 1; + jsonLength += 2; // ], for (uint8_t i = 0; i < _baseLogger->getArrayVarCount(); i++) { jsonLength += 1; // " jsonLength += 36; // variable UUID jsonLength += 4; // ":[] - jsonLength += _baseLogger->getValueStringAtI(i).length(); + + ptr = (char*)&dataBuffer; + ptr += sizeof(uint32_t); // skip timestamp + ptr += i * sizeof(float); // and previous variables + for (int j = 0; j < dataBufferNumRecords; j++) { + float value; + memcpy((void*)&value, ptr, sizeof(float)); + ptr += recordSize; + + jsonLength += _baseLogger->formatValueStringAtI(i, value).length(); + if (j + 1 != dataBufferNumRecords) { + jsonLength += 1; // , + } + } if (i + 1 != _baseLogger->getArrayVarCount()) { jsonLength += 1; // , } @@ -108,6 +127,30 @@ void EnviroDIYPublisher::begin(Logger& baseLogger, // over that connection. // The return is the http status code of the response. int16_t EnviroDIYPublisher::publishData(Client* outClient) { + // record timestamp and variable values into the data buffer + + // compute where we will record to + size_t recordSize = sizeof(uint32_t) + + sizeof(float) * _baseLogger->getArrayVarCount(); + char* ptr = &dataBuffer[recordSize * dataBufferNumRecords]; + + memcpy(ptr, (void*)&Logger::markedLocalEpochTime, sizeof(uint32_t)); + ptr += sizeof(uint32_t); + + for (uint8_t i = 0; i < _baseLogger->getArrayVarCount(); i++) { + float value = _baseLogger->getValueAtI(i); + memcpy(ptr, (void*)&value, sizeof(float)); + ptr += sizeof(float); + } + + dataBufferNumRecords += 1; + + if (dataBufferNumRecords == 2) { return flushDataBuffer(outClient); } + + return 201; // pretend everything went okay? +} + +int16_t EnviroDIYPublisher::flushDataBuffer(Client* outClient) { // Create a buffer for the portions of the request and response char tempBuffer[37] = ""; uint16_t did_respond = 0; @@ -143,21 +186,46 @@ int16_t EnviroDIYPublisher::publishData(Client* outClient) { txBufferAppend(_baseLogger->getSamplingFeatureUUID()); txBufferAppend(timestampTag); - txBufferAppend( - Logger::formatDateTime_ISO8601(Logger::markedLocalEpochTime) - .c_str()); - txBufferAppend('"'); + + // write out list of timestamps + char* ptr = (char*)&dataBuffer; + size_t recordSize = sizeof(uint32_t) + + sizeof(float) * _baseLogger->getArrayVarCount(); + for (int i = 0; i < dataBufferNumRecords; i++) { + uint32_t value; + memcpy((void*)&value, ptr, sizeof(uint32_t)); + ptr += recordSize; + + txBufferAppend('"'); + txBufferAppend(Logger::formatDateTime_ISO8601(value).c_str()); + txBufferAppend('"'); + if (i + 1 != dataBufferNumRecords) { txBufferAppend(','); } + } txBufferAppend(']'); txBufferAppend(','); + // write out a list of the values of each variable for (uint8_t i = 0; i < _baseLogger->getArrayVarCount(); i++) { txBufferAppend('"'); txBufferAppend(_baseLogger->getVarUUIDAtI(i).c_str()); txBufferAppend('"'); txBufferAppend(':'); txBufferAppend('['); - txBufferAppend(_baseLogger->getValueStringAtI(i).c_str()); + + ptr = (char*)&dataBuffer; + ptr += sizeof(uint32_t); // skip timestamp + ptr += i * sizeof(float); // and previous variables + for (int j = 0; j < dataBufferNumRecords; j++) { + float value; + memcpy((void*)&value, ptr, sizeof(float)); + ptr += recordSize; + + txBufferAppend( + _baseLogger->formatValueStringAtI(i, value).c_str()); + if (j + 1 != dataBufferNumRecords) { txBufferAppend(','); } + } txBufferAppend(']'); + if (i + 1 != _baseLogger->getArrayVarCount()) { txBufferAppend(','); } else { @@ -205,5 +273,10 @@ int16_t EnviroDIYPublisher::publishData(Client* outClient) { PRINTOUT(F("\n-- Response Code --")); PRINTOUT(responseCode); + if (responseCode == 201) { + // data was successfully transmitted, we can discard it from the buffer + dataBufferNumRecords = 0; + } + return responseCode; } diff --git a/src/publishers/EnviroDIYPublisher.h b/src/publishers/EnviroDIYPublisher.h index b9793f31e..c42cd25ba 100644 --- a/src/publishers/EnviroDIYPublisher.h +++ b/src/publishers/EnviroDIYPublisher.h @@ -192,6 +192,11 @@ class EnviroDIYPublisher : public dataPublisher { // queued sensor data buffer static char dataBuffer[MS_DATA_BUFFER_SIZE]; + // number of records currently in the buffer + static int dataBufferNumRecords; + + // actually transmit rather than just buffer data + int16_t flushDataBuffer(Client* outClient); private: // Tokens and UUID's for EnviroDIY From 58a17e9e9df85683403a6e064cb92fbdd938158d Mon Sep 17 00:00:00 2001 From: Thomas Watson Date: Thu, 9 Feb 2023 21:43:27 -0600 Subject: [PATCH 07/23] publishers/EnviroDIYPublisher: move log buffer logic to its own class Clean up and make maintenance easier --- src/LogBuffer.cpp | 90 +++++++++++++++++ src/LogBuffer.h | 138 ++++++++++++++++++++++++++ src/publishers/EnviroDIYPublisher.cpp | 98 ++++++++---------- src/publishers/EnviroDIYPublisher.h | 10 +- 4 files changed, 270 insertions(+), 66 deletions(-) create mode 100644 src/LogBuffer.cpp create mode 100644 src/LogBuffer.h diff --git a/src/LogBuffer.cpp b/src/LogBuffer.cpp new file mode 100644 index 000000000..5932ce614 --- /dev/null +++ b/src/LogBuffer.cpp @@ -0,0 +1,90 @@ +/** + * @file LogBuffer.cpp + * @copyright 2023 Thomas Watson + * Part of the EnviroDIY ModularSensors library for Arduino + * @author Thomas Watson + * + * @brief Implements the LogBuffer class. + * + * This class buffers logged timestamps and variable values for transmission. + */ +#include "LogBuffer.h" + +#include + +// Constructor +LogBuffer::LogBuffer() {} +// Destructor +LogBuffer::~LogBuffer() {} + +void LogBuffer::setNumVariables(uint8_t numVariables_) { + // each record is one uint32_t to hold the timestamp, plus N floats to hold + // each variable's value + recordSize = sizeof(uint32_t) + sizeof(float) * numVariables_; + numVariables = numVariables_; + + // this scrambles all the data in the buffer so clear it out + numRecords = 0; +} + +void LogBuffer::clear(void) { + // clear out the buffer + numRecords = 0; +} + +uint8_t LogBuffer::getNumVariables(void) { + return numVariables; +} + +int LogBuffer::getNumRecords(void) { + return numRecords; +} + +int LogBuffer::addRecord(uint32_t timestamp) { + int record = numRecords; + // compute position of the new record's timestamp in the buffer + // (the timestamp is the first data in the record) + size_t pos = record * recordSize; + // verify we have sufficient space for the record and bail if not + if (MS_LOG_DATA_BUFFER_SIZE - pos < recordSize) { return -1; } + + // write the timestamp to the record + memcpy(static_cast(&dataBuffer[pos]), static_cast(×tamp), + sizeof(uint32_t)); + numRecords += 1; // just added another record + + return record; +} + +void LogBuffer::setRecordValue(int record, uint8_t variable, float value) { + // compute position of this value in the buffer + size_t pos = record * recordSize + sizeof(uint32_t) + + variable * sizeof(float); + + // write the value to the record + memcpy(static_cast(&dataBuffer[pos]), static_cast(&value), + sizeof(float)); +} + +uint32_t LogBuffer::getRecordTimestamp(int record) { + // read the timestamp from the record (which is the first data in it) + uint32_t timestamp; + memcpy(static_cast(×tamp), + static_cast(&dataBuffer[record * recordSize]), + sizeof(uint32_t)); + + return timestamp; +} + +float LogBuffer::getRecordValue(int record, uint8_t variable) { + // compute position of this value in the buffer + size_t pos = record * recordSize + sizeof(uint32_t) + + variable * sizeof(float); + + // read the value from the record + float value; + memcpy(static_cast(&value), static_cast(&dataBuffer[pos]), + sizeof(float)); + + return value; +} diff --git a/src/LogBuffer.h b/src/LogBuffer.h new file mode 100644 index 000000000..4422a5608 --- /dev/null +++ b/src/LogBuffer.h @@ -0,0 +1,138 @@ +/** + * @file LogBuffer.cpp + * @copyright 2023 Thomas Watson + * Part of the EnviroDIY ModularSensors library for Arduino + * @author Thomas Watson + * + * @brief Implements the LogBuffer class. + * + * This class buffers logged timestamps and variable values for transmission. + */ + +// Header Guards +#ifndef SRC_LOGBUFFER_H_ +#define SRC_LOGBUFFER_H_ + +/** + * @def MS_LOG_DATA_BUFFER_SIZE + * @brief Log Data Buffer + * + * This determines how much RAM is reserved to buffer log records before + * transmission. Each record consumes 4 bytes for the timestamp plus 4 bytes + * for each logged variable. Increasing this value too far can crash the + * device! The number of log records buffered is controlled by sendEveryX. + * + * This can be changed by setting the build flag MS_LOG_DATA_BUFFER_SIZE when + * compiling. 8192 bytes is a safe value for the Mayfly 1.1 with six variables. + */ +#ifndef MS_LOG_DATA_BUFFER_SIZE +#define MS_LOG_DATA_BUFFER_SIZE 8192 +#endif + +#include +#include + +/** + * @brief This class buffers logged timestamps and variable values for + * transmission. The log is divided into a number of records. Each record + * stores the timestamp of the record as a uint32_t, then the value of each + * variable as a float at that time. + */ +class LogBuffer { + public: + /** + * @brief Constructs a new empty buffer which stores no variables or values. + */ + LogBuffer(); + /** + * @brief Destroys the buffer. + */ + virtual ~LogBuffer(); + + /** + * @brief Sets the number of variables the buffer will store in each record. + * Clears the buffer as a side effect. + * + * @param[in] numVariables_ The number of variables to store. + */ + void setNumVariables(uint8_t numVariables_); + + /** + * @brief Gets the number of variables that will be stored in each record. + * + * @return The variable count. + */ + uint8_t getNumVariables(void); + + /** + * @brief Clears all records from the log. + */ + void clear(void); + + /** + * @brief Gets the number of records currently in the log. + * + * @return The number of records. + */ + int getNumRecords(void); + + /** + * @brief Adds a new record with the given timestamp. + * + * @param[in] timestamp The timestamp + * + * @return Index of the new record, or -1 if there was no space. + */ + int addRecord(uint32_t timestamp); + + /** + * @brief Sets the value of a particular variable in a particular record. + * + * @param[in] record The record + * @param[in] variable The variable + * @param[in] value The value + */ + void setRecordValue(int record, uint8_t variable, float value); + + /** + * @brief Gets the timestamp of a particular record. + * + * @param[in] record The record + * + * @return The record's timestamp. + */ + uint32_t getRecordTimestamp(int record); + + /** + * @brief Gets the value of a particular vaiable in a particular record. + * + * @param[in] record The record + * @param[in] variable The variable + * + * @return The variable's value. + */ + float getRecordValue(int record, uint8_t variable); + + protected: + /** + * @brief Buffer which stores the log data. + */ + uint8_t dataBuffer[MS_LOG_DATA_BUFFER_SIZE]; + + /** + * @brief Number of records currently in the buffer. + */ + int numRecords; + + /** + * @brief Size in bytes of each record in the buffer. + */ + size_t recordSize; + + /** + * @brief Number of variables stored in each record in the buffer. + */ + uint8_t numVariables; +}; + +#endif // SRC_LOGBUFFER_H_ diff --git a/src/publishers/EnviroDIYPublisher.cpp b/src/publishers/EnviroDIYPublisher.cpp index 4e0c46990..29ba16d25 100644 --- a/src/publishers/EnviroDIYPublisher.cpp +++ b/src/publishers/EnviroDIYPublisher.cpp @@ -11,8 +11,7 @@ #include "EnviroDIYPublisher.h" -char EnviroDIYPublisher::dataBuffer[MS_DATA_BUFFER_SIZE]; -int EnviroDIYPublisher::dataBufferNumRecords; +LogBuffer EnviroDIYPublisher::_logBuffer; // ============================================================================ // Functions for the EnviroDIY data portal receivers. @@ -47,6 +46,7 @@ EnviroDIYPublisher::EnviroDIYPublisher(Logger& baseLogger, : dataPublisher(baseLogger, sendEveryX) { setToken(registrationToken); _baseLogger->setSamplingFeatureUUID(samplingFeatureUUID); + _logBuffer.setNumVariables(_baseLogger->getArrayVarCount()); } EnviroDIYPublisher::EnviroDIYPublisher(Logger& baseLogger, Client* inClient, const char* registrationToken, @@ -55,6 +55,7 @@ EnviroDIYPublisher::EnviroDIYPublisher(Logger& baseLogger, Client* inClient, : dataPublisher(baseLogger, inClient, sendEveryX) { setToken(registrationToken); _baseLogger->setSamplingFeatureUUID(samplingFeatureUUID); + _logBuffer.setNumVariables(_baseLogger->getArrayVarCount()); } // Destructor EnviroDIYPublisher::~EnviroDIYPublisher() {} @@ -67,35 +68,30 @@ void EnviroDIYPublisher::setToken(const char* registrationToken) { // Calculates how long the JSON will be uint16_t EnviroDIYPublisher::calculateJsonSize() { - char* ptr = (char*)&dataBuffer; - size_t recordSize = sizeof(uint32_t) + - sizeof(float) * _baseLogger->getArrayVarCount(); + uint8_t variables = _logBuffer.getNumVariables(); + int records = _logBuffer.getNumRecords(); uint16_t jsonLength = strlen(samplingFeatureTag); jsonLength += 36; // sampling feature UUID + jsonLength += 36; // sampling feature UUID jsonLength += strlen(timestampTag); // markedISO8601Time + quotes and commas - jsonLength += dataBufferNumRecords * (25 + 2) + dataBufferNumRecords - 1; + jsonLength += records * (25 + 2) + records - 1; jsonLength += 2; // ], - for (uint8_t i = 0; i < _baseLogger->getArrayVarCount(); i++) { + for (uint8_t var = 0; var < variables; var++) { jsonLength += 1; // " jsonLength += 36; // variable UUID jsonLength += 4; // ":[] - ptr = (char*)&dataBuffer; - ptr += sizeof(uint32_t); // skip timestamp - ptr += i * sizeof(float); // and previous variables - for (int j = 0; j < dataBufferNumRecords; j++) { - float value; - memcpy((void*)&value, ptr, sizeof(float)); - ptr += recordSize; - - jsonLength += _baseLogger->formatValueStringAtI(i, value).length(); - if (j + 1 != dataBufferNumRecords) { + for (int rec = 0; rec < records; rec++) { + float value = _logBuffer.getRecordValue(rec, var); + jsonLength += + _baseLogger->formatValueStringAtI(var, value).length(); + if (rec + 1 != records) { jsonLength += 1; // , } } - if (i + 1 != _baseLogger->getArrayVarCount()) { + if (var + 1 != variables) { jsonLength += 1; // , } } @@ -112,6 +108,7 @@ void EnviroDIYPublisher::begin(Logger& baseLogger, Client* inClient, setToken(registrationToken); dataPublisher::begin(baseLogger, inClient); _baseLogger->setSamplingFeatureUUID(samplingFeatureUUID); + _logBuffer.setNumVariables(_baseLogger->getArrayVarCount()); } void EnviroDIYPublisher::begin(Logger& baseLogger, const char* registrationToken, @@ -119,6 +116,7 @@ void EnviroDIYPublisher::begin(Logger& baseLogger, setToken(registrationToken); dataPublisher::begin(baseLogger); _baseLogger->setSamplingFeatureUUID(samplingFeatureUUID); + _logBuffer.setNumVariables(_baseLogger->getArrayVarCount()); } @@ -127,27 +125,21 @@ void EnviroDIYPublisher::begin(Logger& baseLogger, // over that connection. // The return is the http status code of the response. int16_t EnviroDIYPublisher::publishData(Client* outClient) { - // record timestamp and variable values into the data buffer - - // compute where we will record to - size_t recordSize = sizeof(uint32_t) + - sizeof(float) * _baseLogger->getArrayVarCount(); - char* ptr = &dataBuffer[recordSize * dataBufferNumRecords]; + // create record to hold timestamp and variable values in the log buffer + int record = _logBuffer.addRecord(Logger::markedLocalEpochTime); - memcpy(ptr, (void*)&Logger::markedLocalEpochTime, sizeof(uint32_t)); - ptr += sizeof(uint32_t); - - for (uint8_t i = 0; i < _baseLogger->getArrayVarCount(); i++) { - float value = _baseLogger->getValueAtI(i); - memcpy(ptr, (void*)&value, sizeof(float)); - ptr += sizeof(float); + // write record data if the record was successfully created + if (record >= 0) { + for (uint8_t i = 0; i < _baseLogger->getArrayVarCount(); i++) { + _logBuffer.setRecordValue(record, i, _baseLogger->getValueAtI(i)); + } } - dataBufferNumRecords += 1; - - if (dataBufferNumRecords == 2) { return flushDataBuffer(outClient); } + // flush data if log buffer is full or we've hit the requested interval + if ((record >= 1) || (record < 0)) { return flushDataBuffer(outClient); } return 201; // pretend everything went okay? + return 201; // pretend everything went okay? } int16_t EnviroDIYPublisher::flushDataBuffer(Client* outClient) { @@ -188,45 +180,35 @@ int16_t EnviroDIYPublisher::flushDataBuffer(Client* outClient) { txBufferAppend(timestampTag); // write out list of timestamps - char* ptr = (char*)&dataBuffer; - size_t recordSize = sizeof(uint32_t) + - sizeof(float) * _baseLogger->getArrayVarCount(); - for (int i = 0; i < dataBufferNumRecords; i++) { - uint32_t value; - memcpy((void*)&value, ptr, sizeof(uint32_t)); - ptr += recordSize; - + int records = _logBuffer.getNumRecords(); + for (int rec = 0; rec < records; rec++) { txBufferAppend('"'); - txBufferAppend(Logger::formatDateTime_ISO8601(value).c_str()); + uint32_t timestamp = _logBuffer.getRecordTimestamp(rec); + txBufferAppend(Logger::formatDateTime_ISO8601(timestamp).c_str()); txBufferAppend('"'); - if (i + 1 != dataBufferNumRecords) { txBufferAppend(','); } + if (rec + 1 != records) { txBufferAppend(','); } } txBufferAppend(']'); txBufferAppend(','); // write out a list of the values of each variable - for (uint8_t i = 0; i < _baseLogger->getArrayVarCount(); i++) { + uint8_t variables = _logBuffer.getNumVariables(); + for (uint8_t var = 0; var < variables; var++) { txBufferAppend('"'); - txBufferAppend(_baseLogger->getVarUUIDAtI(i).c_str()); + txBufferAppend(_baseLogger->getVarUUIDAtI(var).c_str()); txBufferAppend('"'); txBufferAppend(':'); txBufferAppend('['); - ptr = (char*)&dataBuffer; - ptr += sizeof(uint32_t); // skip timestamp - ptr += i * sizeof(float); // and previous variables - for (int j = 0; j < dataBufferNumRecords; j++) { - float value; - memcpy((void*)&value, ptr, sizeof(float)); - ptr += recordSize; - + for (int rec = 0; rec < records; rec++) { + float value = _logBuffer.getRecordValue(rec, var); txBufferAppend( - _baseLogger->formatValueStringAtI(i, value).c_str()); - if (j + 1 != dataBufferNumRecords) { txBufferAppend(','); } + _baseLogger->formatValueStringAtI(var, value).c_str()); + if (rec + 1 != records) { txBufferAppend(','); } } txBufferAppend(']'); - if (i + 1 != _baseLogger->getArrayVarCount()) { + if (var + 1 != variables) { txBufferAppend(','); } else { txBufferAppend('}'); @@ -275,7 +257,7 @@ int16_t EnviroDIYPublisher::flushDataBuffer(Client* outClient) { if (responseCode == 201) { // data was successfully transmitted, we can discard it from the buffer - dataBufferNumRecords = 0; + _logBuffer.clear(); } return responseCode; diff --git a/src/publishers/EnviroDIYPublisher.h b/src/publishers/EnviroDIYPublisher.h index c42cd25ba..5a8d0cdfe 100644 --- a/src/publishers/EnviroDIYPublisher.h +++ b/src/publishers/EnviroDIYPublisher.h @@ -18,10 +18,6 @@ // Debugging Statement // #define MS_ENVIRODIYPUBLISHER_DEBUG -#ifndef MS_DATA_BUFFER_SIZE -#define MS_DATA_BUFFER_SIZE 8192 -#endif - #ifdef MS_ENVIRODIYPUBLISHER_DEBUG #define MS_DEBUGGING_STD "EnviroDIYPublisher" #endif @@ -30,6 +26,7 @@ #include "ModSensorDebugger.h" #undef MS_DEBUGGING_STD #include "dataPublisherBase.h" +#include "LogBuffer.h" // ============================================================================ @@ -190,10 +187,7 @@ class EnviroDIYPublisher : public dataPublisher { static const char* timestampTag; ///< The JSON feature timestamp tag /**@}*/ - // queued sensor data buffer - static char dataBuffer[MS_DATA_BUFFER_SIZE]; - // number of records currently in the buffer - static int dataBufferNumRecords; + static LogBuffer _logBuffer; // actually transmit rather than just buffer data int16_t flushDataBuffer(Client* outClient); From bf548d411d5af3fa1204993c90b7a5ef5b0304f0 Mon Sep 17 00:00:00 2001 From: Thomas Watson Date: Thu, 9 Feb 2023 21:46:48 -0600 Subject: [PATCH 08/23] only turn modem on when internet connection is needed by a publisher Avoids unnecessary time and power consumption when publishers just plan to buffer the data. --- src/LoggerBase.cpp | 33 +++++++++++++++++++++++---- src/LoggerBase.h | 7 ++++++ src/dataPublisherBase.cpp | 4 ++++ src/dataPublisherBase.h | 8 +++++++ src/publishers/EnviroDIYPublisher.cpp | 3 +++ src/publishers/EnviroDIYPublisher.h | 8 +++++++ 6 files changed, 58 insertions(+), 5 deletions(-) diff --git a/src/LoggerBase.cpp b/src/LoggerBase.cpp index 36b6f33e2..3ab03c76c 100644 --- a/src/LoggerBase.cpp +++ b/src/LoggerBase.cpp @@ -339,6 +339,18 @@ void Logger::registerDataPublisher(dataPublisher* publisher) { dataPublishers[i] = publisher; } +bool Logger::checkRemotesConnectionNeeded(void) { + MS_DBG(F("Asking publishers if they need a connection.")); + + bool needed = false; + for (uint8_t i = 0; i < MAX_NUMBER_SENDERS; i++) { + if (dataPublishers[i] != nullptr) { + needed = needed || dataPublishers[i]->connectionNeeded(); + } + } + + return needed; +} void Logger::publishDataToRemotes(void) { MS_DBG(F("Sending out remote data.")); @@ -1560,7 +1572,15 @@ void Logger::logDataAndPublish(bool sleepBeforeReturning) { // Create a csv data record and save it to the log file logToSD(); - if (_logModem != nullptr) { + // Sync the clock at noon + bool clockSyncNeeded = + (Logger::markedLocalEpochTime != 0 && + Logger::markedLocalEpochTime % 86400 == 43200) || + !isRTCSane(Logger::markedLocalEpochTime); + bool connectionNeeded = checkRemotesConnectionNeeded() || + clockSyncNeeded; + + if (_logModem != nullptr && connectionNeeded) { MS_DBG(F("Waking up"), _logModem->getModemName(), F("...")); if (_logModem->modemWake()) { // Connect to the network @@ -1572,10 +1592,7 @@ void Logger::logDataAndPublish(bool sleepBeforeReturning) { publishDataToRemotes(); watchDogTimer.resetWatchDog(); - if ((Logger::markedLocalEpochTime != 0 && - Logger::markedLocalEpochTime % 86400 == 43200) || - !isRTCSane(Logger::markedLocalEpochTime)) { - // Sync the clock at noon + if (clockSyncNeeded) { MS_DBG(F("Running a daily clock sync...")); setRTClock(_logModem->getNISTTime()); watchDogTimer.resetWatchDog(); @@ -1595,6 +1612,12 @@ void Logger::logDataAndPublish(bool sleepBeforeReturning) { } // Turn the modem off _logModem->modemSleepPowerDown(); + } else if (_logModem != nullptr) { + MS_DBG(F("Nobody needs it so publishing without connecting...")); + // Call publish function without connection + watchDogTimer.resetWatchDog(); + publishDataToRemotes(); + watchDogTimer.resetWatchDog(); } diff --git a/src/LoggerBase.h b/src/LoggerBase.h index 96a1f7d33..25688e14d 100644 --- a/src/LoggerBase.h +++ b/src/LoggerBase.h @@ -597,6 +597,13 @@ class Logger { * @param publisher A dataPublisher object */ void registerDataPublisher(dataPublisher* publisher); + /** + * @brief Check if any data publishers need an Internet connection for the + * next publish call. + * + * @return True if any remotes need a connection. + */ + bool checkRemotesConnectionNeeded(void); /** * @brief Publish data to all registered data publishers. */ diff --git a/src/dataPublisherBase.cpp b/src/dataPublisherBase.cpp index 5429a696f..1fff62c45 100644 --- a/src/dataPublisherBase.cpp +++ b/src/dataPublisherBase.cpp @@ -122,6 +122,10 @@ void dataPublisher::txBufferFlush() { txBufferLen = 0; } +bool dataPublisher::connectionNeeded(void) { + // connection is always needed unless publisher has special logic + return true; +} // This sends data on the "default" client of the modem int16_t dataPublisher::publishData() { diff --git a/src/dataPublisherBase.h b/src/dataPublisherBase.h index 6ccefa91d..da6175944 100644 --- a/src/dataPublisherBase.h +++ b/src/dataPublisherBase.h @@ -171,6 +171,14 @@ class dataPublisher { virtual String getEndpoint(void) = 0; + /** + * @brief Checks if the publisher needs an Internet connection for the next + * publishData call (as opposed to just buffering data internally). + * + * @return True if an internet connection is needed for the next publish. + */ + virtual bool connectionNeeded(void); + /** * @brief Open a socket to the correct receiver and sends out the formatted * data. diff --git a/src/publishers/EnviroDIYPublisher.cpp b/src/publishers/EnviroDIYPublisher.cpp index 29ba16d25..0ff5c6e78 100644 --- a/src/publishers/EnviroDIYPublisher.cpp +++ b/src/publishers/EnviroDIYPublisher.cpp @@ -119,6 +119,9 @@ void EnviroDIYPublisher::begin(Logger& baseLogger, _logBuffer.setNumVariables(_baseLogger->getArrayVarCount()); } +bool EnviroDIYPublisher::connectionNeeded(void) { + return _logBuffer.getNumRecords() >= 1; +} // This utilizes an attached modem to make a TCP connection to the // EnviroDIY/ODM2DataSharingPortal and then streams out a post request diff --git a/src/publishers/EnviroDIYPublisher.h b/src/publishers/EnviroDIYPublisher.h index 5a8d0cdfe..6a5c7a191 100644 --- a/src/publishers/EnviroDIYPublisher.h +++ b/src/publishers/EnviroDIYPublisher.h @@ -147,6 +147,14 @@ class EnviroDIYPublisher : public dataPublisher { void begin(Logger& baseLogger, const char* registrationToken, const char* samplingFeatureUUID); + /** + * @brief Checks if the publisher needs an Internet connection for the next + * publishData call (as opposed to just buffering data internally). + * + * @return True if an internet connection is needed for the next publish. + */ + bool connectionNeeded(void) override; + /** * @brief Utilize an attached modem to open a a TCP connection to the * EnviroDIY/ODM2DataSharingPortal and then stream out a post request over From 447f5827241e1d5efb661a02339f36441b2c182d Mon Sep 17 00:00:00 2001 From: Thomas Watson Date: Thu, 9 Feb 2023 21:49:37 -0600 Subject: [PATCH 09/23] dataPublisherBase: correctly retry flushing if modem buffer is full The modem might not be able to accept the data written to it if its transmit buffer is full. In that case, the portion not accepted needs to be retried later. If we retry too many times, give up so we don't get stuck in an infinite loop. --- src/dataPublisherBase.cpp | 38 ++++++++++++++++++++++++++++++++++---- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/src/dataPublisherBase.cpp b/src/dataPublisherBase.cpp index 1fff62c45..a1c2111ad 100644 --- a/src/dataPublisherBase.cpp +++ b/src/dataPublisherBase.cpp @@ -110,16 +110,46 @@ void dataPublisher::txBufferAppend(char c) { } void dataPublisher::txBufferFlush() { + if ((txBufferOutClient == nullptr) || (txBufferLen == 0)) { + // sending into the void... + txBufferLen = 0; + return; + } + #if defined(STANDARD_SERIAL_OUTPUT) // write out to the printout stream STANDARD_SERIAL_OUTPUT.write((const uint8_t*)txBuffer, txBufferLen); STANDARD_SERIAL_OUTPUT.flush(); #endif - // write out to the client - txBufferOutClient->write((const uint8_t*)txBuffer, txBufferLen); - txBufferOutClient->flush(); - txBufferLen = 0; + uint8_t tries = 10; + const uint8_t* ptr = (const uint8_t*)txBuffer; + while (true) { + size_t sent = txBufferOutClient->write(ptr, txBufferLen); + txBufferLen -= sent; + ptr += sent; + if (txBufferLen == 0) { + // whole message is successfully sent, we are done + txBufferOutClient->flush(); + return; + } + +#if defined(STANDARD_SERIAL_OUTPUT) + // warn that we only partially sent the buffer + STANDARD_SERIAL_OUTPUT.write('!'); +#endif + if (--tries == 0) { + // can't convince the modem to send the whole message. just break + // the connection now so it will get reset and we can try to + // transmit the data again later + txBufferOutClient = nullptr; + txBufferLen = 0; + return; + } + + // give the modem a chance to transmit buffered data + delay(1000); + } } bool dataPublisher::connectionNeeded(void) { From 212ead3214bd1719755c47dfde4e69b31d85d2f8 Mon Sep 17 00:00:00 2001 From: Thomas Watson Date: Thu, 9 Feb 2023 22:07:25 -0600 Subject: [PATCH 10/23] publishers/EnviroDIYPublisher: send initial logged data immediately Send the first few data points logged after initialization immediately instead of buffering them until the programmed interval elapses. Allows verification of functionality during deployment. --- src/publishers/EnviroDIYPublisher.cpp | 27 ++++++++++++++++++++++----- src/publishers/EnviroDIYPublisher.h | 4 ++++ 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/src/publishers/EnviroDIYPublisher.cpp b/src/publishers/EnviroDIYPublisher.cpp index 0ff5c6e78..d9bd07efc 100644 --- a/src/publishers/EnviroDIYPublisher.cpp +++ b/src/publishers/EnviroDIYPublisher.cpp @@ -120,7 +120,14 @@ void EnviroDIYPublisher::begin(Logger& baseLogger, } bool EnviroDIYPublisher::connectionNeeded(void) { - return _logBuffer.getNumRecords() >= 1; + // the programmed interval is about to be reached by the next record + bool atSendInterval = _logBuffer.getNumRecords() >= (_sendEveryX - 1); + + // the initial log transmissions have not completed (we send every one + // of the first five data points immediately for field validation) + bool initialTransmission = _initialTransmissionsRemaining > 0; + + return atSendInterval || initialTransmission; } // This utilizes an attached modem to make a TCP connection to the @@ -128,6 +135,11 @@ bool EnviroDIYPublisher::connectionNeeded(void) { // over that connection. // The return is the http status code of the response. int16_t EnviroDIYPublisher::publishData(Client* outClient) { + // do we intend to send this call? if so, we have just returned true from + // connectionNeeded() and the internet is connected and waiting. check what + // that function said so we know to do it after we record this data point. + bool willFlush = connectionNeeded(); + // create record to hold timestamp and variable values in the log buffer int record = _logBuffer.addRecord(Logger::markedLocalEpochTime); @@ -138,11 +150,16 @@ int16_t EnviroDIYPublisher::publishData(Client* outClient) { } } - // flush data if log buffer is full or we've hit the requested interval - if ((record >= 1) || (record < 0)) { return flushDataBuffer(outClient); } + if (_initialTransmissionsRemaining > 0) { + _initialTransmissionsRemaining -= 1; + } - return 201; // pretend everything went okay? - return 201; // pretend everything went okay? + // do the data buffer flushing if we previously planned to + if (willFlush) { + return flushDataBuffer(outClient); + } else { + return 201; // pretend everything went okay? + } } int16_t EnviroDIYPublisher::flushDataBuffer(Client* outClient) { diff --git a/src/publishers/EnviroDIYPublisher.h b/src/publishers/EnviroDIYPublisher.h index 6a5c7a191..c7d6e2f7e 100644 --- a/src/publishers/EnviroDIYPublisher.h +++ b/src/publishers/EnviroDIYPublisher.h @@ -200,6 +200,10 @@ class EnviroDIYPublisher : public dataPublisher { // actually transmit rather than just buffer data int16_t flushDataBuffer(Client* outClient); + // we send every one of the first five data points immediately for field + // validation + uint8_t _initialTransmissionsRemaining = 5; + private: // Tokens and UUID's for EnviroDIY const char* _registrationToken = nullptr; From e79cbe4419c407cb633213709ebdcfb2f3093d0d Mon Sep 17 00:00:00 2001 From: Thomas Watson Date: Thu, 9 Feb 2023 22:16:08 -0600 Subject: [PATCH 11/23] publishers/EnviroDIYPublisher: use more intelligent sending algorithm Avoid repeatedly retrying if the server is down, while still retrying a couple times in case the connection is unreliable. --- src/LogBuffer.cpp | 7 ++++++ src/LogBuffer.h | 7 ++++++ src/publishers/EnviroDIYPublisher.cpp | 36 +++++++++++++++++++++++++-- 3 files changed, 48 insertions(+), 2 deletions(-) diff --git a/src/LogBuffer.cpp b/src/LogBuffer.cpp index 5932ce614..46c05ef56 100644 --- a/src/LogBuffer.cpp +++ b/src/LogBuffer.cpp @@ -40,6 +40,13 @@ int LogBuffer::getNumRecords(void) { return numRecords; } +uint8_t LogBuffer::getPercentFull(void) { + uint32_t bytesFull = (uint32_t)numRecords * (uint32_t)recordSize; + uint32_t bytesTotal = MS_LOG_DATA_BUFFER_SIZE; + + return (uint8_t)((bytesFull*(uint32_t)100)/bytesTotal); +} + int LogBuffer::addRecord(uint32_t timestamp) { int record = numRecords; // compute position of the new record's timestamp in the buffer diff --git a/src/LogBuffer.h b/src/LogBuffer.h index 4422a5608..b4b241a53 100644 --- a/src/LogBuffer.h +++ b/src/LogBuffer.h @@ -76,6 +76,13 @@ class LogBuffer { */ int getNumRecords(void); + /** + * @brief Computes the percentage full of the buffer. + * + * @return The current percent full. + */ + uint8_t getPercentFull(void); + /** * @brief Adds a new record with the given timestamp. * diff --git a/src/publishers/EnviroDIYPublisher.cpp b/src/publishers/EnviroDIYPublisher.cpp index d9bd07efc..8385f9502 100644 --- a/src/publishers/EnviroDIYPublisher.cpp +++ b/src/publishers/EnviroDIYPublisher.cpp @@ -120,8 +120,40 @@ void EnviroDIYPublisher::begin(Logger& baseLogger, } bool EnviroDIYPublisher::connectionNeeded(void) { - // the programmed interval is about to be reached by the next record - bool atSendInterval = _logBuffer.getNumRecords() >= (_sendEveryX - 1); + // compute the send interval, reducing it as the buffer gets more full so we + // have less of a chance of losing data + int interval = _sendEveryX; + uint8_t percent = _logBuffer.getPercentFull(); + if (percent >= 50) { + interval /= 2; + } else if (percent >= 75) { + interval /= 4; + } else if (percent >= 90) { + interval = 1; + } + + // the programmed interval is about to be reached by the next record, or it + // was just reached and we are trying again + bool atSendInterval = false; + if (interval <= 1) { + atSendInterval = true; + } else { + int numRecords = _logBuffer.getNumRecords(); + // where we are relative to the interval + int relative = (numRecords % interval); + if (relative == (interval - 1)) { + // the next sample will put us right at the interval + atSendInterval = true; + } else if (numRecords >= interval) { // don't send the first sample + if (relative == 0) { + // the last sample was the interval, this is the first retry + atSendInterval = true; + } else if (relative == 1) { + // two samples ago was the interval, this is the second retry + atSendInterval = true; + } + } + } // the initial log transmissions have not completed (we send every one // of the first five data points immediately for field validation) From 8c4e590cb169845650f4712d12e7e54a4d1392d4 Mon Sep 17 00:00:00 2001 From: Thomas Watson Date: Thu, 9 Feb 2023 12:46:28 -0600 Subject: [PATCH 12/23] LoggerBase: make testing button log immediately using normal procedure Allows easy field verification of functionality as opposed to previous testing function which only served a purpose when attached to a computer (and consumed lots of flash). --- src/LoggerBase.cpp | 25 +++++++++++-------------- src/LoggerBase.h | 18 ------------------ 2 files changed, 11 insertions(+), 32 deletions(-) diff --git a/src/LoggerBase.cpp b/src/LoggerBase.cpp index 3ab03c76c..15e0889bc 100644 --- a/src/LoggerBase.cpp +++ b/src/LoggerBase.cpp @@ -32,7 +32,6 @@ uint32_t Logger::markedLocalEpochTime = 0; uint32_t Logger::markedUTCEpochTime = 0; // Initialize the testing/logging flags volatile bool Logger::isLoggingNow = false; -volatile bool Logger::isTestingNow = false; volatile bool Logger::startTesting = false; // Initialize the RTC for the SAMD boards using build in RTC @@ -54,7 +53,6 @@ Logger::Logger(const char* loggerID, uint16_t loggingIntervalMinutes, // Set the testing/logging flags to false isLoggingNow = false; - isTestingNow = false; startTesting = false; // Clear arrays @@ -71,7 +69,6 @@ Logger::Logger(const char* loggerID, uint16_t loggingIntervalMinutes, // Set the testing/logging flags to false isLoggingNow = false; - isTestingNow = false; startTesting = false; // Clear arrays @@ -82,7 +79,6 @@ Logger::Logger(const char* loggerID, uint16_t loggingIntervalMinutes, Logger::Logger() { // Set the testing/logging flags to false isLoggingNow = false; - isTestingNow = false; startTesting = false; // Clear arrays @@ -606,7 +602,7 @@ bool Logger::checkInterval(void) { MS_DBG(F("Mod of Logging Interval:"), checkTime % (_loggingIntervalMinutes * 60)); - if (checkTime % (_loggingIntervalMinutes * 60) == 0) { + if ((checkTime % (interval * 60) == 0)) { // Update the time variables with the current time markTime(); MS_DBG(F("Time marked at (unix):"), Logger::markedLocalEpochTime); @@ -1285,7 +1281,7 @@ bool Logger::logToSD(void) { // A static function if you'd prefer to enter testing based on an interrupt void Logger::testingISR() { MS_DEEP_DBG(F("Testing interrupt!")); - if (!Logger::isTestingNow && !Logger::isLoggingNow) { + if (!Logger::isLoggingNow) { Logger::startTesting = true; MS_DEEP_DBG(F("Testing flag has been set.")); } @@ -1486,10 +1482,12 @@ void Logger::logData(bool sleepBeforeReturning) { watchDogTimer.resetWatchDog(); // Assuming we were woken up by the clock, check if the current time is an - // even interval of the logging interval + // even interval of the logging interval or that we have been specifically + // requested to log by pushbutton if (checkInterval()) { // Flag to notify that we're in already awake and logging a point Logger::isLoggingNow = true; + // Reset the watchdog watchDogTimer.resetWatchDog(); @@ -1520,11 +1518,10 @@ void Logger::logData(bool sleepBeforeReturning) { // Unset flag Logger::isLoggingNow = false; + // Acknowledge testing button if pressed + Logger::startTesting = false; } - // Check if it was instead the testing interrupt that woke us up - if (Logger::startTesting) testingMode(); - if (sleepBeforeReturning) { // Sleep systemSleep(); @@ -1536,7 +1533,8 @@ void Logger::logDataAndPublish(bool sleepBeforeReturning) { watchDogTimer.resetWatchDog(); // Assuming we were woken up by the clock, check if the current time is an - // even interval of the logging interval + // even interval of the logging interval or that we have been specifically + // requested to log by pushbutton if (checkInterval()) { // Flag to notify that we're in already awake and logging a point Logger::isLoggingNow = true; @@ -1635,11 +1633,10 @@ void Logger::logDataAndPublish(bool sleepBeforeReturning) { // Unset flag Logger::isLoggingNow = false; + // Acknowledge testing button if pressed + Logger::startTesting = false; } - // Check if it was instead the testing interrupt that woke us up - if (Logger::startTesting) testingMode(sleepBeforeReturning); - if (sleepBeforeReturning) { // Sleep systemSleep(); diff --git a/src/LoggerBase.h b/src/LoggerBase.h index 25688e14d..bd83bdc27 100644 --- a/src/LoggerBase.h +++ b/src/LoggerBase.h @@ -1200,19 +1200,6 @@ class Logger { * on the pin assigned for "testing" mode. */ static void testingISR(void); - - /** - * @brief Execute testing mode. - * - * In testing mode, the logger uses the loggerModem, if attached, to connect - * to the internet. It then powers up all sensors tied to variable in the - * internal variable array. The logger then updates readings from all - * sensors 25 times with a 5 second wait in between. All results are output - * to the "main" output - ie Serial - and NOT to the SD card. After 25 - * measurements, the sensors are put to sleep, the modem is disconnected - * from the internet, and the logger goes back to sleep. - */ - virtual void testingMode(bool sleepBeforeReturning = true); /**@}*/ // ===================================================================== // @@ -1296,11 +1283,6 @@ class Logger { * sensors or writing to the SD card */ static volatile bool isLoggingNow; - /** - * @brief Internal flag set to true when the logger is going through the - * "testing mode" routine. - */ - static volatile bool isTestingNow; /** * @brief Internal flag set to true with then logger should begin the * "testing mode" routine when it finishes other operations. From b092f32b6032144949e65cfd085f8180782dc6fb Mon Sep 17 00:00:00 2001 From: Thomas Watson Date: Thu, 9 Feb 2023 22:50:03 -0600 Subject: [PATCH 13/23] flush log buffer immediately when test button is pushed Allows immediate transmission of data in buffer without loss before powering off and decomissioning a deployed datalogger. --- src/LoggerBase.cpp | 14 +++++++++----- src/LoggerBase.h | 4 +++- src/dataPublisherBase.cpp | 4 ++-- src/dataPublisherBase.h | 7 +++++-- src/publishers/DreamHostPublisher.cpp | 2 +- src/publishers/DreamHostPublisher.h | 3 ++- src/publishers/EnviroDIYPublisher.cpp | 7 ++++--- src/publishers/EnviroDIYPublisher.h | 3 ++- src/publishers/ThingSpeakPublisher.cpp | 2 +- src/publishers/ThingSpeakPublisher.h | 2 +- src/publishers/UbidotsPublisher.cpp | 2 +- src/publishers/UbidotsPublisher.h | 3 ++- 12 files changed, 33 insertions(+), 20 deletions(-) diff --git a/src/LoggerBase.cpp b/src/LoggerBase.cpp index 15e0889bc..75740fe0f 100644 --- a/src/LoggerBase.cpp +++ b/src/LoggerBase.cpp @@ -348,14 +348,14 @@ bool Logger::checkRemotesConnectionNeeded(void) { return needed; } -void Logger::publishDataToRemotes(void) { +void Logger::publishDataToRemotes(bool forceFlush) { MS_DBG(F("Sending out remote data.")); for (uint8_t i = 0; i < MAX_NUMBER_SENDERS; i++) { if (dataPublishers[i] != nullptr) { PRINTOUT(F("\nSending data to ["), i, F("]"), dataPublishers[i]->getEndpoint()); - dataPublishers[i]->publishData(); + dataPublishers[i]->publishData(forceFlush); watchDogTimer.resetWatchDog(); } } @@ -1570,13 +1570,17 @@ void Logger::logDataAndPublish(bool sleepBeforeReturning) { // Create a csv data record and save it to the log file logToSD(); + // flush the publisher buffers (if any) if we have been invoked by the + // testing button + bool forceFlush = Logger::startTesting; + // Sync the clock at noon bool clockSyncNeeded = (Logger::markedLocalEpochTime != 0 && Logger::markedLocalEpochTime % 86400 == 43200) || !isRTCSane(Logger::markedLocalEpochTime); bool connectionNeeded = checkRemotesConnectionNeeded() || - clockSyncNeeded; + clockSyncNeeded || forceFlush; if (_logModem != nullptr && connectionNeeded) { MS_DBG(F("Waking up"), _logModem->getModemName(), F("...")); @@ -1587,7 +1591,7 @@ void Logger::logDataAndPublish(bool sleepBeforeReturning) { if (_logModem->connectInternet()) { // Publish data to remotes watchDogTimer.resetWatchDog(); - publishDataToRemotes(); + publishDataToRemotes(forceFlush); watchDogTimer.resetWatchDog(); if (clockSyncNeeded) { @@ -1614,7 +1618,7 @@ void Logger::logDataAndPublish(bool sleepBeforeReturning) { MS_DBG(F("Nobody needs it so publishing without connecting...")); // Call publish function without connection watchDogTimer.resetWatchDog(); - publishDataToRemotes(); + publishDataToRemotes(false); // can't flush without a connection watchDogTimer.resetWatchDog(); } diff --git a/src/LoggerBase.h b/src/LoggerBase.h index bd83bdc27..4c4b9420f 100644 --- a/src/LoggerBase.h +++ b/src/LoggerBase.h @@ -606,8 +606,10 @@ class Logger { bool checkRemotesConnectionNeeded(void); /** * @brief Publish data to all registered data publishers. + * + * @param forceFlush Ask the publishers to flush buffered data immediately. */ - void publishDataToRemotes(void); + void publishDataToRemotes(bool forceFlush = false); /** * @brief Retained for backwards compatibility, use publishDataToRemotes() * in new code. diff --git a/src/dataPublisherBase.cpp b/src/dataPublisherBase.cpp index a1c2111ad..0c5a0407c 100644 --- a/src/dataPublisherBase.cpp +++ b/src/dataPublisherBase.cpp @@ -158,12 +158,12 @@ bool dataPublisher::connectionNeeded(void) { } // This sends data on the "default" client of the modem -int16_t dataPublisher::publishData() { +int16_t dataPublisher::publishData(bool forceFlush) { if (_inClient == nullptr) { PRINTOUT(F("ERROR! No web client assigned to publish data!")); return 0; } else { - return publishData(_inClient); + return publishData(_inClient, forceFlush); } } // Duplicates for backwards compatibility diff --git a/src/dataPublisherBase.h b/src/dataPublisherBase.h index da6175944..0f3058305 100644 --- a/src/dataPublisherBase.h +++ b/src/dataPublisherBase.h @@ -189,10 +189,11 @@ class dataPublisher { * @param outClient An Arduino client instance to use to print data to. * Allows the use of any type of client and multiple clients tied to a * single TinyGSM modem instance + * @param forceFlush Ask the publisher to flush buffered data immediately. * @return **int16_t** The result of publishing data. May be an http * response code or a result code from PubSubClient. */ - virtual int16_t publishData(Client* outClient) = 0; + virtual int16_t publishData(Client* outClient, bool forceFlush = false) = 0; /** * @brief Open a socket to the correct receiver and send out the formatted * data. @@ -201,10 +202,12 @@ class dataPublisher { * either a client having been linked to the publisher or a logger modem * having been linked to the logger linked to the publisher. * + * @param forceFlush Ask the publisher to flush buffered data immediately. + * * @return **int16_t** The result of publishing data. May be an http * response code or a result code from PubSubClient. */ - virtual int16_t publishData(); + virtual int16_t publishData(bool forceFlush = false); /** * @brief Retained for backwards compatibility; use publishData(Client* diff --git a/src/publishers/DreamHostPublisher.cpp b/src/publishers/DreamHostPublisher.cpp index 40ffad715..d30b7cbab 100644 --- a/src/publishers/DreamHostPublisher.cpp +++ b/src/publishers/DreamHostPublisher.cpp @@ -65,7 +65,7 @@ void DreamHostPublisher::begin(Logger& baseLogger, const char* dhUrl) { // Post the data to dream host. // int16_t DreamHostPublisher::postDataDreamHost(void) -int16_t DreamHostPublisher::publishData(Client* outClient) { +int16_t DreamHostPublisher::publishData(Client* outClient, bool forceFlush) { // Create a buffer for the portions of the request and response char tempBuffer[37] = ""; uint16_t did_respond = 0; diff --git a/src/publishers/DreamHostPublisher.h b/src/publishers/DreamHostPublisher.h index ba88e4d11..34a5690ed 100644 --- a/src/publishers/DreamHostPublisher.h +++ b/src/publishers/DreamHostPublisher.h @@ -131,10 +131,11 @@ class DreamHostPublisher : public dataPublisher { * @param outClient An Arduino client instance to use to print data to. * Allows the use of any type of client and multiple clients tied to a * single TinyGSM modem instance + * @param forceFlush Ask the publisher to flush buffered data immediately. * * @return **int16_t** The http status code of the response. */ - int16_t publishData(Client* outClient) override; + int16_t publishData(Client* outClient, bool forceFlush = false) override; protected: // portions of the GET request diff --git a/src/publishers/EnviroDIYPublisher.cpp b/src/publishers/EnviroDIYPublisher.cpp index 8385f9502..702398c81 100644 --- a/src/publishers/EnviroDIYPublisher.cpp +++ b/src/publishers/EnviroDIYPublisher.cpp @@ -166,11 +166,12 @@ bool EnviroDIYPublisher::connectionNeeded(void) { // EnviroDIY/ODM2DataSharingPortal and then streams out a post request // over that connection. // The return is the http status code of the response. -int16_t EnviroDIYPublisher::publishData(Client* outClient) { - // do we intend to send this call? if so, we have just returned true from +int16_t EnviroDIYPublisher::publishData(Client* outClient, bool forceFlush) { + // do we intend to flush this call? if so, we have just returned true from // connectionNeeded() and the internet is connected and waiting. check what // that function said so we know to do it after we record this data point. - bool willFlush = connectionNeeded(); + // we also flush if requested (in which case the internet is connected too) + bool willFlush = connectionNeeded() || forceFlush; // create record to hold timestamp and variable values in the log buffer int record = _logBuffer.addRecord(Logger::markedLocalEpochTime); diff --git a/src/publishers/EnviroDIYPublisher.h b/src/publishers/EnviroDIYPublisher.h index c7d6e2f7e..1d12e0be3 100644 --- a/src/publishers/EnviroDIYPublisher.h +++ b/src/publishers/EnviroDIYPublisher.h @@ -166,9 +166,10 @@ class EnviroDIYPublisher : public dataPublisher { * @param outClient An Arduino client instance to use to print data to. * Allows the use of any type of client and multiple clients tied to a * single TinyGSM modem instance + * @param forceFlush Ask the publisher to flush buffered data immediately. * @return **int16_t** The http status code of the response. */ - int16_t publishData(Client* outClient) override; + int16_t publishData(Client* outClient, bool forceFlush = false) override; protected: /** diff --git a/src/publishers/ThingSpeakPublisher.cpp b/src/publishers/ThingSpeakPublisher.cpp index f13780260..efdf997dc 100644 --- a/src/publishers/ThingSpeakPublisher.cpp +++ b/src/publishers/ThingSpeakPublisher.cpp @@ -102,7 +102,7 @@ void ThingSpeakPublisher::begin(Logger& baseLogger, // This sends the data to ThingSpeak // bool ThingSpeakPublisher::mqttThingSpeak(void) -int16_t ThingSpeakPublisher::publishData(Client* outClient) { +int16_t ThingSpeakPublisher::publishData(Client* outClient, bool forceFlush) { bool retVal = false; // Make sure we don't have too many fields diff --git a/src/publishers/ThingSpeakPublisher.h b/src/publishers/ThingSpeakPublisher.h index 1a924b500..511c766fb 100644 --- a/src/publishers/ThingSpeakPublisher.h +++ b/src/publishers/ThingSpeakPublisher.h @@ -186,7 +186,7 @@ class ThingSpeakPublisher : public dataPublisher { // This sends the data to ThingSpeak // bool mqttThingSpeak(void); - int16_t publishData(Client* outClient) override; + int16_t publishData(Client* outClient, bool forceFlush = false) override; protected: /** diff --git a/src/publishers/UbidotsPublisher.cpp b/src/publishers/UbidotsPublisher.cpp index 4a111d631..702baafd7 100644 --- a/src/publishers/UbidotsPublisher.cpp +++ b/src/publishers/UbidotsPublisher.cpp @@ -113,7 +113,7 @@ void UbidotsPublisher::begin(Logger& baseLogger, // over that connection. // The return is the http status code of the response. // int16_t EnviroDIYPublisher::postDataEnviroDIY(void) -int16_t UbidotsPublisher::publishData(Client* outClient) { +int16_t UbidotsPublisher::publishData(Client* outClient, bool forceFlush) { // Create a buffer for the portions of the request and response char tempBuffer[37] = ""; uint16_t did_respond = 0; diff --git a/src/publishers/UbidotsPublisher.h b/src/publishers/UbidotsPublisher.h index c8926eb2e..61e335f05 100644 --- a/src/publishers/UbidotsPublisher.h +++ b/src/publishers/UbidotsPublisher.h @@ -164,9 +164,10 @@ class UbidotsPublisher : public dataPublisher { * @param outClient An Arduino client instance to use to print data to. * Allows the use of any type of client and multiple clients tied to a * single TinyGSM modem instance + * @param forceFlush Ask the publisher to flush buffered data immediately. * @return **int16_t** The http status code of the response. */ - int16_t publishData(Client* outClient) override; + int16_t publishData(Client* outClient, bool forceFlush) override; protected: /** From b98d3e421775d970bc6cb9d5ec3e5d229a6f5c4f Mon Sep 17 00:00:00 2001 From: Thomas Watson Date: Thu, 16 Mar 2023 14:43:23 -0500 Subject: [PATCH 14/23] libraries/ModularSensors: run clang-format on *.cpp and *.h --- src/LogBuffer.cpp | 4 ++-- src/LoggerBase.cpp | 15 +++++++++++---- src/publishers/EnviroDIYPublisher.cpp | 6 +++--- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/src/LogBuffer.cpp b/src/LogBuffer.cpp index 46c05ef56..b79b51edf 100644 --- a/src/LogBuffer.cpp +++ b/src/LogBuffer.cpp @@ -41,10 +41,10 @@ int LogBuffer::getNumRecords(void) { } uint8_t LogBuffer::getPercentFull(void) { - uint32_t bytesFull = (uint32_t)numRecords * (uint32_t)recordSize; + uint32_t bytesFull = (uint32_t)numRecords * (uint32_t)recordSize; uint32_t bytesTotal = MS_LOG_DATA_BUFFER_SIZE; - return (uint8_t)((bytesFull*(uint32_t)100)/bytesTotal); + return (uint8_t)((bytesFull * (uint32_t)100) / bytesTotal); } int LogBuffer::addRecord(uint32_t timestamp) { diff --git a/src/LoggerBase.cpp b/src/LoggerBase.cpp index 75740fe0f..a556cf719 100644 --- a/src/LoggerBase.cpp +++ b/src/LoggerBase.cpp @@ -596,11 +596,18 @@ void Logger::markTime(void) { bool Logger::checkInterval(void) { bool retval; uint32_t checkTime = getNowLocalEpoch(); + uint16_t interval = _loggingIntervalMinutes; + if (_initialShortIntervals > 0) { + // log the first few samples at an interval of 1 minute so that + // operation can be quickly verified in the field + _initialShortIntervals -= 1; + interval = 1; + } + MS_DBG(F("Current Unix Timestamp:"), checkTime, F("->"), formatDateTime_ISO8601(checkTime)); - MS_DBG(F("Logging interval in seconds:"), (_loggingIntervalMinutes * 60)); - MS_DBG(F("Mod of Logging Interval:"), - checkTime % (_loggingIntervalMinutes * 60)); + MS_DBG(F("Logging interval in seconds:"), (interval * 60)); + MS_DBG(F("Mod of Logging Interval:"), checkTime % (interval * 60)); if ((checkTime % (interval * 60) == 0)) { // Update the time variables with the current time @@ -1618,7 +1625,7 @@ void Logger::logDataAndPublish(bool sleepBeforeReturning) { MS_DBG(F("Nobody needs it so publishing without connecting...")); // Call publish function without connection watchDogTimer.resetWatchDog(); - publishDataToRemotes(false); // can't flush without a connection + publishDataToRemotes(false); // can't flush without a connection watchDogTimer.resetWatchDog(); } diff --git a/src/publishers/EnviroDIYPublisher.cpp b/src/publishers/EnviroDIYPublisher.cpp index 702398c81..9c53f33de 100644 --- a/src/publishers/EnviroDIYPublisher.cpp +++ b/src/publishers/EnviroDIYPublisher.cpp @@ -122,8 +122,8 @@ void EnviroDIYPublisher::begin(Logger& baseLogger, bool EnviroDIYPublisher::connectionNeeded(void) { // compute the send interval, reducing it as the buffer gets more full so we // have less of a chance of losing data - int interval = _sendEveryX; - uint8_t percent = _logBuffer.getPercentFull(); + int interval = _sendEveryX; + uint8_t percent = _logBuffer.getPercentFull(); if (percent >= 50) { interval /= 2; } else if (percent >= 75) { @@ -144,7 +144,7 @@ bool EnviroDIYPublisher::connectionNeeded(void) { if (relative == (interval - 1)) { // the next sample will put us right at the interval atSendInterval = true; - } else if (numRecords >= interval) { // don't send the first sample + } else if (numRecords >= interval) { // don't send the first sample if (relative == 0) { // the last sample was the interval, this is the first retry atSendInterval = true; From fa3aaabbcf46f77d5d41f40ed164c0836e4405f1 Mon Sep 17 00:00:00 2001 From: Sara Damiano Date: Fri, 15 Sep 2023 11:39:07 -0400 Subject: [PATCH 15/23] Added begin. Can't figure out why needed. Signed-off-by: Sara Damiano --- .../DRWI_Mayfly1_WiFi/DRWI_Mayfly1_WiFi.ino | 2 ++ src/LoggerBase.cpp | 8 ++++++++ src/publishers/EnviroDIYPublisher.cpp | 17 +++++++++++++---- 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/examples/DRWI_Mayfly1_WiFi/DRWI_Mayfly1_WiFi.ino b/examples/DRWI_Mayfly1_WiFi/DRWI_Mayfly1_WiFi.ino index 022cc0638..b36ae1245 100644 --- a/examples/DRWI_Mayfly1_WiFi/DRWI_Mayfly1_WiFi.ino +++ b/examples/DRWI_Mayfly1_WiFi/DRWI_Mayfly1_WiFi.ino @@ -306,6 +306,8 @@ void setup() { // Begin the logger dataLogger.begin(); + EnviroDIYPOST.begin(dataLogger, &modem.gsmClient, registrationToken, + samplingFeature); // Note: Please change these battery voltages to match your battery // Set up the sensors, except at lowest battery level diff --git a/src/LoggerBase.cpp b/src/LoggerBase.cpp index a556cf719..88745498e 100644 --- a/src/LoggerBase.cpp +++ b/src/LoggerBase.cpp @@ -1479,6 +1479,14 @@ void Logger::begin() { PRINTOUT(F("Sampling feature UUID is:"), _samplingFeatureUUID); } + + for (uint8_t i = 0; i < MAX_NUMBER_SENDERS; i++) { + if (dataPublishers[i] != nullptr) { + PRINTOUT(F("Data will be published to ["), i, F("]"), + dataPublishers[i]->getEndpoint()); + } + } + PRINTOUT(F("Logger portion of setup finished.\n")); } diff --git a/src/publishers/EnviroDIYPublisher.cpp b/src/publishers/EnviroDIYPublisher.cpp index 9c53f33de..7bbc64eee 100644 --- a/src/publishers/EnviroDIYPublisher.cpp +++ b/src/publishers/EnviroDIYPublisher.cpp @@ -35,10 +35,14 @@ const char* EnviroDIYPublisher::timestampTag = "\",\"timestamp\":["; // Constructors EnviroDIYPublisher::EnviroDIYPublisher() : dataPublisher() {} EnviroDIYPublisher::EnviroDIYPublisher(Logger& baseLogger, int sendEveryX) - : dataPublisher(baseLogger, sendEveryX) {} + : dataPublisher(baseLogger, sendEveryX) { + _logBuffer.setNumVariables(_baseLogger->getArrayVarCount()); +} EnviroDIYPublisher::EnviroDIYPublisher(Logger& baseLogger, Client* inClient, int sendEveryX) - : dataPublisher(baseLogger, inClient, sendEveryX) {} + : dataPublisher(baseLogger, inClient, sendEveryX) { + _logBuffer.setNumVariables(_baseLogger->getArrayVarCount()); +} EnviroDIYPublisher::EnviroDIYPublisher(Logger& baseLogger, const char* registrationToken, const char* samplingFeatureUUID, @@ -70,6 +74,10 @@ void EnviroDIYPublisher::setToken(const char* registrationToken) { uint16_t EnviroDIYPublisher::calculateJsonSize() { uint8_t variables = _logBuffer.getNumVariables(); int records = _logBuffer.getNumRecords(); + MS_DBG(F("Number of records in log buffer:"), records); + MS_DBG(F("Number of variables in log buffer:"), variables); + MS_DBG(F("Number of variables in base logger:"), + _baseLogger->getArrayVarCount()); uint16_t jsonLength = strlen(samplingFeatureTag); jsonLength += 36; // sampling feature UUID @@ -96,6 +104,7 @@ uint16_t EnviroDIYPublisher::calculateJsonSize() { } } jsonLength += 1; // } + MS_DBG(F("Outgoing JSON size:"), jsonLength); return jsonLength; } @@ -124,6 +133,7 @@ bool EnviroDIYPublisher::connectionNeeded(void) { // have less of a chance of losing data int interval = _sendEveryX; uint8_t percent = _logBuffer.getPercentFull(); + MS_DBG(F("Buffer is"), percent, F("percent full")); if (percent >= 50) { interval /= 2; } else if (percent >= 75) { @@ -172,6 +182,7 @@ int16_t EnviroDIYPublisher::publishData(Client* outClient, bool forceFlush) { // that function said so we know to do it after we record this data point. // we also flush if requested (in which case the internet is connected too) bool willFlush = connectionNeeded() || forceFlush; + MS_DBG(F("Publishing record to buffer. Will flush:"), willFlush); // create record to hold timestamp and variable values in the log buffer int record = _logBuffer.addRecord(Logger::markedLocalEpochTime); @@ -200,8 +211,6 @@ int16_t EnviroDIYPublisher::flushDataBuffer(Client* outClient) { char tempBuffer[37] = ""; uint16_t did_respond = 0; - MS_DBG(F("Outgoing JSON size:"), calculateJsonSize()); - // Open a TCP/IP connection to the Enviro DIY Data Portal (WebSDL) MS_DBG(F("Connecting client")); MS_START_DEBUG_TIMER; From 81b6963e7dd743f4b81b1ea67cb15b5eae0480cc Mon Sep 17 00:00:00 2001 From: Sara Damiano Date: Thu, 21 Sep 2023 12:13:21 -0400 Subject: [PATCH 16/23] Non-static buffer Signed-off-by: Sara Damiano --- examples/DRWI_Mayfly1_WiFi/DRWI_Mayfly1_WiFi.ino | 2 -- src/publishers/EnviroDIYPublisher.cpp | 2 -- src/publishers/EnviroDIYPublisher.h | 2 +- 3 files changed, 1 insertion(+), 5 deletions(-) diff --git a/examples/DRWI_Mayfly1_WiFi/DRWI_Mayfly1_WiFi.ino b/examples/DRWI_Mayfly1_WiFi/DRWI_Mayfly1_WiFi.ino index b36ae1245..022cc0638 100644 --- a/examples/DRWI_Mayfly1_WiFi/DRWI_Mayfly1_WiFi.ino +++ b/examples/DRWI_Mayfly1_WiFi/DRWI_Mayfly1_WiFi.ino @@ -306,8 +306,6 @@ void setup() { // Begin the logger dataLogger.begin(); - EnviroDIYPOST.begin(dataLogger, &modem.gsmClient, registrationToken, - samplingFeature); // Note: Please change these battery voltages to match your battery // Set up the sensors, except at lowest battery level diff --git a/src/publishers/EnviroDIYPublisher.cpp b/src/publishers/EnviroDIYPublisher.cpp index 7bbc64eee..a8ca2c433 100644 --- a/src/publishers/EnviroDIYPublisher.cpp +++ b/src/publishers/EnviroDIYPublisher.cpp @@ -11,8 +11,6 @@ #include "EnviroDIYPublisher.h" -LogBuffer EnviroDIYPublisher::_logBuffer; - // ============================================================================ // Functions for the EnviroDIY data portal receivers. // ============================================================================ diff --git a/src/publishers/EnviroDIYPublisher.h b/src/publishers/EnviroDIYPublisher.h index 1d12e0be3..28e1c4555 100644 --- a/src/publishers/EnviroDIYPublisher.h +++ b/src/publishers/EnviroDIYPublisher.h @@ -196,7 +196,7 @@ class EnviroDIYPublisher : public dataPublisher { static const char* timestampTag; ///< The JSON feature timestamp tag /**@}*/ - static LogBuffer _logBuffer; + LogBuffer _logBuffer; // actually transmit rather than just buffer data int16_t flushDataBuffer(Client* outClient); From ba35c843e1dddfa45b4eaa5461a8a7edaa1db3f0 Mon Sep 17 00:00:00 2001 From: Sara Damiano Date: Thu, 21 Sep 2023 12:13:32 -0400 Subject: [PATCH 17/23] steps toward fifo Signed-off-by: Sara Damiano --- src/LogBuffer.cpp | 9 +++++++-- src/LogBuffer.h | 15 ++++++++++++++- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/src/LogBuffer.cpp b/src/LogBuffer.cpp index b79b51edf..ca4eab926 100644 --- a/src/LogBuffer.cpp +++ b/src/LogBuffer.cpp @@ -24,12 +24,15 @@ void LogBuffer::setNumVariables(uint8_t numVariables_) { numVariables = numVariables_; // this scrambles all the data in the buffer so clear it out - numRecords = 0; + clear(); } void LogBuffer::clear(void) { // clear out the buffer - numRecords = 0; + numRecords = 0; + dataBufferTail = 0; + dataBufferHead = 0; + _bufferOverflow = false; } uint8_t LogBuffer::getNumVariables(void) { @@ -48,6 +51,7 @@ uint8_t LogBuffer::getPercentFull(void) { } int LogBuffer::addRecord(uint32_t timestamp) { + // check how many records currently exist int record = numRecords; // compute position of the new record's timestamp in the buffer // (the timestamp is the first data in the record) @@ -60,6 +64,7 @@ int LogBuffer::addRecord(uint32_t timestamp) { sizeof(uint32_t)); numRecords += 1; // just added another record + // return the index of the record number just created return record; } diff --git a/src/LogBuffer.h b/src/LogBuffer.h index b4b241a53..756ac3d8e 100644 --- a/src/LogBuffer.h +++ b/src/LogBuffer.h @@ -111,7 +111,7 @@ class LogBuffer { uint32_t getRecordTimestamp(int record); /** - * @brief Gets the value of a particular vaiable in a particular record. + * @brief Gets the value of a particular variable in a particular record. * * @param[in] record The record * @param[in] variable The variable @@ -126,6 +126,19 @@ class LogBuffer { */ uint8_t dataBuffer[MS_LOG_DATA_BUFFER_SIZE]; + /** + * @brief Index of buffer head. + */ + uint16_t dataBufferTail; + /** + * @brief Index of buffer tail. + */ + uint16_t dataBufferHead; + /** + * @brief The buffer overflow status + */ + bool _bufferOverflow = false; + /** * @brief Number of records currently in the buffer. */ From 54bc18a7d5f34722ab16da7511808e4f9f074d6a Mon Sep 17 00:00:00 2001 From: Sara Damiano Date: Mon, 20 May 2024 14:39:23 -0400 Subject: [PATCH 18/23] Made the EnviroDIY host/path/port settable Signed-off-by: Sara Damiano --- src/publishers/EnviroDIYPublisher.cpp | 54 ++++++++++++++++++++++++--- src/publishers/EnviroDIYPublisher.h | 48 ++++++++++++++++++++++-- 2 files changed, 93 insertions(+), 9 deletions(-) diff --git a/src/publishers/EnviroDIYPublisher.cpp b/src/publishers/EnviroDIYPublisher.cpp index a8ca2c433..baee43954 100644 --- a/src/publishers/EnviroDIYPublisher.cpp +++ b/src/publishers/EnviroDIYPublisher.cpp @@ -18,9 +18,6 @@ // Constant values for post requests // I want to refer to these more than once while ensuring there is only one copy // in memory -const char* EnviroDIYPublisher::postEndpoint = "/api/data-stream/"; -const char* EnviroDIYPublisher::enviroDIYHost = "data.envirodiy.org"; -const int EnviroDIYPublisher::enviroDIYPort = 80; const char* EnviroDIYPublisher::tokenHeader = "\r\nTOKEN: "; const char* EnviroDIYPublisher::contentLengthHeader = "\r\nContent-Length: "; const char* EnviroDIYPublisher::contentTypeHeader = @@ -31,15 +28,25 @@ const char* EnviroDIYPublisher::timestampTag = "\",\"timestamp\":["; // Constructors -EnviroDIYPublisher::EnviroDIYPublisher() : dataPublisher() {} +EnviroDIYPublisher::EnviroDIYPublisher() : dataPublisher() { + setHost("monitormywatershed.org"); + setPath("/api/data-stream/"); + setPort(80); +} EnviroDIYPublisher::EnviroDIYPublisher(Logger& baseLogger, int sendEveryX) : dataPublisher(baseLogger, sendEveryX) { _logBuffer.setNumVariables(_baseLogger->getArrayVarCount()); + setHost("monitormywatershed.org"); + setPath("/api/data-stream/"); + setPort(80); } EnviroDIYPublisher::EnviroDIYPublisher(Logger& baseLogger, Client* inClient, int sendEveryX) : dataPublisher(baseLogger, inClient, sendEveryX) { _logBuffer.setNumVariables(_baseLogger->getArrayVarCount()); + setHost("monitormywatershed.org"); + setPath("/api/data-stream/"); + setPort(80); } EnviroDIYPublisher::EnviroDIYPublisher(Logger& baseLogger, const char* registrationToken, @@ -49,6 +56,9 @@ EnviroDIYPublisher::EnviroDIYPublisher(Logger& baseLogger, setToken(registrationToken); _baseLogger->setSamplingFeatureUUID(samplingFeatureUUID); _logBuffer.setNumVariables(_baseLogger->getArrayVarCount()); + setHost("monitormywatershed.org"); + setPath("/api/data-stream/"); + setPort(80); } EnviroDIYPublisher::EnviroDIYPublisher(Logger& baseLogger, Client* inClient, const char* registrationToken, @@ -58,11 +68,45 @@ EnviroDIYPublisher::EnviroDIYPublisher(Logger& baseLogger, Client* inClient, setToken(registrationToken); _baseLogger->setSamplingFeatureUUID(samplingFeatureUUID); _logBuffer.setNumVariables(_baseLogger->getArrayVarCount()); + setHost("monitormywatershed.org"); + setPath("/api/data-stream/"); + setPort(80); } // Destructor EnviroDIYPublisher::~EnviroDIYPublisher() {} +// Returns the data destination +String EnviroDIYPublisher::getHost(void) { + return String(enviroDIYHost); +} + +// Returns the data destination +void EnviroDIYPublisher::setHost(const char* host) { + enviroDIYHost = host; +} + +// Returns the data destination +String EnviroDIYPublisher::getPath(void) { + return String(enviroDIYPath); +} + +// Returns the data destination +void EnviroDIYPublisher::setPath(const char* endpoint) { + enviroDIYPath = endpoint; +} + +// Returns the data destination +int EnviroDIYPublisher::getPort(void) { + return enviroDIYPort; +} + +// Returns the data destination +void EnviroDIYPublisher::setPort(int port) { + enviroDIYPort = port; +} + + void EnviroDIYPublisher::setToken(const char* registrationToken) { _registrationToken = registrationToken; } @@ -218,7 +262,7 @@ int16_t EnviroDIYPublisher::flushDataBuffer(Client* outClient) { // copy the initial post header into the tx buffer txBufferAppend(postHeader); - txBufferAppend(postEndpoint); + txBufferAppend(enviroDIYPath); txBufferAppend(HTTPtag); // add the rest of the HTTP POST headers to the outgoing buffer diff --git a/src/publishers/EnviroDIYPublisher.h b/src/publishers/EnviroDIYPublisher.h index 28e1c4555..025dc80ad 100644 --- a/src/publishers/EnviroDIYPublisher.h +++ b/src/publishers/EnviroDIYPublisher.h @@ -107,9 +107,49 @@ class EnviroDIYPublisher : public dataPublisher { // Returns the data destination String getEndpoint(void) override { - return String(enviroDIYHost); + return String(enviroDIYHost) + String(enviroDIYPath); } + /** + * @brief Get the EnviroDIY/Monitor My Watershed web host + * + * @return *String* The EnviroDIY/Monitor My Watershed web host + */ + String getHost(void); + + /** + * @brief Set the EnviroDIY/Monitor My Watershed web host + * + * @param host The EnviroDIY/Monitor My Watershed web host + */ + void setHost(const char* host); + + /** + * @brief Get the EnviroDIY/Monitor My Watershed API path + * + * @return *String* The EnviroDIY/Monitor My Watershed API path + */ + String getPath(void); + /** + * @brief Set the EnviroDIY/Monitor My Watershed API path + * + * @param endpoint The EnviroDIY/Monitor My Watershed API path + */ + void setPath(const char* endpoint); + + /** + * @brief Get the EnviroDIY/Monitor My Watershed API port + * + * @return *int* The EnviroDIY/Monitor My Watershed API port + */ + int getPort(void); + /** + * @brief Set the EnviroDIY/Monitor My Watershed API port + * + * @param port The EnviroDIY/Monitor My Watershed API port + */ + void setPort(int port); + // Adds the site registration token /** * @brief Set the site registration token @@ -178,9 +218,9 @@ class EnviroDIYPublisher : public dataPublisher { * * @{ */ - static const char* postEndpoint; ///< The endpoint - static const char* enviroDIYHost; ///< The host name - static const int enviroDIYPort; ///< The host port + const char* enviroDIYPath; ///< The api path + const char* enviroDIYHost; ///< The host name + int enviroDIYPort; ///< The host port static const char* tokenHeader; ///< The token header text static const char* contentLengthHeader; ///< The content length header text static const char* contentTypeHeader; ///< The content type header text From 7a9f6e6c8888a675437cac2a2b4ea20cbd4d1f02 Mon Sep 17 00:00:00 2001 From: Sara Damiano Date: Mon, 20 May 2024 14:45:39 -0400 Subject: [PATCH 19/23] CR Signed-off-by: Sara Damiano --- src/publishers/EnviroDIYPublisher.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/publishers/EnviroDIYPublisher.cpp b/src/publishers/EnviroDIYPublisher.cpp index baee43954..457c8f3a4 100644 --- a/src/publishers/EnviroDIYPublisher.cpp +++ b/src/publishers/EnviroDIYPublisher.cpp @@ -11,6 +11,7 @@ #include "EnviroDIYPublisher.h" + // ============================================================================ // Functions for the EnviroDIY data portal receivers. // ============================================================================ From 2b97787290655a0eb0d6324f0adbb65d1b8409b6 Mon Sep 17 00:00:00 2001 From: Sara Damiano Date: Mon, 20 May 2024 15:30:40 -0400 Subject: [PATCH 20/23] Re-add testing mode Signed-off-by: Sara Damiano --- ChangeLog.md | 3 +++ src/LoggerBase.cpp | 14 ++++++++++++-- src/LoggerBase.h | 23 +++++++++++++++++++++++ 3 files changed, 38 insertions(+), 2 deletions(-) diff --git a/ChangeLog.md b/ChangeLog.md index f28d4a38e..e378f344e 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -13,6 +13,9 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm ### Changed ### Added +- Added support for caching readings in RAM and sending in batches. +This currently only works on the EnviroDIY/Monitor My Watershed Publisher. +Thank you to [Thomas Watson](https://github.com/tpwrules) for this work. ### Removed diff --git a/src/LoggerBase.cpp b/src/LoggerBase.cpp index 88745498e..a556a2bc7 100644 --- a/src/LoggerBase.cpp +++ b/src/LoggerBase.cpp @@ -32,6 +32,7 @@ uint32_t Logger::markedLocalEpochTime = 0; uint32_t Logger::markedUTCEpochTime = 0; // Initialize the testing/logging flags volatile bool Logger::isLoggingNow = false; +volatile bool Logger::isTestingNow = false; volatile bool Logger::startTesting = false; // Initialize the RTC for the SAMD boards using build in RTC @@ -53,6 +54,7 @@ Logger::Logger(const char* loggerID, uint16_t loggingIntervalMinutes, // Set the testing/logging flags to false isLoggingNow = false; + isTestingNow = false; startTesting = false; // Clear arrays @@ -69,6 +71,7 @@ Logger::Logger(const char* loggerID, uint16_t loggingIntervalMinutes, // Set the testing/logging flags to false isLoggingNow = false; + isTestingNow = false; startTesting = false; // Clear arrays @@ -79,6 +82,7 @@ Logger::Logger(const char* loggerID, uint16_t loggingIntervalMinutes, Logger::Logger() { // Set the testing/logging flags to false isLoggingNow = false; + isTestingNow = false; startTesting = false; // Clear arrays @@ -1288,7 +1292,7 @@ bool Logger::logToSD(void) { // A static function if you'd prefer to enter testing based on an interrupt void Logger::testingISR() { MS_DEEP_DBG(F("Testing interrupt!")); - if (!Logger::isLoggingNow) { + if (!Logger::isTestingNow && !Logger::isLoggingNow) { Logger::startTesting = true; MS_DEEP_DBG(F("Testing flag has been set.")); } @@ -1537,6 +1541,9 @@ void Logger::logData(bool sleepBeforeReturning) { Logger::startTesting = false; } + // Check if it was instead the testing interrupt that woke us up + if (Logger::startTesting) testingMode(); + if (sleepBeforeReturning) { // Sleep systemSleep(); @@ -1656,8 +1663,11 @@ void Logger::logDataAndPublish(bool sleepBeforeReturning) { Logger::startTesting = false; } + // Check if it was instead the testing interrupt that woke us up + if (Logger::startTesting) testingMode(); + if (sleepBeforeReturning) { - // Sleep + // Call the processor sleep systemSleep(); } } diff --git a/src/LoggerBase.h b/src/LoggerBase.h index 4c4b9420f..b293fcf17 100644 --- a/src/LoggerBase.h +++ b/src/LoggerBase.h @@ -399,6 +399,11 @@ class Logger { * @brief The logging interval in minutes */ uint16_t _loggingIntervalMinutes = 5; + /** + * @brief The initial number of samples to log at an interval of 1 minute + * for fast field verification + */ + uint8_t _initialShortIntervals = 0; /** * @brief Digital pin number on the mcu controlling the SD card slave * select. @@ -1202,6 +1207,19 @@ class Logger { * on the pin assigned for "testing" mode. */ static void testingISR(void); + + /** + * @brief Execute testing mode. + * + * In testing mode, the logger uses the loggerModem, if attached, to connect + * to the internet. It then powers up all sensors tied to variable in the + * internal variable array. The logger then updates readings from all + * sensors 25 times with a 5 second wait in between. All results are output + * to the "main" output - ie Serial - and NOT to the SD card. After 25 + * measurements, the sensors are put to sleep, the modem is disconnected + * from the internet, and the logger goes back to sleep. + */ + virtual void testingMode(); /**@}*/ // ===================================================================== // @@ -1285,6 +1303,11 @@ class Logger { * sensors or writing to the SD card */ static volatile bool isLoggingNow; + /** + * @brief Internal flag set to true when the logger is going through the + * "testing mode" routine. + */ + static volatile bool isTestingNow; /** * @brief Internal flag set to true with then logger should begin the * "testing mode" routine when it finishes other operations. From 055c11a538d2db72f708e4594c924aad123f2c66 Mon Sep 17 00:00:00 2001 From: Sara Damiano Date: Tue, 28 May 2024 15:21:14 -0400 Subject: [PATCH 21/23] Decrease buffer size for Mega, fix warnings Signed-off-by: Sara Damiano --- src/LogBuffer.h | 4 ++++ src/publishers/DreamHostPublisher.cpp | 2 +- src/publishers/ThingSpeakPublisher.cpp | 2 +- src/publishers/UbidotsPublisher.cpp | 2 +- 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/LogBuffer.h b/src/LogBuffer.h index 756ac3d8e..9d3151256 100644 --- a/src/LogBuffer.h +++ b/src/LogBuffer.h @@ -26,8 +26,12 @@ * compiling. 8192 bytes is a safe value for the Mayfly 1.1 with six variables. */ #ifndef MS_LOG_DATA_BUFFER_SIZE +#ifdef ARDUINO_AVR_MEGA2560 +#define MS_LOG_DATA_BUFFER_SIZE 2048 +#else #define MS_LOG_DATA_BUFFER_SIZE 8192 #endif +#endif #include #include diff --git a/src/publishers/DreamHostPublisher.cpp b/src/publishers/DreamHostPublisher.cpp index d30b7cbab..9089028ac 100644 --- a/src/publishers/DreamHostPublisher.cpp +++ b/src/publishers/DreamHostPublisher.cpp @@ -65,7 +65,7 @@ void DreamHostPublisher::begin(Logger& baseLogger, const char* dhUrl) { // Post the data to dream host. // int16_t DreamHostPublisher::postDataDreamHost(void) -int16_t DreamHostPublisher::publishData(Client* outClient, bool forceFlush) { +int16_t DreamHostPublisher::publishData(Client* outClient, bool) { // Create a buffer for the portions of the request and response char tempBuffer[37] = ""; uint16_t did_respond = 0; diff --git a/src/publishers/ThingSpeakPublisher.cpp b/src/publishers/ThingSpeakPublisher.cpp index efdf997dc..3b070088a 100644 --- a/src/publishers/ThingSpeakPublisher.cpp +++ b/src/publishers/ThingSpeakPublisher.cpp @@ -102,7 +102,7 @@ void ThingSpeakPublisher::begin(Logger& baseLogger, // This sends the data to ThingSpeak // bool ThingSpeakPublisher::mqttThingSpeak(void) -int16_t ThingSpeakPublisher::publishData(Client* outClient, bool forceFlush) { +int16_t ThingSpeakPublisher::publishData(Client* outClient, bool ) { bool retVal = false; // Make sure we don't have too many fields diff --git a/src/publishers/UbidotsPublisher.cpp b/src/publishers/UbidotsPublisher.cpp index 702baafd7..33f2993f9 100644 --- a/src/publishers/UbidotsPublisher.cpp +++ b/src/publishers/UbidotsPublisher.cpp @@ -113,7 +113,7 @@ void UbidotsPublisher::begin(Logger& baseLogger, // over that connection. // The return is the http status code of the response. // int16_t EnviroDIYPublisher::postDataEnviroDIY(void) -int16_t UbidotsPublisher::publishData(Client* outClient, bool forceFlush) { +int16_t UbidotsPublisher::publishData(Client* outClient, bool) { // Create a buffer for the portions of the request and response char tempBuffer[37] = ""; uint16_t did_respond = 0; From c2f646c405e8949262578b422f23ce8f6f2eb1fb Mon Sep 17 00:00:00 2001 From: Sara Damiano Date: Tue, 11 Jun 2024 13:14:25 -0400 Subject: [PATCH 22/23] Update Doxygen, add docs for private members Signed-off-by: Sara Damiano --- src/LogBuffer.h | 41 ++++++++++++++--------------- src/publishers/EnviroDIYPublisher.h | 25 +++++++++++++++--- 2 files changed, 41 insertions(+), 25 deletions(-) diff --git a/src/LogBuffer.h b/src/LogBuffer.h index 9d3151256..7186024a6 100644 --- a/src/LogBuffer.h +++ b/src/LogBuffer.h @@ -13,8 +13,11 @@ #ifndef SRC_LOGBUFFER_H_ #define SRC_LOGBUFFER_H_ +#ifndef MS_LOG_DATA_BUFFER_SIZE +#ifdef ARDUINO_AVR_MEGA2560 +#define MS_LOG_DATA_BUFFER_SIZE 2048 +#else /** - * @def MS_LOG_DATA_BUFFER_SIZE * @brief Log Data Buffer * * This determines how much RAM is reserved to buffer log records before @@ -25,10 +28,6 @@ * This can be changed by setting the build flag MS_LOG_DATA_BUFFER_SIZE when * compiling. 8192 bytes is a safe value for the Mayfly 1.1 with six variables. */ -#ifndef MS_LOG_DATA_BUFFER_SIZE -#ifdef ARDUINO_AVR_MEGA2560 -#define MS_LOG_DATA_BUFFER_SIZE 2048 -#else #define MS_LOG_DATA_BUFFER_SIZE 8192 #endif #endif @@ -57,14 +56,14 @@ class LogBuffer { * @brief Sets the number of variables the buffer will store in each record. * Clears the buffer as a side effect. * - * @param[in] numVariables_ The number of variables to store. + * @param numVariables_ The number of variables to store. */ void setNumVariables(uint8_t numVariables_); /** * @brief Gets the number of variables that will be stored in each record. * - * @return The variable count. + * @return The variable count. */ uint8_t getNumVariables(void); @@ -74,53 +73,53 @@ class LogBuffer { void clear(void); /** - * @brief Gets the number of records currently in the log. + * @brief Gets the number of records currently in the log. * - * @return The number of records. + * @return The number of records. */ int getNumRecords(void); /** - * @brief Computes the percentage full of the buffer. + * @brief Computes the percentage full of the buffer. * - * @return The current percent full. + * @return The current percent full. */ uint8_t getPercentFull(void); /** * @brief Adds a new record with the given timestamp. * - * @param[in] timestamp The timestamp + * @param timestamp The timestamp * - * @return Index of the new record, or -1 if there was no space. + * @return Index of the new record, or -1 if there was no space. */ int addRecord(uint32_t timestamp); /** * @brief Sets the value of a particular variable in a particular record. * - * @param[in] record The record - * @param[in] variable The variable - * @param[in] value The value + * @param record The record + * @param variable The variable + * @param value The value */ void setRecordValue(int record, uint8_t variable, float value); /** * @brief Gets the timestamp of a particular record. * - * @param[in] record The record + * @param record The record * - * @return The record's timestamp. + * @return **uint32_t** The record's timestamp. */ uint32_t getRecordTimestamp(int record); /** * @brief Gets the value of a particular variable in a particular record. * - * @param[in] record The record - * @param[in] variable The variable + * @param record The record + * @param variable The variable * - * @return The variable's value. + * @return **float** The variable's value. */ float getRecordValue(int record, uint8_t variable); diff --git a/src/publishers/EnviroDIYPublisher.h b/src/publishers/EnviroDIYPublisher.h index 025dc80ad..f499daec4 100644 --- a/src/publishers/EnviroDIYPublisher.h +++ b/src/publishers/EnviroDIYPublisher.h @@ -236,17 +236,34 @@ class EnviroDIYPublisher : public dataPublisher { static const char* timestampTag; ///< The JSON feature timestamp tag /**@}*/ - LogBuffer _logBuffer; + + LogBuffer _logBuffer; ///< Internal reference to the logger buffer // actually transmit rather than just buffer data + /** + * @brief Transmit data from the data buffer to an external site + * + * @param outClient The client to publish the data over + * @return **int16_t** The HTTP response code from the publish attempt + * + * @note A 504 will be returned automatically if the server does not + * respond within 30 seconds. + */ int16_t flushDataBuffer(Client* outClient); - // we send every one of the first five data points immediately for field - // validation + /** + * @brief The number of transmissions remaing at the single minute intervals + * + * We send every one of the first five data points at only one minute + * intervals for faster in-field validation. + */ uint8_t _initialTransmissionsRemaining = 5; private: - // Tokens and UUID's for EnviroDIY + /** + * @brief Internal reference to the EnviroDIY/Monitor My Watershed + * registration token. + */ const char* _registrationToken = nullptr; }; From c3a1520907aa38ccc189166b0bb1921e6c2f9ad9 Mon Sep 17 00:00:00 2001 From: Sara Damiano Date: Thu, 19 Sep 2024 16:36:15 -0400 Subject: [PATCH 23/23] Remove extra emphasized return types. Signed-off-by: Sara Damiano --- src/LogBuffer.h | 4 ++-- src/publishers/EnviroDIYPublisher.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/LogBuffer.h b/src/LogBuffer.h index 7186024a6..c648dfd3e 100644 --- a/src/LogBuffer.h +++ b/src/LogBuffer.h @@ -109,7 +109,7 @@ class LogBuffer { * * @param record The record * - * @return **uint32_t** The record's timestamp. + * @return The record's timestamp. */ uint32_t getRecordTimestamp(int record); @@ -119,7 +119,7 @@ class LogBuffer { * @param record The record * @param variable The variable * - * @return **float** The variable's value. + * @return The variable's value. */ float getRecordValue(int record, uint8_t variable); diff --git a/src/publishers/EnviroDIYPublisher.h b/src/publishers/EnviroDIYPublisher.h index f499daec4..c54291253 100644 --- a/src/publishers/EnviroDIYPublisher.h +++ b/src/publishers/EnviroDIYPublisher.h @@ -244,7 +244,7 @@ class EnviroDIYPublisher : public dataPublisher { * @brief Transmit data from the data buffer to an external site * * @param outClient The client to publish the data over - * @return **int16_t** The HTTP response code from the publish attempt + * @return The HTTP response code from the publish attempt * * @note A 504 will be returned automatically if the server does not * respond within 30 seconds.