0

So I've recently started a project where I am using an accelerometer, along with a SD card breakout board. I've been able to get information to write properly to the SD card with no issues whatsoever. However, my speed is a bit lower than I would like, at about only 40Hz or so, where my accelerometer can log data at 260Hz (MPU-6050). Each reading contains an X, Y, Z, and timestamp value since recording started. Each reading is output into a .txt file for later separation via comma-separated-value sorting in Excel(I wish) or GNUplot, which handles the massive sample size much better.

I've referenced https://forum.arduino.cc/t/how-to-write-data-with-high-sampling-rates-to-a-sd-card/281496/9 to try and figure this out, but I couldn't get much out of it. I don't understand arrays very well beyond the fact they are just tables like in excel, and can be set up in 1, 2, 3, or more dimensions.

I've read around a bit and it seems my problem is that I'm closing and opening the file each and every write cycle. However, I don't know how to get out of doing this as the program seems to not run properly if I don't close out the file or if I use flush. My general idea of how this could flow without opening/close cycles every single time would be:

  1. Create arrays for each variable (X, Y, Z, elapsedTime)
  2. Take all my recordings for a given time, maybe five-hundred readings or so which would come out to about 12 seconds, or whatever possible value I can squeeze out of my Atmega328P and maintain program stability, and shove them into their respective arrays. Ideally the longer the better, but I understand I have memory constraints.
  3. In one write-cycle, I write all the values into CSV form, or even multiple print cycles cycling through each array cell continuously until all the values are input with the proper format.
  4. Clear the arrays of all the info they held, and return to step 2.

Really appreciate any help you guys can offer. Thank you!

//

#include <Adafruit_MPU6050.h> #include <Adafruit_Sensor.h> #include <Wire.h> #include <SPI.h> #include <SD.h> int recordStatus = false; //Default recordStatus is zero. int recordingLed = 3; const int chipSelect = 10; Adafruit_MPU6050 mpu;

void setup(void) { pinMode(2, INPUT_PULLUP); pinMode(3, OUTPUT); Serial.begin(500000); while (!Serial) delay(10); // will pause Zero, Leonardo, etc until serial console opens

                                                                       //SD CARD WRITE CODE BEGIN                                                                              

{ while (!Serial) { ; // wait for serial port to connect. Needed for native USB port only }

Serial.print("Initializing SD card...");

// see if the card is present and can be initialized: if (!SD.begin(chipSelect)) { Serial.println("Card failed, or not present"); // don't do anything more: while (1); } Serial.println("card initialized."); } //SD CARD WRITE CODE END

Serial.println("Adafruit MPU6050 test!"); //MPU6050 ACCELEROMETER INITIALIZATION CODE BEGIN // Try to initialize! if (!mpu.begin()) { Serial.println("Failed to find MPU6050 chip"); while (1) { delay(10); } } Serial.println("MPU6050 Found!");

mpu.setAccelerometerRange(MPU6050_RANGE_8_G); switch (mpu.getAccelerometerRange()) { case MPU6050_RANGE_2_G: break; case MPU6050_RANGE_4_G: break; case MPU6050_RANGE_8_G: break; case MPU6050_RANGE_16_G: break; } mpu.setGyroRange(MPU6050_RANGE_500_DEG); switch (mpu.getGyroRange()) { case MPU6050_RANGE_250_DEG: break; case MPU6050_RANGE_500_DEG: break; case MPU6050_RANGE_1000_DEG: break; case MPU6050_RANGE_2000_DEG: break; }

mpu.setFilterBandwidth(MPU6050_BAND_260_HZ); Serial.print("Filter bandwidth set to: "); switch (mpu.getFilterBandwidth()) { case MPU6050_BAND_260_HZ: Serial.println("260 Hz"); break; case MPU6050_BAND_184_HZ: Serial.println("184 Hz"); break; case MPU6050_BAND_94_HZ: Serial.println("94 Hz"); break; case MPU6050_BAND_44_HZ: Serial.println("44 Hz"); break; case MPU6050_BAND_21_HZ: Serial.println("21 Hz"); break; case MPU6050_BAND_10_HZ: Serial.println("10 Hz"); break; case MPU6050_BAND_5_HZ: Serial.println("5 Hz"); break; } //MPU6050 ACCELEROMETER INITIALIZATION CODE END }

unsigned long startMillis; unsigned long currentMillis; unsigned long elapsedTime; int rawMillis; void loop() { digitalWrite(recordingLed, LOW); int buttonRecord = digitalRead(2); //Check button for GND input /* Get new sensor events with the readings */ sensors_event_t a, g, temp; mpu.getEvent(&a, &g, &temp);

// make a string for assembling the data to log: //SD CARD VALUES ASSIGN BEGIN int X = ""; int Y = ""; int Z = ""; String timeofRecord = String(millis()/1000.0, 10); X = (a.acceleration.x/9.81); Y = (a.acceleration.y/9.81); Z = (a.acceleration.z/9.81); if(buttonRecord == false){ //Toggle record status upon pin 2 going to GND delay(500); recordStatus = !recordStatus; //If button is pressed, make recordStatus switch from true to false. startMillis = millis(); }

if(recordStatus == true){ //While record is toggled, do below rawMillis = millis()-startMillis; String elapsedTime = String(rawMillis/1000.00, 3); File dataFile = SD.open("datalog.txt", FILE_WRITE); if (dataFile) { // if the file is available, write to it: digitalWrite(recordingLed, HIGH); dataFile.print(elapsedTime); dataFile.print(", "); dataFile.print(X); dataFile.print(", "); dataFile.print(Y); dataFile.print(", "); dataFile.print(Z); dataFile.println(""); dataFile.close(); } } // if the file isn't open, pop up an error: else { Serial.println("error opening datalog.txt"); } }

UPDATE

So I've been messing with this a while, and I believe I just about have it figured out. I have arrays set up for time, X, Y, and Z and they are populating values correctly. However, I can't seem to get into my SD card file. There is no issue when the setup initializes, it's like it skips the

if (dataFile){

entirely. The modified code is below here. There's no shutdowns, and the program loops continuously so I don't think I'm running out of memory or having any power issues. The red LED on the SD card is flashing as though it's being written. Even if I remove the if(dataFile) check and force the commands contained in the if statement to execute, my txt file is still left empty. Previously I was writing to this just fine. Memory problem? I have less than 500 bytes of memory available. Possibly this is the cause? Or is it the for loop causing my trouble?

#include <Adafruit_MPU6050.h>
#include <Adafruit_Sensor.h>
#include <Wire.h>
#include <SPI.h>
#include <SD.h>
int recordStatus = false;                                                   //Default recordStatus is zero.
int recordingLed = 3; 
const int chipSelect = 10;
Adafruit_MPU6050 mpu;

void setup(void) { pinMode(2, INPUT_PULLUP); pinMode(3, OUTPUT); Serial.begin(500000); while (!Serial) delay(10);
//SD CARD WRITE CODE BEGIN
{ Serial.print("Initializing SD card...");

// see if the card is present and can be initialized: if (!SD.begin(chipSelect)) { Serial.println("Card failed, or not present"); // don't do anything more: while (1); } Serial.println("card initialized."); } //SD CARD WRITE CODE END

Serial.println("Adafruit MPU6050 test!"); //MPU6050 ACCELEROMETER INITIALIZATION CODE BEGIN // Try to initialize! if (!mpu.begin()) { Serial.println("Failed to find MPU6050 chip"); while (1) { delay(10); } } Serial.println("MPU6050 Found!");

mpu.setAccelerometerRange(MPU6050_RANGE_4_G); switch (mpu.getAccelerometerRange()) { case MPU6050_RANGE_2_G: break; case MPU6050_RANGE_4_G: break; case MPU6050_RANGE_8_G: break; case MPU6050_RANGE_16_G: break; } mpu.setGyroRange(MPU6050_RANGE_500_DEG); switch (mpu.getGyroRange()) { case MPU6050_RANGE_250_DEG: break; case MPU6050_RANGE_500_DEG: break; case MPU6050_RANGE_1000_DEG: break; case MPU6050_RANGE_2000_DEG: break; }

mpu.setFilterBandwidth(MPU6050_BAND_260_HZ); Serial.print("Filter bandwidth set to: "); switch (mpu.getFilterBandwidth()) { case MPU6050_BAND_260_HZ: Serial.println("260 Hz"); break; case MPU6050_BAND_184_HZ: Serial.println("184 Hz"); break; case MPU6050_BAND_94_HZ: Serial.println("94 Hz"); break; case MPU6050_BAND_44_HZ: Serial.println("44 Hz"); break; case MPU6050_BAND_21_HZ: Serial.println("21 Hz"); break; case MPU6050_BAND_10_HZ: Serial.println("10 Hz"); break; case MPU6050_BAND_5_HZ: Serial.println("5 Hz"); break; } //MPU6050 ACCELEROMETER INITIALIZATION CODE END }

float startMillis; float elapsedTime[3]; float rawMillis;

float X[3]; float Y[3]; float Z[3];

void loop() { digitalWrite(recordingLed, LOW); int buttonRecord = digitalRead(2); //Check button for GND input /* Get new sensor events with the readings */ if(buttonRecord == false){ //Toggle record status upon pin 2 being pulled down to GND delay(500); recordStatus = !recordStatus; //If button is pressed, make recordStatus switch from true to false. startMillis = millis(); Serial.print("BUTTON PRESSED"); } if(recordStatus == true){ //While record is toggled, do below digitalWrite(recordingLed, HIGH); sensors_event_t a, g, temp; mpu.getEvent(&a, &g, &temp); for(int arrayNumber = 0; arrayNumber <= 3; ++arrayNumber){ sensors_event_t a, g, temp; mpu.getEvent(&a, &g, &temp);

X[arrayNumber] = (a.acceleration.x);                                             //SD CARD VALUES ASSIGN BEGIN/DO MATH TO CALCULATE G'S
Y[arrayNumber] = (a.acceleration.y);
Z[arrayNumber] = (a.acceleration.z);

rawMillis = millis()-startMillis;
elapsedTime[arrayNumber] = rawMillis;
Serial.println(elapsedTime[arrayNumber]);
Serial.println(X[arrayNumber]);
Serial.println(Y[arrayNumber]);
Serial.println(Z[arrayNumber]);
Serial.println(arrayNumber);                                                     // diagnostic for checking to make sure arrays are being populated properly

}

File dataFile = SD.open(&quot;datalog.txt&quot;, FILE_WRITE);
Serial.print(&quot;LEFT LOGGING LOOP, ENTERING WRITE LOOP&quot;);

if (dataFile) { // if the file is available, write to it for(int arrayNumber = 0; arrayNumber <= 3; ++arrayNumber){ Serial.print("NOW INSIDE WRITE LOOP"); // confirmation of being inside the write loop, and writing data to the file String(dataString) = ( elapsedTime[arrayNumber] + String(", " ) + X[arrayNumber] + String(", ") + Y[arrayNumber] + String(", ") + Z[arrayNumber] //Format data for output to the comma separated format for excel/gnuplot ); dataFile.println(dataString); } dataFile.close(); Serial.println("WRITE LOOP DONE"); } } }

  • Can you be more specific regarding "does not run properly"? In any case, there's this, this, and this. Without knowing what the issue/s with arrays is/are it's hard to help. – Dave Newton Jun 30 '21 at 19:42
  • How about writing to external memory such as RAM or Ferroelectric RAM. That will take it as fast as you can write it, there is no write delay. Then when the test if finished transfer it to the SD card. The Ferroelectric RAM will remember even if the power is off. – Gil Jun 30 '21 at 20:28
  • Have you considered writing a binary file instead of text? – Edgar Bonet Jul 01 '21 at 06:49
  • Why not writing a whole string of data at one time ? "faster" ? – Antonio51 Jul 01 '21 at 06:49
  • @DaveNewton Thank you for the links. The third one is especially helpful for me. The link recommends executing the SD.open and SD.close command outside the loop. However if I try to do that, this goes back to my "not run properly" statement as the program seems to go through one cycle and never repeat. I'm not really sure why this is? I figure the file should be able to be opened in setup and closed at any time you really want, and while the file is open, new data can be brought in, manipulated, and written. Does having a file open stop other functions or prevent looping? Thank you – Colby Johnson Jul 01 '21 at 13:20
  • @Gil Hey Gil, appreciate the info on FRAM. Unfortunately, I don't think using that will solve my problem as I believe this is more a problem with my code being slow. I will keep that in mind for the future though. – Colby Johnson Jul 01 '21 at 13:21
  • @jsotola I didn't even notice. When I merged two example programs together to make this, I must have left it in there by mistake. Removed from the code as of now. – Colby Johnson Jul 01 '21 at 13:22
  • @EdgarBonet I would consider doing that, but this data is likely going to be going to other people who will be simply taking the device, popping an SD card in, getting data, and pulling the SD card out. If I make them mess around with binary, I feel like it's just going to wind up not getting used. I could process the data for them, but then they'll be forced to come to me every time in which I think the same outcome would follow. – Colby Johnson Jul 01 '21 at 13:23
  • @Antonio51 Antonio, that was a great addition to the program. I am now only printing to the file once instead of multiple times each cycle. Thanks! – Colby Johnson Jul 01 '21 at 13:24
  • Logical ... for a caracter you open the file, write, then close. All operations that takes "some" time ! I had to resolve that problem in saving parameters of Temp, humidity and some others things (100 caracters) every 5 seconds, so I know ... – Antonio51 Jul 01 '21 at 13:28
  • @Antonio51 Ideally, I'd like to squeeze at least 260Hz out of these write operations, as that's the speed of my MPU-6050s recording ability. I'm measuring some really high accelerations that are dampened quickly, so I really want to maximize my readings/second – Colby Johnson Jul 01 '21 at 13:30
  • Don't add datas that you really don't use ... as labels ... You know what and how you are saving your data, save only these, eventually as binary datas in place of strings of caracters. You can also use a "data set" record. A little more complicated but usefull. – Antonio51 Jul 01 '21 at 13:33
  • @Antonio51 I also noticed I was using a lot of strings unnecessarily. Unfortunately I need both decimal places as well as positive/negatives, so I'm guessing floats are the only way to have all that, even though the acceleration values will never exceed 64 m/S^2 – Colby Johnson Jul 01 '21 at 13:38
  • If I remember well, I could read 8 kbytes /second and writing quasi similar but did not measure this because every 5 seconds. (on an Arduino 16 MHz). I thing it would be beter on a LGT328 (?) which is running at 32 MHz. – Antonio51 Jul 01 '21 at 13:42
  • @Antonio51 A faster microcontroller is probably what I need, but I'm certain this has to be possible. I'm merely logging XYZ values and time. I guess now I've got some personal attachment to trying to solve this. I'm also a beginner so it's a good learning opportunity for me. – Colby Johnson Jul 01 '21 at 13:45
  • If datas are 2 bytes large enough (15 bits + sign), you can save all as word ... but when reading out, change or correct the type of variables accordingly. Good luck ! – Antonio51 Jul 01 '21 at 13:46

1 Answers1

0

So I spent a while longer banging my head on the wall, and I finally found a solution to my issue. @DaveNewton provided several links that lead to other links, and I found my way to here where some not-so-common knowledge is explained. It appears that for some reason the FILE_WRITE function is slow, and takes a lot of time to execute.

This can be substituted by using

O_CREAT | O_WRITE | O_APPEND

as this first checks if the file is created, and if not, creates it. Once the or check passes for the O_CREAT section, it moves on into the O_WRITE command. Then the O_APPEND checks if there is any data already present and if so, move to the end of the line to begin new data logging.

My finished code in it's entirety is below, I hope it helps someone in the future. I'm using a MicroCenter 32GB (class 10) Micro SD card to store the data, this SD card breakout board, and this MPU-6050 based accelerometer.

// This program is designed to log data to an SD card via an arduino Uno, and SD card breakout board, and a MPU-6050 accelerometer module.

// MPU-6050: https://www.amazon.com/HiLetgo-MPU-6050-Accelerometer-Gyroscope-Converter/dp/B078SS8NQV // Micro SD Card Breakout: https://www.adafruit.com/product/254 // Micro SD Card I used: https://www.microcenter.com/product/485584/micro-center-32gb-microsdhc-card-class-10-flash-memory-card-with-adapter

// Link to original Arduino StackExchange submission where I worked through this code: https://arduino.stackexchange.com/questions/84961/seeking-to-write-a-ton-of-information-to-an-sd-card-as-close-to-live-as-possible

#include <Adafruit_MPU6050.h> #include <Adafruit_Sensor.h> #include <Wire.h> #include <SPI.h> #include <SD.h>

int recordStatus = false; //Default recordStatus is zero. int recordingLed = 3; //Output to show us whether we are recording or not. const int chipSelect = 10; //Use pin 10 as our CS pin Adafruit_MPU6050 mpu;

void setup(void) { pinMode(2, INPUT_PULLUP); //Set pin 2 to have a default HIGH state using the internal pull up resistor. pinMode(3, OUTPUT); Serial.begin(500000); SPI.begin(); SPI.beginTransaction(SPISettings(16000000, MSBFIRST, SPI_MODE0)); //I have no idea if this really does anything, but I figure I'd just leave it since it's not causing any problems. while (!Serial) delay(10); // will pause Zero, Leonardo, etc until serial console opens
//SD CARD WRITE CODE BEGIN
{

Serial.print("Initializing SD card...");

// see if the card is present and can be initialized: if (!SD.begin(chipSelect)) { Serial.println("Card failed, or not present"); // don't do anything more: while (1); } Serial.println("card initialized."); } //SD CARD WRITE CODE END

Serial.println("Adafruit MPU6050 test!"); //MPU6050 ACCELEROMETER INITIALIZATION CODE BEGIN // Try to initialize! if (!mpu.begin()) { Serial.println("Failed to find MPU6050 chip"); while (1) { delay(10); } } Serial.println("MPU6050 Found!");

mpu.setAccelerometerRange(MPU6050_RANGE_16_G); //I originally had this at 4 but was capping results, so I moved it up to 16. Feel free to lower this if you know your application will not exceed 4Gs switch (mpu.getAccelerometerRange()) { case MPU6050_RANGE_2_G: break; case MPU6050_RANGE_4_G: break; case MPU6050_RANGE_8_G: break; case MPU6050_RANGE_16_G: break; } mpu.setGyroRange(MPU6050_RANGE_500_DEG); //I'm not using the gyro at all, but 500 degrees seems like a good value to use here. switch (mpu.getGyroRange()) { case MPU6050_RANGE_250_DEG: break; case MPU6050_RANGE_500_DEG: break; case MPU6050_RANGE_1000_DEG: break; case MPU6050_RANGE_2000_DEG: break; }

mpu.setFilterBandwidth(MPU6050_BAND_260_HZ); //Select maximum frequency for recording. Serial.print("Filter bandwidth set to: "); switch (mpu.getFilterBandwidth()) { case MPU6050_BAND_260_HZ: Serial.println("260 Hz"); break; case MPU6050_BAND_184_HZ: Serial.println("184 Hz"); break; case MPU6050_BAND_94_HZ: Serial.println("94 Hz"); break; case MPU6050_BAND_44_HZ: Serial.println("44 Hz"); break; case MPU6050_BAND_21_HZ: Serial.println("21 Hz"); break; case MPU6050_BAND_10_HZ: Serial.println("10 Hz"); break; case MPU6050_BAND_5_HZ: Serial.println("5 Hz"); break;

SPI.endTransaction(); } //MPU6050 ACCELEROMETER INITIALIZATION CODE END } float X; float Y; float Z; float startMillis; float currentMillis; float rawMillis; int tickMillis = LOW;

void loop() { int buttonRecord = digitalRead(2); //Check button (pin 2) to see if it is grounded through momentary switch. // Get new sensor events with the readings.

if(buttonRecord == LOW){ //If buttonRecord switches to GND, switch tickMillis to it's opposite value as a toggle and store startMillis, then wait 300ms as a debounce. tickMillis = !tickMillis; startMillis = millis(); delay(300); } if(tickMillis == HIGH){ File dataFile = SD.open("datalog.txt", O_CREAT | O_WRITE | O_APPEND); // Create datalog.txt. If already created, move to end of stored data, and begin write function if (dataFile) { // if the file is available, write to it:

for(uint8_t i = 0; i < 250; i++){ // For i = 0, execute the below code. Then increment i by 1. Once past 250, exit for() loop digitalWrite(recordingLed, HIGH);

sensors_event_t a, g, temp;                                                     // Commands to trigger event sensing and acceleration logging.
mpu.getEvent(&amp;a, &amp;g, &amp;temp);

X = a.acceleration.x/9.81;                                                      // Divide normal acceleration return values to calculate G forces
Y = a.acceleration.y/9.81;
Z = a.acceleration.z/9.81;

rawMillis = millis()-startMillis;                                               // Take our current millis value and subtract our beginning startMillis from it to get how long we've actually been recording.

dataFile.print(rawMillis/1000, 3);                                              // Store data in a Comma-Seperated-Value format. The '3' after rawMillis/1000 says print out to 3 decimal places.
dataFile.print(&quot;, &quot;);                                                           // I just use a txt file since GNUplot can plot from that value, but a CSV library could probably be used here.
dataFile.print(X);                                                              // Excel doesn't allow plots beyond 255 datapoints for god knows why. So GNUplot is basically a necessity to plot these accurately to timestamp.
dataFile.print(&quot;, &quot;);
dataFile.print(Y);
dataFile.print(&quot;, &quot;);
dataFile.print(Z);
dataFile.println();
}
dataFile.flush();
dataFile.close();
}

else { delay(500); Serial.println("error opening datalog.txt"); } } else { digitalWrite(recordingLed, LOW); } }

Thanks so much for the comments/advice everyone. It's really appreciated. The final recording frequency for this is ~170Hz, which is good enough for me.