Performing an operation on all values in certain fields as specified in a list, then overwrite those fields in their layer file using PyQGIS

Question

I'm stumped here. I want to multiply all of the values by 100 in all of the CSV files I'm using in this project for all fields other than the join field. I don't need to create a new field. I simply want to overwrite the one that already exists. I know how to do it using the field calculator, but it seems a little cumbersome and/or confusing with PyQGIS. Still, it would be nice to know. I've been trying to accomplish this with the built-in Python csv class.

The threads below describe the how to create a new field and populate it with values resulting from an operation on 2 or more existing fields. I just want to do an operation on the values in an existing field and repopulate the original field with the new values.

PyQGIS: Multiply fields and populate new field

How to multiply two field values in pyqgis?

https://gis.stackexchange.com/a/309738/174192

Why can't I multiply values from column named 'length' using field calculator via PyQGIS?

Here's what I was trying:

from qgis.core import QgsProject, QgsVectorLayer, QgsLayerTreeGroup, QgsLayerTreeLayer, QgsLayoutNodesItem, QgsMapThemeCollection
from PyQt5 import QtGui
import csv
import os
#creates the project directories and reformats for compatibility with all methods used in script
projectDirRawUri = r"E:\Jason\Documents\2020-01\mapping project\gis\test_automate_all"
projectDirUri = projectDirRawUri.replace("\", "/")
layerDirName = "layers"
#layerDirPath = os.path.join(projectPathUri, layerDirName)
layerDirUri = projectDirUri + "/" + layerDirName
if not os.path.isdir(layerDirUri):
    os.mkdir(layerDirUri)
else:
    print("Directory already exists. Please specify another.")
dataLayerDirName = "datasets"
dataLayerDirUri = layerDirUri + "/" + dataLayerDirName
if not os.path.isdir(dataLayerDirUri):
    os.mkdir(dataLayerDirUri)
else:
    print("Directory already exists. Please specify another.")
#deselects all nodes in the "layers panel"/"map legend"
QgsLayoutNodesItem.deselectNode
#creates the root variable
root = QgsProject.instance().layerTreeRoot()
#creates the group for storing the CSV layers
#dataGroup = root.addGroup("")
dataGroup = QgsLayerTreeGroup("data_layers")
root.addChildNode(dataGroup)
#assign/initiate CSV layer variables
csvSrcRawUri = r"E:\Jason\Documents\2020-01\mapping project\gis\test_automate_all\sources\datasetsrekeithsstudy"
csvSrcUri = csvSrcRawUri.replace("\", "/")
#print(csvSrcUri)
csvSrcUriList = []
csvSrcUriIndex = 0
csvLayerDirUri = dataLayerDirUri
csvLayerUriList = []
csvLayerNameList = []
csvLayerNameIndex = 0
csvLayerUriIndex0 = 0
csvLayerUriIndex1 = 0
csvLayerIndex = 0
csvSavedLayerList = []
wktRawUri = r"E:\Jason\Documents\2020-01\mapping project\gis\test_automate_all\sources\Public Use Microdata Areas (PUMA)\geo_export_0ca223b7-207e-4bc2-bf55-9947aa4998c5.prj"
wktUri = wktRawUri.replace("\", "/")
reading prj file
#with open(wktUri, 'r') as wkt:
# creating a wkt reader object
wktInfo = wkt.read()
# extracting field names through first row
crs = QgsCoordinateReferenceSystem(wkt)
assert crs.isValid()
#print(crs)
#populates the CSV source URI and layer name lists
for root, dirs, files in os.walk(csvSrcUri):
    for file in files:
        if(file.endswith(".csv")):
            csvSrcUriList.append(os.path.join(root,file))
            csvLayerNameList.append(os.path.join(file))
#sorts the CSV source URI and layer name lists alphabetically to match eachother
csvSrcUriList.sort()
csvLayerNameList.sort()
#populates the CSV layer URI list
while csvLayerUriIndex0 < len(csvSrcUriList):
    csvLayerUri = csvLayerDirUri + "/" + csvLayerNameList[csvLayerUriIndex0]
    csvLayerUriList.append(csvLayerUri)
    csvLayerUriIndex0 += 1
#print(csvLayerUriList)
#reformat fields as percentages
for csvFile in csvSrcUriList:
    with open(csvFile, 'r') as csvSrcFile, open(csvLayerUriList[csvLayerUriIndex1], 'w') as csvTrgtFile:
        csvreader = csv.reader(csvSrcFile)
        csvwriter = csv.writer(csvTrgtFile)
        fldList = next(csvreader)
        rowList = []
        for row in csvreader:
            rowList.append(row)
        for row in rowList:
            rowIdx = 1
            while rowIdx < len(row):
                percentValue = str(float(row[rowIdx])*100)
                row[rowIdx] = percentValue
                rowIdx += 1
        csvwriter.writerow(fldList)
        csvwriter.writerows(rowList)
    csvLayerUriIndex1 +=1
#formats the CSV source URI list so the items in it can be used by the QgsVectorLayer() method
for file in csvSrcUriList:
    filePrefix = 'file:///'
    fileSuffix = '?delimeter=,'
    fileLocation = filePrefix + file.replace("\", "/") + fileSuffix
    csvSrcUriList[csvSrcUriIndex] = fileLocation
#removes the .csv extention from the items in the CSV layer name list
for file in csvLayerNameList:
    layerName =  file.replace(".csv", "")
    csvLayerNameList[csvLayerNameIndex] = layerName
    csvLayerNameIndex += 1
#print(csvSrcUriList)
#print(csvLayerNameList)
#loop to create and perform all operations on all CSV layers including grouping
for csvLayerName in csvLayerNameList:
csvSrcUri = csvSrcUriList[csvLayerIndex]
csvLayerUri = csvLayerUriList[csvLayerIndex] #replaced code
#creates the CSV layer name from the list of csv names and fields
csvLayerName = csvLayerNameList[csvLayerIndex]
#adds the initial CSV layer to the project and checks the layer validity

csvLayer = QgsVectorLayer(csvSrcUri, csvLayerName, 'delimitedtext')
csvLayer = QgsVectorLayer(csvLayerUri, csvLayerName, 'delimitedtext') #replaced code
QgsProject.instance().addMapLayer(csvLayer)
if not csvLayer or not csvLayer.isValid():
    csvAddLayerErrorMsg = csvLayerName + &quot; failed to load!&quot;
    print(csvAddLayerErrorMsg)

#copies the shape layer and places the copy in the correct group
root = QgsProject.instance().layerTreeRoot()
csvGroup = root.findGroup(dataGroup.name())

csvOriginalLayer = root.findLayer(csvLayer.id())
csvGroupedLayer = csvOriginalLayer.clone()
csvGroup.insertChildNode(-1, csvGroupedLayer)
csvNodeLayer = root.findLayer(csvLayer.id()) #replaced code
csvGroup.insertChildNode(-1, csvNodeLayer) #replaced code

csvSaveOptions = QgsVectorFileWriter.SaveVectorOptions()
csvSaveOptions.driverName = "CSV"
csvSaveOptions.fileEncoding = "UTF-8"
csvSavedLayer = csvGroupedLayer.layer()
QgsVectorFileWriter.writeAsVectorFormatV2(csvSavedLayer, csvLayerUriList[csvLayerIndex], QgsCoordinateTransformContext(), csvSaveOptions)
if not csvGroupedLayer.layer() or not csvGroupedLayer.layer().isValid():
csvSaveLayerErrorMsg = csvLayerName + " failed to save!"
print(csvAddLayerErrorMsg)
#removes the ungrouped shape layer
parent = csvOriginalLayer.parent()
parent.removeChildNode(root.findLayer(csvLayer.id()))
#populates in the CSV saved layer list for use in table joins
csvSavedLayerList.append(csvSavedLayer)
csvSavedLayerList.append(csvLayer)

csvLayerIndex +=1


#deselects all nodes in the "layers panel"/"map legend"
QgsLayoutNodesItem.deselectNode
#assign/initiate SHP layer variables
shpGroupNameList = csvLayerNameList
shpGroupNameIndex = 0
shpSrcRawUri = r"E:\Jason\Documents\2020-01\mapping project\gis\test_automate_all\sources\Public Use Microdata Areas (PUMA)\geo_export_0ca223b7-207e-4bc2-bf55-9947aa4998c5.shp"
shpSrcUri = shpSrcRawUri.replace("\", "/")
#print(shpSrcUri)
shpLayerDirUri = layerDirUri
shpLayerUriList = []
shpLayerUriIndex0 = 0
shpLayerUriIndex1 = 0
shpLayerNameList = []
shpLayerNameIndex0 = 0
shpLayerNameIndex1 = 0
shpLayerIndex0 = 0
shpLayerIndex1 = 0
#print(csvLayerNameList)
#creates the groups for storing the SHP layers
for shpGroup in shpGroupNameList:
    shpGroup = QgsLayerTreeGroup(shpGroupNameList[shpGroupNameIndex])
    root.addChildNode(shpGroup)
    shpGroupNameIndex +=1
#populates the SHP layer name list in nested list consistant with the SHP group name list
for file in csvLayerUriList:
    fields0 = []
    shpLayerNameIndex1 = 0
    with open(csvLayerUriList[shpLayerNameIndex0], 'r') as csvfile:
        csvreader = csv.reader(csvfile)
        fields0 = next(csvreader)
        fields0.sort()
        rItem = "puma"
        for field0 in fields0:
            if(field0 == rItem):
                fields0.remove(rItem)
    for field0 in fields0:
        field0 = "{}-{}".format(shpGroupNameList[shpLayerNameIndex0], fields0[shpLayerNameIndex1])
        fields0[shpLayerNameIndex1] = field0
        shpLayerNameIndex1 += 1
    shpLayerNameList.append(fields0)
    shpLayerNameIndex0 += 1
#populates the SHP layer URI list in nested list consistant with the SHP layer name list
for shpLayerNameList1 in shpLayerNameList:
    shpLayerUriList1 = []
    shpLayerUriIndex1 = 0
    for shpLayerName in shpLayerNameList1:
        shpLayerUri = shpLayerDirUri + "/" + shpLayerNameList[shpLayerUriIndex0][shpLayerUriIndex1] + '.shp'
        shpLayerUriList1.append(shpLayerUri)
        shpLayerUriIndex1 +=1
    shpLayerUriList.append(shpLayerUriList1)
    shpLayerUriIndex0 +=1
#loop to create and perform all operations on all SHP layers including grouping
for shpLayerNameList1 in shpLayerNameList:
    shpLayerIndex1 = 0
    for shpLayerName in shpLayerNameList1:
        #creates the shape layer name from the list of csv names and fields
        shpLayerName = shpLayerNameList[shpLayerIndex0][shpLayerIndex1]
        #adds the initial shape layer to the project and checks the layer validity
        shpLayer = QgsVectorLayer(shpSrcUri, shpLayerName, "ogr")
        QgsProject.instance().addMapLayer(shpLayer)
        if not shpLayer or not shpLayer.isValid():
            shpAddLayerErrorMsg = shpLayerName + " failed to load!"
            print(shpAddLayerErrorMsg)
        #copies the shape layer and places the copy in the correct group
        root = QgsProject.instance().layerTreeRoot()
        shpGroupName = shpGroupNameList[shpLayerIndex0]
        shpGroup = root.findGroup(shpGroupName)
        shpOriginalLayer = root.findLayer(shpLayer.id())
        shpGroupedLayer = shpOriginalLayer.clone()
        shpGroup.insertChildNode(-1, shpGroupedLayer)
        #saves the grouped shape layer and checks the layer validity
        shpSaveOptions = QgsVectorFileWriter.SaveVectorOptions()
        shpSaveOptions.driverName = "ESRI Shapefile"
        shpSaveOptions.fileEncoding = "UTF-8"
        shpSavedLayer = shpGroupedLayer.layer()
        QgsVectorFileWriter.writeAsVectorFormatV2(shpSavedLayer, shpLayerUriList[shpLayerIndex0][shpLayerIndex1], QgsCoordinateTransformContext(), shpSaveOptions)
        if not shpGroupedLayer.layer() or not shpGroupedLayer.layer().isValid():
            shpSaveLayerErrorMsg = shpLayerName + " failed to save!"
            print(shpAddLayerErrorMsg)
        #joins the shape layer to the proper CSV field
        shpField = 'puma'
        csvField = 'puma'
        joinCsvName = shpSavedLayer.name().split("-")[0]
        for csvSavedLayer in csvSavedLayerList:
            csvSavedLayerName = csvSavedLayer.name()
            if csvSavedLayerName == joinCsvName:
                joinCsvLayer = csvSavedLayer
        joinCsvField = []
        joinCsvField.append(shpSavedLayer.name().split("-")[1])
        joinObject = QgsVectorLayerJoinInfo()
        joinObject.setJoinLayer(joinCsvLayer)
        joinObject.setJoinFieldName(csvField)
        joinObject.setTargetFieldName(shpField)
        joinObject.setUsingMemoryCache(True)
        joinObject.setJoinFieldNamesSubset(joinCsvField)
        shpSavedLayer.addJoin(joinObject)
    #creates the variables for the join field gradient renderer
    myColumn = shpSavedLayer.name().replace('-','_')
    myRangeList = []
    myOpacity = 1
    ranges = []
    gradientIndex = 0
    gradientLevels = 20
    #creates the gradient levels for the join field gradient renderer
    while gradientIndex &lt; gradientLevels:
        myMin = gradientIndex * 5
        myMax = (gradientIndex + 1) * 5
        myLabel = &quot;{}-{}&quot;.format(myMin, myMax)
        colorRGB = '{:x}'.format(255-(gradientIndex * (255 // gradientLevels)))
        myColor = QtGui.QColor(&quot;#{}{}{}&quot;.format(colorRGB, colorRGB, colorRGB))
        ranges.append((myMin, myMax, myLabel, myColor))
        gradientIndex += 1
    #sets the gradient levels for the join field gradient renderer
    for myMin, myMax, myLabel, myColor in ranges:
        mySymbol = QgsSymbol.defaultSymbol(2)
        mySymbol.setColor(myColor)
        mySymbol.setOpacity(myOpacity)
        myRange = QgsRendererRange(myMin, myMax, mySymbol, myLabel)
        myRangeList.append(myRange)
    #sets the renderer for
    myRenderer = QgsGraduatedSymbolRenderer('', myRangeList)
    myRenderer.setMode(QgsGraduatedSymbolRenderer.Quantile)
    myRenderer.setClassAttribute(myColumn)
    #applies the renderer to the saved shape layer
    layerType = shpSavedLayer.type()
    if layerType == QgsMapLayerType.VectorLayer:
        shpSavedLayer.setRenderer(myRenderer)

    #layerTheme = QgsMapThemeCollection

    #removes the ungrouped shape layer
    parent = shpOriginalLayer.parent()
    parent.removeChildNode(root.findLayer(shpLayer.id()))

    shpLayerIndex1 += 1

shpLayerIndex0 +=1


#print(vectorLayerNameIndex0)
#print(vectorLayerNameIndex1)
#print(fields0)
#print(fields1)
#print(vectorLayerNameList)
#print(vectorLayerUriList)

The problematic code starts with the comment, #reformat fields as percentages. This works well to create CSV files with values as percentages. However, the script seems to be writing the rows twice to each file on every other line (the original CSV files have 56 rows, the new files have 111), they fail to load as layers, and joins cannot be done with the SHP files. The Layer Panel also shows an error icon, shows the layers loaded twice (though the files are not written twice), and the information tab of the layer properties window shows a feature count of over a billion, which cannot possibly be true because the CSV has 111 rows and 8 columns as shown in the attached image.

Originally, this part of the scipt handled saving the CSV files:

for csvLayerName in csvLayerNameList:
csvSrcUri = csvSrcUriList[csvLayerIndex]
#creates the CSV layer name from the list of csv names and fields
csvLayerName = csvLayerNameList[csvLayerIndex]
#adds the initial CSV layer to the project and checks the layer validity
csvLayer = QgsVectorLayer(csvSrcUri, csvLayerName, 'delimitedtext')
QgsProject.instance().addMapLayer(csvLayer)
if not csvLayer or not csvLayer.isValid():
    csvAddLayerErrorMsg = csvLayerName + " failed to load!"
    print(csvAddLayerErrorMsg)
#copies the shape layer and places the copy in the correct group
root = QgsProject.instance().layerTreeRoot()
csvGroup = root.findGroup(dataGroup.name())
csvOriginalLayer = root.findLayer(csvLayer.id())
csvGroupedLayer = csvOriginalLayer.clone()
csvGroup.insertChildNode(-1, csvGroupedLayer)
csvSaveOptions = QgsVectorFileWriter.SaveVectorOptions()
csvSaveOptions.driverName = "CSV"
csvSaveOptions.fileEncoding = "UTF-8"
csvSavedLayer = csvGroupedLayer.layer()
QgsVectorFileWriter.writeAsVectorFormatV2(csvSavedLayer, csvLayerUriList[csvLayerIndex], QgsCoordinateTransformContext(), csvSaveOptions)
if not csvGroupedLayer.layer() or not csvGroupedLayer.layer().isValid():
    csvSaveLayerErrorMsg = csvLayerName + " failed to save!"
    print(csvAddLayerErrorMsg)
#removes the ungrouped shape layer
parent = csvOriginalLayer.parent()
parent.removeChildNode(root.findLayer(csvLayer.id()))
#populates in the CSV saved layer list for use in table joins
csvSavedLayerList.append(csvSavedLayer)
csvSavedLayerList.append(csvLayer)
csvLayerIndex +=1

This works well, but would necessitate using QGIS API classes to modify the CSV layer files because they are write-protected after they are written (I think).

Update: Trying to close QGIS after running this script causes it to crash here is the stack trace if that is of any importance:

Stack Trace

QMap::value qmap.h:656
QgsObjectCustomProperties::value qgsobjectcustomproperties.cpp:37
QgsLayerTreeNode::customProperty qgslayertreenode.cpp:176
QgsLayerTreeModel::flags qgslayertreemodel.cpp:369
QModelIndex::flags :
QTreeViewPrivate::isIndexExpanded :
QTreeViewPrivate::layout :
QTreeView::doItemsLayout :
QTreeViewPrivate::updateScrollBars :
QTreeView::scrollTo :
QAbstractItemView::currentChanged :
QTreeView::currentChanged :
QMetaObject::activate :
QItemSelectionModel::default constructor closure :
QItemSelectionModel::qt_static_metacall :
QMetaObject::activate :
QAbstractItemModel::beginRemoveRows :
QgsLayerTreeModel::nodeWillRemoveChildren qgslayertreemodel.cpp:761
QMetaObject::activate :
QgsLayerTreeNode::willRemoveChildren moc_qgslayertreenode.cpp:290
QMetaObject::activate :
QgsLayerTreeNode::willRemoveChildren moc_qgslayertreenode.cpp:290
QgsLayerTreeNode::removeChildrenPrivate qgslayertreenode.cpp:245
QgsLayerTreeGroup::removeChildren qgslayertreegroup.cpp:161
QgsLayerTreeGroup::removeChildNode qgslayertreegroup.cpp:136
QgsLayerTreeRegistryBridge::layersWillBeRemoved qgslayertreeregistrybridge.cpp:92
QMetaObject::activate :
QgsProject::layersWillBeRemoved moc_qgsproject.cpp:1085
QMetaObject::activate :
QgsMapLayerStore::layersWillBeRemoved moc_qgsmaplayerstore.cpp:266
QgsMapLayerStore::removeMapLayers qgsmaplayerstore.cpp:147
QgsMapLayerStore::removeMapLayers qgsmaplayerstore.cpp:121
QgsMapLayerStore::removeAllMapLayers qgsmaplayerstore.cpp:203
QgsProject::clear qgsproject.cpp:793

Answer 1

So, the empty row issue was resolved for the following line:

    with open(csvFile, 'r') as csvSrcFile, open(csvLayerUriList[csvLayerUriIndex1], 'w') as csvTrgtFile:

The Python open() function requires the newline = ''argument in Windows as explained here. So, the corrected code looks like this:

    with open(csvFile, 'r', newline='') as csvSrcFile, open(csvLayerUriList[csvLayerUriIndex1], 'w', newline='') as csvTrgtFile:

However, all of the other issues still exist.

The rest of the issues were resolved by updating variable names consistently throughout the script and changing some methodology used in handling the CSV layers. This is the finished script:

from qgis.core import QgsProject, QgsVectorLayer, QgsLayerTreeGroup, QgsLayerTreeLayer, QgsLayoutNodesItem, QgsMapThemeCollection
from PyQt5 import QtGui
import csv
import os
#creates the project directory and reformats for compatibility with all methods used in script
projectDirRawUri = r"E:\Jason\Documents\2020-01\mapping project\gis\test_automate_all"
projectDirUri = projectDirRawUri.replace("\", "/")
projDirErrorMsg = "{} directory already exists."
#creates the layer directory
layerDirName = "layers"
layerDirUri = projectDirUri + "/" + layerDirName
if not os.path.isdir(layerDirUri):
    os.mkdir(layerDirUri)
else:
    print(projDirErrorMsg.format(layerDirName))
#creates the data layer directory
dataLayerDirName = "datasets"
dataLayerDirUri = layerDirUri + "/" + dataLayerDirName
if not os.path.isdir(dataLayerDirUri):
    os.mkdir(dataLayerDirUri)
else:
    print(projDirErrorMsg.format(dataLayerDirName))
#creates the data preprocessing directory
dataPreprocDirName = "preprocessing"
dataPreprocDirUri = dataLayerDirUri + "/" + dataPreprocDirName
if not os.path.isdir(dataPreprocDirUri):
    os.mkdir(dataPreprocDirUri)
else:
    print(projDirErrorMsg.format(dataPreprocDirName))
#deselects all nodes in the "layers panel"/"map legend"
QgsLayoutNodesItem.deselectNode
#creates the root variable
root = QgsProject.instance().layerTreeRoot()
#creates the group for storing the CSV layers
dataGroup = QgsLayerTreeGroup("data_layers")
root.addChildNode(dataGroup)
reading prj file
wktRawUri = r"E:\Jason\Documents\2020-01\mapping project\gis\test_automate_all\sources\Public Use Microdata Areas (PUMA)\geo_export_0ca223b7-207e-4bc2-bf55-9947aa4998c5.prj"
wktUri = wktRawUri.replace("\", "/")
#with open(wktUri, 'r') as wkt:
# creating a wkt reader object
wktInfo = wkt.read()
# extracting field names through first row
crs = QgsCoordinateReferenceSystem(wkt)
assert crs.isValid()
#print(crs)
#populates the CSV source URI and layer name lists
csvSrcRawUri = r"E:\Jason\Documents\2020-01\mapping project\gis\test_automate_all\sources\datasetsrekeithsstudy"
csvSrcUri = csvSrcRawUri.replace("\", "/")
csvSrcUriList = []
csvSrcUriIndex = 0
csvLayerNameList = []
for root, dirs, files in os.walk(csvSrcUri):
    for file in files:
        if(file.endswith(".csv")):
            csvSrcUriList.append(os.path.join(root,file))
            csvLayerNameList.append(os.path.join(file))
#sorts the CSV source URI and layer name lists alphabetically to match eachother
csvSrcUriList.sort()
csvLayerNameList.sort()
#populates the CSV preprocessing URI list
csvPreprocDirUri = dataPreprocDirUri
csvPreprocUriList = []
csvPreprocUriIndex0 = 0
while csvPreprocUriIndex0 < len(csvSrcUriList):
    csvPreprocUri = csvPreprocDirUri + "/" + csvLayerNameList[csvPreprocUriIndex0]
    csvPreprocUriList.append(csvPreprocUri)
    csvPreprocUriIndex0 += 1
#reformat fields as percentages
csvPreprocUriIndex1 = 0
for csvFile in csvSrcUriList:
    rowList = []
    fldList = []
    with open(csvFile, 'r', encoding="utf-8", newline='') as csvSrcFile, open(csvPreprocUriList[csvPreprocUriIndex1], 'w', encoding="utf-8", newline='') as csvTrgtFile:
        csvreader = csv.reader(csvSrcFile)
        csvwriter = csv.writer(csvTrgtFile)
        fldList = next(csvreader)
        csvwriter.writerow(fldList)
        for row in csvreader:
            rowList.append(row)
        for row in rowList:
            rowIdx = 1
            while rowIdx < len(row):
                percentValue = format(float(row[rowIdx])*100, ".9g")
percentValue = str(float(row[rowIdx])*100)
            row[rowIdx] = percentValue
            rowIdx += 1
    csvwriter.writerows(rowList)
csvPreprocUriIndex1 += 1


#copies the CSV preprocessing URI list and reformats the items in it so they can be used by the QgsVectorLayer() method
csvVectorLayerUriList = csvPreprocUriList.copy()
csvVectorLayerUriIndex = 0
for file in csvVectorLayerUriList:
    filePrefix = 'file:///'
    fileSuffix = '?delimeter=,'
    fileLocation = filePrefix + file + fileSuffix
    csvVectorLayerUriList[csvVectorLayerUriIndex] = fileLocation
    csvVectorLayerUriIndex += 1
#populates the CSV layer URI list
csvLayerDirUri = dataLayerDirUri
csvLayerUriList = []
csvLayerUriIndex = 0
while csvLayerUriIndex < len(csvSrcUriList):
    csvLayerUri = csvLayerDirUri + "/" + csvLayerNameList[csvLayerUriIndex]
    csvLayerUriList.append(csvLayerUri)
    csvLayerUriIndex += 1
#removes the .csv extention from the items in the CSV layer name list
csvLayerNameIndex = 0
for file in csvLayerNameList:
    layerName =  file.replace(".csv", "")
    csvLayerNameList[csvLayerNameIndex] = layerName
    csvLayerNameIndex += 1
#loop to create and perform all operations on all CSV layers including grouping
csvSavedLayerList = []
csvLayerIndex = 0
for csvLayerName in csvLayerNameList:
    csvLayerUri = csvVectorLayerUriList[csvLayerIndex]
    #creates the CSV layer name from the list of csv names and fields
    csvLayerName = csvLayerNameList[csvLayerIndex]
    #adds the initial CSV layer to the project and checks the layer validity
    csvLayer = QgsVectorLayer(csvLayerUri, csvLayerName, 'delimitedtext')
    QgsProject.instance().addMapLayer(csvLayer)
    if not csvLayer or not csvLayer.isValid():
        csvAddLayerErrorMsg = csvLayerName + " failed to load!"
        print(csvAddLayerErrorMsg)
    #copies the shape layer and places the copy in the correct group
    root = QgsProject.instance().layerTreeRoot()
    csvGroup = root.findGroup(dataGroup.name())
    csvOriginalLayer = root.findLayer(csvLayer.id())
    csvGroupedLayer = csvOriginalLayer.clone()
    csvGroupedLayer = csvOriginalLayer.clone()
    csvGroup.insertChildNode(-1, csvGroupedLayer)
    csvSaveOptions = QgsVectorFileWriter.SaveVectorOptions()
    csvSaveOptions.driverName = "CSV"
    csvSaveOptions.fileEncoding = "UTF-8"
    csvSavedLayer = csvGroupedLayer.layer()
    QgsVectorFileWriter.writeAsVectorFormatV2(csvSavedLayer, csvLayerUriList[csvLayerIndex], QgsCoordinateTransformContext(), csvSaveOptions)
    if not csvGroupedLayer.layer() or not csvGroupedLayer.layer().isValid():
        csvSaveLayerErrorMsg = csvLayerName + " failed to save!"
        print(csvAddLayerErrorMsg)
    #removes the ungrouped shape layer
    parent = csvOriginalLayer.parent()
    parent.removeChildNode(root.findLayer(csvLayer.id()))
    #populates in the CSV saved layer list for use in table joins
    csvSavedLayerList.append(csvSavedLayer)
csvLayerIndex +=1

This section of code creates a "preprocessing" directory for storing the CSV files modified with Python's built-in csv() method:

#creates the data preprocessing directory
dataPreprocDirName = "preprocessing"
dataPreprocDirUri = dataLayerDirUri + "/" + dataPreprocDirName
if not os.path.isdir(dataPreprocDirUri):
    os.mkdir(dataPreprocDirUri)
else:
    print(projDirErrorMsg.format(dataPreprocDirName))

This section does the preprocessing(multiplying the CSV values by 100) and saving the modified CSV values in a new CSV file:

#populates the CSV source URI and layer name lists
csvSrcRawUri = r"E:\Jason\Documents\2020-01\mapping project\gis\test_automate_all\sources\datasetsrekeithsstudy"
csvSrcUri = csvSrcRawUri.replace("\\", "/")
csvSrcUriList = []
csvSrcUriIndex = 0
csvLayerNameList = []
for root, dirs, files in os.walk(csvSrcUri):
    for file in files:
        if(file.endswith(".csv")):
            csvSrcUriList.append(os.path.join(root,file))
            csvLayerNameList.append(os.path.join(file))
#sorts the CSV source URI and layer name lists alphabetically to match eachother
csvSrcUriList.sort()
csvLayerNameList.sort()
#populates the CSV preprocessing URI list
csvPreprocDirUri = dataPreprocDirUri
csvPreprocUriList = []
csvPreprocUriIndex0 = 0
while csvPreprocUriIndex0 < len(csvSrcUriList):
    csvPreprocUri = csvPreprocDirUri + "/" + csvLayerNameList[csvPreprocUriIndex0]
    csvPreprocUriList.append(csvPreprocUri)
    csvPreprocUriIndex0 += 1
#reformat fields as percentages
csvPreprocUriIndex1 = 0
for csvFile in csvSrcUriList:
    rowList = []
    fldList = []
    with open(csvFile, 'r', encoding="utf-8", newline='') as csvSrcFile, open(csvPreprocUriList[csvPreprocUriIndex1], 'w', encoding="utf-8", newline='') as csvTrgtFile:
        csvreader = csv.reader(csvSrcFile)
        csvwriter = csv.writer(csvTrgtFile)
        fldList = next(csvreader)
        csvwriter.writerow(fldList)
        for row in csvreader:
            rowList.append(row)
        for row in rowList:
            rowIdx = 1
            while rowIdx < len(row):
                percentValue = format(float(row[rowIdx])*100, ".9g")
percentValue = str(float(row[rowIdx])*100)
            row[rowIdx] = percentValue
            rowIdx += 1
    csvwriter.writerows(rowList)
csvPreprocUriIndex1 += 1

The encoding="utf-8" might no be necessary here, but I left it in.

Finally, several lines were added to the section that groups and saves the layer to clean up the duplicate layers that were appearing in the Layers Panel:

    #copies the shape layer and places the copy in the correct group
    root = QgsProject.instance().layerTreeRoot()
    csvGroup = root.findGroup(dataGroup.name())
    csvOriginalLayer = root.findLayer(csvLayer.id())
    csvGroupedLayer = csvOriginalLayer.clone()
    csvGroupedLayer = csvOriginalLayer.clone()
    csvGroup.insertChildNode(-1, csvGroupedLayer)
    csvSaveOptions = QgsVectorFileWriter.SaveVectorOptions()
    csvSaveOptions.driverName = "CSV"
    csvSaveOptions.fileEncoding = "UTF-8"
    csvSavedLayer = csvGroupedLayer.layer()
    QgsVectorFileWriter.writeAsVectorFormatV2(csvSavedLayer, csvLayerUriList[csvLayerIndex], QgsCoordinateTransformContext(), csvSaveOptions)
    if not csvGroupedLayer.layer() or not csvGroupedLayer.layer().isValid():
        csvSaveLayerErrorMsg = csvLayerName + " failed to save!"
        print(csvAddLayerErrorMsg)
    #removes the ungrouped shape layer
    parent = csvOriginalLayer.parent()
    parent.removeChildNode(root.findLayer(csvLayer.id()))

I was trying to cheat and skip the routine where the layer is duplicated, saved, and the original removed from the project, but that seems to break it. If anyone knows how to do this, please reply to this thread.