Skip to content

Commit

Permalink
Add rBoot partition support (#2258)
Browse files Browse the repository at this point in the history
This PR adds partition table support to the ESP8266 bootloader (rBoot). See #2254 #2251.

The ESP8266 flash layout has been reverted to the original Sming 4.2 layout, with the addition of the partition table in the sector before the RF calibration data at end of flash.

Update rBoot

Update bootloader to read ROM addresses from partition table on boot. These overwrite whatever values have been set in its own configuration, thus avoiding inconsistencies. rBoot has been forked to simplify management.

An additional partition subtype has been added for RF calibration data, so existing applications running off the develop branch will need re-building and re-flashing.

The partition table may now be freely relocated for the Esp8266 (+ Host) to allow compatibility with existing devices which must be upgrade OTA.

Methods have been added to OTA classes to allow update regions to be set using partitions; this is safer than working with raw flash addresses but the underlying mechanism hasn't changed.
  • Loading branch information
mikee47 authored Mar 17, 2021
1 parent 8218fe7 commit fec464b
Show file tree
Hide file tree
Showing 36 changed files with 449 additions and 1,007 deletions.
2 changes: 1 addition & 1 deletion .gitmodules
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@
ignore = dirty
[submodule "rboot"]
path = Sming/Components/rboot/rboot
url = https://github.com/raburton/rboot.git
url = https://github.com/mikee47/rboot
ignore = dirty
[submodule "spiffs"]
path = Sming/Components/spiffs/spiffs
Expand Down
13 changes: 5 additions & 8 deletions Sming/Arch/Esp8266/Components/esp8266/startup.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -55,9 +55,8 @@ extern "C" void WEAK_ATTR user_rf_pre_init(void)

extern "C" uint32 ICACHE_FLASH_ATTR WEAK_ATTR user_rf_cal_sector_set(void)
{
// RF calibration stored in last sector of sysParam
auto sysParam = *Storage::findPartition(Storage::Partition::SubType::Data::sysParam);
return ((sysParam.address() + sysParam.size()) / SPI_FLASH_SEC_SIZE) - 1;
auto rfCal = *Storage::findPartition(Storage::Partition::SubType::Data::rfCal);
return rfCal.address();
}

#ifdef SDK_INTERNAL
Expand All @@ -71,16 +70,14 @@ extern "C" void ICACHE_FLASH_ATTR WEAK_ATTR user_pre_init(void)
Storage::initialize();

auto sysParam = *Storage::findPartition(Storage::Partition::SubType::Data::sysParam);
auto rfCal = *Storage::findPartition(Storage::Partition::SubType::Data::rfCal);
auto phy = *Storage::findPartition(Storage::Partition::SubType::Data::phy);

// RF calibration stored in last sector of sysParam
auto sysParamSize = sysParam.size() - SPI_FLASH_SEC_SIZE;

static const partition_item_t partitions[] = {
{SYSTEM_PARTITION_BOOTLOADER, 0, SPI_FLASH_SEC_SIZE},
{SYSTEM_PARTITION_PHY_DATA, phy.address(), phy.size()},
{SYSTEM_PARTITION_SYSTEM_PARAMETER, sysParam.address(), sysParamSize},
{SYSTEM_PARTITION_RF_CAL, sysParam.address() + sysParamSize, SPI_FLASH_SEC_SIZE},
{SYSTEM_PARTITION_SYSTEM_PARAMETER, sysParam.address(), sysParam.size()},
{SYSTEM_PARTITION_RF_CAL, rfCal.address(), rfCal.size()},
};

enum flash_size_map sizeMap = system_get_flash_size_map();
Expand Down
18 changes: 18 additions & 0 deletions Sming/Arch/Esp8266/options.json
Original file line number Diff line number Diff line change
Expand Up @@ -6,5 +6,23 @@
"filename": "$(FLASH_INIT_DATA_VCC)"
}
}
},
"alternate": {
"description": "ESP8266 layout with critical partitions at start of flash",
"partition_table_offset": "0x2000",
"partitions": {
"rf_cal": {
"address": "0x3000"
},
"phy_init": {
"address": "0x4000"
},
"sys_param": {
"address": "0x5000"
},
"rom0": {
"address": "0x8000"
}
}
}
}
5 changes: 4 additions & 1 deletion Sming/Arch/Esp8266/spiffs-two-roms.hw
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,14 @@
"name": "Two ROM slots with single SPIFFS",
"base_config": "spiffs",
"partitions": {
"rom0": {
"subtype": "ota_0"
},
"rom1": {
"address": "0x108000",
"size": "992K",
"type": "app",
"subtype": "ota_0",
"subtype": "ota_1",
"filename": "$(RBOOT_ROM_1_BIN)"
}
}
Expand Down
30 changes: 18 additions & 12 deletions Sming/Arch/Esp8266/standard.hw
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
"name": "Standard config with single ROM",
"comment": "Should work with any Esp8266 variant",
"arch": "Esp8266",
"partition_table_offset": "0x2000",
"partition_table_offset": "self.devices[0].size - 0x6000",
"devices": {
"spiFlash": {
"type": "flash",
Expand All @@ -12,25 +12,31 @@
}
},
"partitions": {
"rom0": {
"address": "0x002000",
"size": "992K",
"type": "app",
"subtype": "factory",
"filename": "$(RBOOT_ROM_0_BIN)"
},
"rf_cal": {
"address": "self.device.size - 0x5000",
"size": "4K",
"type": "data",
"subtype": "rfcal"
},
"phy_init": {
"address": "0x003000",
"address": "self.device.size - 0x4000",
"size": "4K",
"type": "data",
"subtype": "phy",
"filename": "$(FLASH_INIT_DATA)"
},
"sys_param": {
"address": "0x004000",
"size": "16K",
"address": "self.device.size - 0x3000",
"size": "12K",
"type": "data",
"subtype": "sysparam"
},
"rom0": {
"address": "0x008000",
"size": "992K",
"type": "app",
"subtype": "factory",
"filename": "$(RBOOT_ROM_0_BIN)"
}
}
}
}
5 changes: 3 additions & 2 deletions Sming/Arch/Esp8266/two-rom-mode.hw
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,14 @@
"base_config": "standard",
"partitions": {
"rom0": {
"subtype": "ota_0",
"size": "480K"
},
"rom1": {
"address": "0x80000",
"size": "512K",
"size": "488K",
"type": "app",
"subtype": "ota_0",
"subtype": "ota_1",
"filename": "$(RBOOT_ROM_1_BIN)"
}
}
Expand Down
25 changes: 24 additions & 1 deletion Sming/Components/Storage/README.rst
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,16 @@ If using this approach, remember to updated your project's ``component.mk`` with
and verify the layout is correct using ``make map``.


OTA updates
-----------

When planning OTA updates please check that the displayed partition map corresponds to your project.
For example, the partition table requires a free sector so must not overlap other partitions.

Your OTA update process must include a step to write the partition table to the correct location.

It is not necessary to update the bootloader. See :component:`rboot` for further information.


Custom configurations
---------------------
Expand Down Expand Up @@ -173,6 +183,19 @@ To customise the hardware configuration for a project, for example 'my_project':
This will flash everything: bootloader, partition table and all defined partitions (those with a ``filename`` entry).


.. note::

The build system isn't smart enough to track dependencies for partition build targets.

To rebuild these manually type::

make partbuild

These will be removed when ``make clean`` is run, but you can also clean them separately thus::

make part-clean


Partition maps
--------------

Expand Down Expand Up @@ -356,7 +379,7 @@ you can take advantage of the partition API to manage them as follows:
- Create an instance of your custom device and make a call to :cpp:func:`Storage::registerDevice`
in your ``init()`` function (or elsewhere if more appropriate).


API
---

Expand Down
8 changes: 6 additions & 2 deletions Sming/Components/Storage/Tools/hwconfig/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ def from_name(cls, name):
config.load(name)
if options != '':
config.parse_options(options.split(','))
config.resolve_expressions()
config.partitions.sort()
return config

Expand Down Expand Up @@ -73,6 +74,9 @@ def parse_options(self, options):
temp.pop('description', None)
self.parse_dict(temp)

def resolve_expressions(self):
self.partitions.offset = eval(str(self.partitions.offset))

def parse_dict(self, data):
base_config = data.pop('base_config', None)
if base_config is not None:
Expand All @@ -89,7 +93,7 @@ def parse_dict(self, data):
elif k == 'arch':
self.arch = v
elif k == 'partition_table_offset':
self.partitions.offset = parse_int(v)
self.partitions.offset = v
elif k == 'devices':
self.devices.parse_dict(v)
elif k == 'comment':
Expand Down Expand Up @@ -132,7 +136,7 @@ def buildVars(self):
return res

def verify(self, secure):
self.partitions.verify(self.arch, secure)
self.partitions.verify(self.arch, self.devices[0], secure)

def map(self):
return partition.Map(self.partitions, self.devices)
Expand Down
4 changes: 2 additions & 2 deletions Sming/Components/Storage/Tools/hwconfig/hwconfig.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ def handle_flashcheck(args, config, part):
for e in list:
addr, filename = e.split('=')
addr = int(addr, 0)
part = config.partitions.find_by_address(addr)
part = config.partitions.find_by_address(config.devices[0], addr)
if part is None:
raise InputError("No partition contains address 0x%08x" % addr)
if part.address != addr:
Expand Down Expand Up @@ -113,5 +113,5 @@ def main():
try:
main()
except InputError as e:
print(e, file=sys.stderr)
print("** ERROR! %s" % e, file=sys.stderr)
sys.exit(2)
40 changes: 31 additions & 9 deletions Sming/Components/Storage/Tools/hwconfig/partition.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,11 +16,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.

import struct, hashlib, storage, binascii
import struct, hashlib, storage, binascii, copy
from common import *

MAX_PARTITION_LENGTH = 0xC00 # 3K for partition data (96 entries) leaves 1K in a 4K sector for signature
MD5_PARTITION_BEGIN = b"\xEB\xEB" + b"\xFF" * 14 # The first 2 bytes are like magic numbers for MD5 sum
FLASH_SECTOR_SIZE = 0x1000
PARTITION_TABLE_SIZE = 0x1000 # Size of partition table
PARTITION_ENTRY_SIZE = 32

Expand Down Expand Up @@ -84,6 +85,7 @@
"nvs_keys": 0x04,
"efuse": 0x05,
"sysparam": 0x40,
"rfcal": 0x41,
"esphttpd": 0x80,
"fat": 0x81,
"spiffs": 0x82,
Expand Down Expand Up @@ -154,6 +156,10 @@ def offset_str(self):
def buildVars(self):
dict = {}
dict['PARTITION_NAMES'] = " ".join(p.name for p in self)
buildparts = [p for p in self if p.build is not None]
dict['PARTITIONS_WITH_TARGETS'] = " ".join(p.name for p in buildparts)
dict['PARTITION_BUILD_TARGETS'] = " ".join(p.filename for p in buildparts)

for p in self:
dict.update(p.buildVars())
return dict
Expand Down Expand Up @@ -198,19 +204,28 @@ def find_by_name(self, name):
return p
return None

def find_by_address(self, addr):
def find_by_address(self, device, addr):
for p in self:
if p.contains(addr):
if p.device == device and p.contains(addr):
return p
return None

def verify(self, arch, secure):
def verify(self, arch, spiFlash, secure):
"""Verify partition layout
"""
# verify each partition individually
for p in self:
p.verify(arch, secure)

if self.offset % FLASH_SECTOR_SIZE != 0:
raise InputError("Partition table offset not aligned to flash sector")

p = self.find_by_address(spiFlash, self.offset)
if p is None:
p = self.find_by_address(spiFlash, self.offset + PARTITION_TABLE_SIZE - 1)
if not p is None:
raise InputError("Partition table conflict with '%s'" % p.name)

# check on duplicate name
names = [p.name for p in self]
duplicates = set(n for n in names if names.count(n) > 1)
Expand All @@ -224,7 +239,10 @@ def verify(self, arch, secure):
raise InputError("Partition names must be unique")

# check for overlaps
minPartitionAddress = self.offset + PARTITION_TABLE_SIZE
if arch == 'Esp32':
minPartitionAddress = self.offset + PARTITION_TABLE_SIZE
else:
minPartitionAddress = 0x00002000
dev = ''
last = None
for p in self:
Expand Down Expand Up @@ -345,7 +363,7 @@ def parse_dict(self, data, devices):
if k == 'device':
self.device = devices.find_by_name(v)
elif k == 'address':
self.address = parse_int(v)
self.address = eval(str(v))
elif k == 'size':
self.size = parse_int(v)
elif k == 'filename':
Expand Down Expand Up @@ -523,13 +541,17 @@ def add_unused(address, last_end):
if address > last_end + 1:
add('(unused)', last_end + 1, address - last_end - 1)

partitions = copy.copy(table)

if table.offset == 0:
last = None
else:
add("Boot Sector", 0, table.offset)
last = add("Partition Table", table.offset, PARTITION_TABLE_SIZE)
last = add('Boot Sector', 0, min(table.offset, partitions[0].address))
p = Entry(device, 'Partition Table', table.offset, PARTITION_TABLE_SIZE, 0xff, 0xff)
partitions.append(p)
partitions.sort()

for p in table:
for p in partitions:
if last is not None:
if p.device != last.device:
add_unused(last.device.size, last.end())
Expand Down
Loading

0 comments on commit fec464b

Please sign in to comment.