From 931d30ea2efd5a0090588b0da25d95af97d1f1e6 Mon Sep 17 00:00:00 2001 From: Dmitry Ilyin <6576495+widgetii@users.noreply.github.com> Date: Wed, 22 Apr 2026 09:47:33 +0300 Subject: [PATCH 1/3] Add UBI/UBIFS-aware backup and restore for NAND cameras On NAND cameras with UBI/UBIFS, raw MTD dumps include physical block mappings that are chip-specific and cannot be safely restored. This adds UBI volume-level backup and restore using kernel sysfs and ioctls. Detection: scan /sys/class/ubi/ to find UBI devices attached to MTD partitions and enumerate their volumes with data_bytes and names. Backup: read UBI volumes from /dev/ubiN_V instead of raw /dev/mtdblockN for UBI-managed partitions. Each volume becomes a separate data block. YAML output: add dump_type, ubi_device, and ubi_volumes array with per-volume vol_id, vol_name, data_bytes, and sha1. Restore: UBI-aware path using ioctls (UBI_IOCDET, UBI_IOCATT, UBI_IOCMKVOL, UBI_IOCVOLUP) to detach, erase, reattach, create volumes, and write data. Raw MTD partitions use existing path. Tested on HI3516AV200 with 128MB SPI NAND and 5 UBI volumes. Closes #141 Co-Authored-By: Claude Opus 4.6 (1M context) --- src/backup.c | 240 +++++++++++++++++++++++++++++++++++++++++++++++++-- src/mtd.c | 142 +++++++++++++++++++++++++++++- src/mtd.h | 16 +++- 3 files changed, 391 insertions(+), 7 deletions(-) diff --git a/src/backup.c b/src/backup.c index 91ee732..d198ee3 100644 --- a/src/backup.c +++ b/src/backup.c @@ -23,6 +23,8 @@ #include #include +#include + #include "backup.h" #include "boards/common.h" #include "boards/xm.h" @@ -50,6 +52,23 @@ static bool cb_mtd_backup(int i, const char *name, struct mtd_info_user *mtd, void *ctx) { mtd_backup_ctx *c = (mtd_backup_ctx *)ctx; + int ubi_num = find_ubi_for_mtd(i); + if (ubi_num >= 0) { + ubi_vol_info_t vols[MAX_UBI_VOLS]; + int nvols = enum_ubi_volumes(ubi_num, vols, MAX_UBI_VOLS); + for (int v = 0; v < nvols && c->count < c->cap; v++) { + size_t out_len = 0; + char *buf = read_ubi_volume(ubi_num, vols[v].vol_id, + vols[v].data_bytes, &out_len); + if (!buf) + continue; + c->blocks[c->count].data = buf; + c->blocks[c->count].len = out_len; + c->count++; + } + return true; + } + int fd; char *addr = open_mtdblock(i, &fd, mtd->size, 0); if (!addr) @@ -176,6 +195,10 @@ typedef struct { char sha1[9]; char name[64]; char *data; + bool is_ubi; + int ubi_device; + int vol_id; + char vol_name[64]; } stored_mtd_t; static int yaml_parseblock(char *start, int indent, stored_mtd_t *mi) { @@ -189,6 +212,12 @@ static int yaml_parseblock(char *start, int indent, stored_mtd_t *mi) { int i = -1; int rootlvl = -1; size_t offset = 0; + bool in_ubi_vols = false; + int ubi_vols_lvl = -1; + int cur_ubi_device = -1; + char cur_part_name[64] = {0}; + size_t cur_part_size = 0; + int ubi_parent = -1; while (ptr < start + len) { if (linestart) { @@ -201,9 +230,23 @@ static int yaml_parseblock(char *start, int indent, stored_mtd_t *mi) { if (rootlvl == -1) rootlvl = spaces; if (rootlvl == spaces) { + in_ubi_vols = false; + ubi_vols_lvl = -1; + cur_ubi_device = -1; + memset(cur_part_name, 0, sizeof(cur_part_name)); + cur_part_size = 0; + ubi_parent = -1; + i++; + if (i == MAX_MTDBLOCKS) + break; + } else if (in_ubi_vols && spaces == ubi_vols_lvl) { i++; if (i == MAX_MTDBLOCKS) break; + mi[i].is_ubi = true; + mi[i].ubi_device = cur_ubi_device; + strncpy(mi[i].name, cur_part_name, + sizeof(mi[i].name) - 1); } } linestart = false; @@ -213,16 +256,46 @@ static int yaml_parseblock(char *start, int indent, stored_mtd_t *mi) { } } if (*ptr == '\n') { - if (param && spaces == rootlvl) { + if (param && in_ubi_vols && spaces == ubi_vols_lvl) { + if (!strncmp(param, "data_bytes: ", 12)) { + mi[i].size = strtoul(param + 12, NULL, 16); + } else if (!strncmp(param, "vol_name: ", 10)) { + size_t n = + MIN(ptr - param - 10, (int)sizeof(mi[i].vol_name) - 1); + memcpy(mi[i].vol_name, param + 10, n); + } else if (!strncmp(param, "vol_id: ", 8)) { + mi[i].vol_id = atoi(param + 8); + } else if (!strncmp(param, "sha1: ", 6)) { + memcpy(mi[i].sha1, param + 6, MIN(ptr - param - 6, 8)); + } + } else if (param && spaces == rootlvl) { if (!strncmp(param, "size: ", 6)) { mi[i].off_flashb = offset; mi[i].size = strtoul(param + 6, NULL, 16); + cur_part_size = mi[i].size; offset += mi[i].size; } else if (!strncmp(param, "name: ", 6)) { - memcpy(mi[i].name, param + 6, - MIN(ptr - param - 6, (int)sizeof(mi[i]) - 1)); - } else if (!strncmp(param, "sha1: ", 6)) + size_t n = + MIN(ptr - param - 6, (int)sizeof(mi[i].name) - 1); + memcpy(mi[i].name, param + 6, n); + memcpy(cur_part_name, param + 6, n); + cur_part_name[n] = '\0'; + } else if (!strncmp(param, "sha1: ", 6)) { memcpy(mi[i].sha1, param + 6, MIN(ptr - param - 6, 8)); + } else if (!strncmp(param, "dump_type: ubifs", 16)) { + mi[i].is_ubi = true; + ubi_parent = i; + } else if (!strncmp(param, "ubi_device: ", 12)) { + cur_ubi_device = atoi(param + 12); + mi[i].ubi_device = cur_ubi_device; + } else if (!strncmp(param, "ubi_volumes:", 12)) { + in_ubi_vols = true; + ubi_vols_lvl = -1; + // Remove the placeholder partition entry — volumes + // will replace it. Rewind index so first volume + // overwrites the parent entry. + i--; + } } linestart = true; spaces = 0; @@ -325,7 +398,9 @@ static bool umount_all() { if (sscanf(mount, "%s %s %s %s", dev, path, fs, attrs)) { if (!strncmp(dev, "/dev/mtdblock", 13) && strstr(attrs, "rw")) umount_fs(path); - else if (!strcmp(fs, "squashfs") || (!strcmp(fs, "cramfs"))) + else if (!strcmp(fs, "squashfs") || !strcmp(fs, "cramfs")) + umount_fs(path); + else if (!strcmp(fs, "ubifs")) umount_fs(path); } } @@ -370,12 +445,167 @@ static int map_old_new_mtd(int old_num, size_t old_offset, size_t *new_offset, return -1; } +static bool ubi_restore_partition(int mtd_num, stored_mtd_t *vols, int nvols, + bool simulate) { + if (simulate) + return true; + + char devpath[64]; + + // Detach existing UBI device if any + int ubi_num = find_ubi_for_mtd(mtd_num); + if (ubi_num >= 0) { + int ctrl_fd = open("/dev/ubi_ctrl", O_RDONLY); + if (ctrl_fd >= 0) { + int32_t dev = ubi_num; + ioctl(ctrl_fd, UBI_IOCDET, &dev); + close(ctrl_fd); + } + } + + // Erase entire MTD partition + snprintf(devpath, sizeof(devpath), "/dev/mtd%d", mtd_num); + int mtd_fd = open(devpath, O_RDWR); + if (mtd_fd < 0) { + fprintf(stderr, "Cannot open %s\n", devpath); + return false; + } + struct mtd_info_user mtd_info; + if (ioctl(mtd_fd, MEMGETINFO, &mtd_info) == 0) { + for (uint32_t off = 0; off < mtd_info.size; off += mtd_info.erasesize) { + mtd_erase_block(mtd_fd, off, mtd_info.erasesize); + } + } + close(mtd_fd); + + // Attach UBI + int ctrl_fd = open("/dev/ubi_ctrl", O_RDONLY); + if (ctrl_fd < 0) { + fprintf(stderr, "Cannot open /dev/ubi_ctrl\n"); + return false; + } + + struct ubi_attach_req att_req; + memset(&att_req, 0, sizeof(att_req)); + att_req.ubi_num = UBI_DEV_NUM_AUTO; + att_req.mtd_num = mtd_num; + if (ioctl(ctrl_fd, UBI_IOCATT, &att_req) < 0) { + fprintf(stderr, "UBI attach failed for mtd%d: %s\n", mtd_num, + strerror(errno)); + close(ctrl_fd); + return false; + } + close(ctrl_fd); + + ubi_num = att_req.ubi_num; + + // Create and write each volume + snprintf(devpath, sizeof(devpath), "/dev/ubi%d", ubi_num); + int ubi_fd = open(devpath, O_RDONLY); + if (ubi_fd < 0) { + fprintf(stderr, "Cannot open %s\n", devpath); + return false; + } + + for (int v = 0; v < nvols; v++) { + struct ubi_mkvol_req mk_req; + memset(&mk_req, 0, sizeof(mk_req)); + mk_req.vol_id = vols[v].vol_id; + mk_req.alignment = 1; + mk_req.bytes = vols[v].size; + mk_req.vol_type = UBI_DYNAMIC_VOLUME; + mk_req.name_len = strlen(vols[v].vol_name); + strncpy(mk_req.name, vols[v].vol_name, UBI_MAX_VOLUME_NAME); + + if (ioctl(ubi_fd, UBI_IOCMKVOL, &mk_req) < 0) { + fprintf(stderr, "UBI mkvol failed for '%s': %s\n", vols[v].vol_name, + strerror(errno)); + close(ubi_fd); + return false; + } + + // Write volume data + char vol_path[64]; + snprintf(vol_path, sizeof(vol_path), "/dev/ubi%d_%d", ubi_num, + vols[v].vol_id); + int vol_fd = open(vol_path, O_RDWR); + if (vol_fd < 0) { + fprintf(stderr, "Cannot open %s\n", vol_path); + close(ubi_fd); + return false; + } + + int64_t bytes = vols[v].size; + if (ioctl(vol_fd, UBI_IOCVOLUP, &bytes) < 0) { + fprintf(stderr, "UBI volup failed for '%s': %s\n", vols[v].vol_name, + strerror(errno)); + close(vol_fd); + close(ubi_fd); + return false; + } + + size_t written = 0; + while (written < vols[v].size) { + ssize_t n = + write(vol_fd, vols[v].data + written, vols[v].size - written); + if (n <= 0) { + fprintf(stderr, "UBI write failed for '%s': %s\n", + vols[v].vol_name, strerror(errno)); + close(vol_fd); + close(ubi_fd); + return false; + } + written += n; + } + close(vol_fd); + printf(" Wrote UBI volume '%s' (%zu bytes)\n", vols[v].vol_name, + vols[v].size); + } + + close(ubi_fd); + return true; +} + static bool do_flash(const char *phase, stored_mtd_t *mtdbackup, mtd_restore_ctx_t *mtd, bool skip_env, bool simulate) { for (int i = 0; i < MAX_MTDBLOCKS; i++) { if (!*mtdbackup[i].name) continue; + if (mtdbackup[i].is_ubi) { + // Collect consecutive UBI volume entries for the same partition + int first = i; + int nvols = 0; + while (i < MAX_MTDBLOCKS && mtdbackup[i].is_ubi && + !strcmp(mtdbackup[i].name, mtdbackup[first].name)) { + nvols++; + i++; + } + i--; // will be incremented by for loop + + // Find which MTD device this partition maps to + int mtd_num = -1; + for (int m = 0; m < MAX_MTDBLOCKS; m++) { + if (!strcmp(mtd->part[m].name, mtdbackup[first].name)) { + mtd_num = m; + break; + } + } + if (mtd_num < 0) { + fprintf(stderr, "Cannot find MTD for UBI partition '%s'\n", + mtdbackup[first].name); + return false; + } + + printf("%s UBI partition %s (%d volumes)\n", phase, + mtdbackup[first].name, nvols); + + if (!ubi_restore_partition(mtd_num, &mtdbackup[first], nvols, + simulate)) + return false; + continue; + } + printf("%s %s\n", phase, mtdbackup[i].name); size_t chunk = mtd->erasesize; int cnt = mtdbackup[i].size / chunk; diff --git a/src/mtd.c b/src/mtd.c index 52fced7..56d4d51 100644 --- a/src/mtd.c +++ b/src/mtd.c @@ -1,3 +1,4 @@ +#include #include #include #include @@ -120,6 +121,112 @@ char *open_mtdblock(int i, int *fd, uint32_t size, int flags) { return addr; } +int find_ubi_for_mtd(int mtd_num) { + DIR *d = opendir("/sys/class/ubi"); + if (!d) + return -1; + struct dirent *de; + while ((de = readdir(d))) { + if (strncmp(de->d_name, "ubi", 3) != 0) + continue; + if (strchr(de->d_name, '_')) + continue; + char path[128]; + snprintf(path, sizeof(path), "/sys/class/ubi/%s/mtd_num", de->d_name); + FILE *f = fopen(path, "r"); + if (f) { + int num; + if (fscanf(f, "%d", &num) == 1 && num == mtd_num) { + fclose(f); + closedir(d); + int ubi_num; + sscanf(de->d_name, "ubi%d", &ubi_num); + return ubi_num; + } + fclose(f); + } + } + closedir(d); + return -1; +} + +int enum_ubi_volumes(int ubi_num, ubi_vol_info_t *vols, int max_vols) { + char base[128]; + snprintf(base, sizeof(base), "/sys/class/ubi/ubi%d", ubi_num); + + DIR *d = opendir(base); + if (!d) + return 0; + + int count = 0; + char prefix[16]; + snprintf(prefix, sizeof(prefix), "ubi%d_", ubi_num); + size_t plen = strlen(prefix); + + struct dirent *de; + while ((de = readdir(d)) && count < max_vols) { + if (strncmp(de->d_name, prefix, plen) != 0) + continue; + int vol_id = atoi(de->d_name + plen); + + char path[192]; + snprintf(path, sizeof(path), "%s/%s/data_bytes", base, de->d_name); + FILE *f = fopen(path, "r"); + if (!f) + continue; + long long data_bytes = 0; + fscanf(f, "%lld", &data_bytes); + fclose(f); + + snprintf(path, sizeof(path), "%s/%s/name", base, de->d_name); + f = fopen(path, "r"); + char name[64] = {0}; + if (f) { + if (fgets(name, sizeof(name), f)) { + size_t len = strlen(name); + if (len > 0 && name[len - 1] == '\n') + name[len - 1] = '\0'; + } + fclose(f); + } + + vols[count].vol_id = vol_id; + vols[count].data_bytes = data_bytes; + strncpy(vols[count].name, name, sizeof(vols[count].name) - 1); + count++; + } + closedir(d); + return count; +} + +char *read_ubi_volume(int ubi_num, int vol_id, size_t data_bytes, + size_t *out_len) { + char devpath[64]; + snprintf(devpath, sizeof(devpath), "/dev/ubi%d_%d", ubi_num, vol_id); + + int fd = open(devpath, O_RDONLY); + if (fd == -1) + return NULL; + + char *buf = malloc(data_bytes); + if (!buf) { + close(fd); + return NULL; + } + + size_t total = 0; + while (total < data_bytes) { + ssize_t n = read(fd, buf + total, data_bytes - total); + if (n <= 0) + break; + total += n; + } + close(fd); + + *out_len = total; + return buf; +} + static bool uenv_detected; static bool examine_part(int part_num, size_t size, size_t erasesize, @@ -218,7 +325,40 @@ static bool cb_mtd_info(int i, const char *name, struct mtd_info_user *mtd, if (i < MAX_MPOINTS && *c->mpoints[i].path) { ADD_PARAM("path", c->mpoints[i].path); } - if (!c->mpoints[i].rw) { + + int ubi_num = find_ubi_for_mtd(i); + if (ubi_num >= 0) { + ADD_PARAM("dump_type", "ubifs"); + ADD_PARAM_FMT("ubi_device", "%d", ubi_num); + + ubi_vol_info_t vols[MAX_UBI_VOLS]; + int nvols = enum_ubi_volumes(ubi_num, vols, MAX_UBI_VOLS); + if (nvols > 0) { + cJSON *j_vols = cJSON_CreateArray(); + for (int v = 0; v < nvols; v++) { + cJSON *j_vol = cJSON_CreateObject(); + cJSON_AddItemToArray(j_vols, j_vol); + { + cJSON *j_inner = j_vol; + ADD_PARAM_FMT("vol_id", "%d", vols[v].vol_id); + ADD_PARAM("vol_name", vols[v].name); + ADD_PARAM_FMT("data_bytes", "0x%llx", vols[v].data_bytes); + + size_t out_len = 0; + char *vdata = read_ubi_volume(ubi_num, vols[v].vol_id, + vols[v].data_bytes, &out_len); + if (vdata && out_len > 0) { + char digest[21] = {0}; + SHA1(digest, vdata, out_len); + uint32_t sha1v = ntohl(*(uint32_t *)&digest); + ADD_PARAM_FMT("sha1", "%.8x", sha1v); + } + free(vdata); + } + } + cJSON_AddItemToObject(j_inner, "ubi_volumes", j_vols); + } + } else if (!c->mpoints[i].rw) { cJSON *contains = NULL; uint32_t sha1 = 0; if (examine_part(i, mtd->size, mtd->erasesize, &sha1, &contains)) { diff --git a/src/mtd.h b/src/mtd.h index 3c7350b..c5ae926 100644 --- a/src/mtd.h +++ b/src/mtd.h @@ -1,8 +1,8 @@ #ifndef MTD_H #define MTD_H -#include #include "cjson/cJSON.h" +#include typedef bool (*cb_mtd)(int i, const char *name, struct mtd_info_user *mtd, void *ctx); @@ -13,5 +13,19 @@ void enum_mtd_info(void *ctx, cb_mtd cb); bool mtd_write(int mtd, uint32_t offset, uint32_t erasesize, const char *data, size_t size); int mtd_unlock_cmd(); +int mtd_erase_block(int fd, int offset, int erasesize); + +#define MAX_UBI_VOLS 8 + +typedef struct { + int vol_id; + char name[64]; + long long data_bytes; +} ubi_vol_info_t; + +int find_ubi_for_mtd(int mtd_num); +int enum_ubi_volumes(int ubi_num, ubi_vol_info_t *vols, int max_vols); +char *read_ubi_volume(int ubi_num, int vol_id, size_t data_bytes, + size_t *out_len); #endif /* MTD_H */ From 627bb15fc41da6255a2fe869f2db3952ac36b532 Mon Sep 17 00:00:00 2001 From: Dmitry Ilyin <6576495+widgetii@users.noreply.github.com> Date: Wed, 22 Apr 2026 10:01:21 +0300 Subject: [PATCH 2/3] Add CI build check for pull requests Cross-compile for both ARM and MIPS targets on PRs to catch build errors before merge. Uses the same OpenIPC toolchains as the release workflows but skips packaging, uploads, and notifications. Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/pr-build-check.yml | 40 ++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 .github/workflows/pr-build-check.yml diff --git a/.github/workflows/pr-build-check.yml b/.github/workflows/pr-build-check.yml new file mode 100644 index 0000000..eb2a932 --- /dev/null +++ b/.github/workflows/pr-build-check.yml @@ -0,0 +1,40 @@ +name: PR Build Check + +on: + pull_request: + branches: [master] + +jobs: + build-arm: + name: Build ARM (musl static) + runs-on: ubuntu-latest + env: + ARCHIVE: toolchain.hisilicon-hi3516cv100 + PLATFORM: arm-openipc-linux-musleabi_sdk-buildroot + TOOLCHAIN: arm-openipc-linux-musleabi + steps: + - uses: actions/checkout@v4 + - name: Download toolchain and build + run: | + wget -qO- https://github.com/OpenIPC/firmware/releases/download/toolchain/$ARCHIVE.tgz | \ + tar xfz - -C /opt + export PATH=/opt/$PLATFORM/bin:$PATH + cmake -H. -Bbuild -DCMAKE_C_COMPILER=${TOOLCHAIN}-gcc -DCMAKE_BUILD_TYPE=Release + cmake --build build + + build-mips: + name: Build MIPS (musl static) + runs-on: ubuntu-latest + env: + ARCHIVE: toolchain.ingenic-t31 + PLATFORM: mipsel-openipc-linux-musl_sdk-buildroot + TOOLCHAIN: mipsel-openipc-linux-musl + steps: + - uses: actions/checkout@v4 + - name: Download toolchain and build + run: | + wget -qO- https://github.com/OpenIPC/firmware/releases/download/toolchain/$ARCHIVE.tgz | \ + tar xfz - -C /opt + export PATH=/opt/$PLATFORM/bin:$PATH + cmake -H. -Bbuild -DCMAKE_C_COMPILER=${TOOLCHAIN}-gcc -DCMAKE_BUILD_TYPE=Release + cmake --build build From 9178fed66bf9e2c4b8487219542b451e2de4ae84 Mon Sep 17 00:00:00 2001 From: Dmitry Ilyin <6576495+widgetii@users.noreply.github.com> Date: Wed, 22 Apr 2026 10:04:23 +0300 Subject: [PATCH 3/3] Inline UBI ioctl definitions to fix old musl toolchain build The OpenIPC ARM musl toolchain has a broken where __packed is not defined, causing build failures. Replace the include with minimal inlined struct and ioctl definitions guarded by ifndef UBI_IOC_MAGIC. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/backup.c | 2 -- src/mtd.h | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/src/backup.c b/src/backup.c index d198ee3..ff18ba0 100644 --- a/src/backup.c +++ b/src/backup.c @@ -23,8 +23,6 @@ #include #include -#include - #include "backup.h" #include "boards/common.h" #include "boards/xm.h" diff --git a/src/mtd.h b/src/mtd.h index c5ae926..729d639 100644 --- a/src/mtd.h +++ b/src/mtd.h @@ -15,6 +15,46 @@ bool mtd_write(int mtd, uint32_t offset, uint32_t erasesize, const char *data, int mtd_unlock_cmd(); int mtd_erase_block(int fd, int offset, int erasesize); +// UBI ioctl definitions — inlined to avoid broken in old +// musl toolchains where __packed is not defined as __attribute__((packed)). +#include + +#ifndef UBI_IOC_MAGIC +#define UBI_IOC_MAGIC 'o' +#define UBI_CTRL_IOC_MAGIC 'o' +#define UBI_VOL_IOC_MAGIC 'O' + +#define UBI_DEV_NUM_AUTO (-1) +#define UBI_MAX_VOLUME_NAME 127 +#define UBI_DYNAMIC_VOLUME 3 + +struct ubi_attach_req { + int32_t ubi_num; + int32_t mtd_num; + int32_t vid_hdr_offset; + int16_t max_beb_per1024; + int8_t disable_fm; + int8_t need_resv_pool; + int8_t padding[8]; +}; + +struct ubi_mkvol_req { + int32_t vol_id; + int32_t alignment; + int64_t bytes; + int8_t vol_type; + uint8_t flags; + int16_t name_len; + int8_t padding2[4]; + char name[UBI_MAX_VOLUME_NAME + 1]; +} __attribute__((packed)); + +#define UBI_IOCMKVOL _IOW(UBI_IOC_MAGIC, 0, struct ubi_mkvol_req) +#define UBI_IOCATT _IOW(UBI_CTRL_IOC_MAGIC, 64, struct ubi_attach_req) +#define UBI_IOCDET _IOW(UBI_CTRL_IOC_MAGIC, 65, int32_t) +#define UBI_IOCVOLUP _IOW(UBI_VOL_IOC_MAGIC, 0, int64_t) +#endif + #define MAX_UBI_VOLS 8 typedef struct {