diff options
author | Doug Zongker <dougz@android.com> | 2010-02-17 16:11:44 -0800 |
---|---|---|
committer | Doug Zongker <dougz@android.com> | 2010-02-18 14:22:12 -0800 |
commit | 512536a54a1a211a9f582e76cbf12850dc7d5466 (patch) | |
tree | 724012f5ea1a3053adecb512baf342490bb94d02 | |
parent | 21854ccdb250e6e81311b4317934e8c953b252a8 (diff) | |
download | bootable_recovery-512536a54a1a211a9f582e76cbf12850dc7d5466.zip bootable_recovery-512536a54a1a211a9f582e76cbf12850dc7d5466.tar.gz bootable_recovery-512536a54a1a211a9f582e76cbf12850dc7d5466.tar.bz2 |
relocate applypatch; add type system and new functions to edify
- Move applypatch to this package (from build).
- Add a rudimentary type system to edify: instead of just returning a
char*, functions now return a Value*, which is a struct that can
carry different types of value (currently just STRING and BLOB).
Convert all functions to this new scheme.
- Change the one-argument form of package_extract_file to return a
Value of the new BLOB type.
- Add read_file() to load a local file and return a blob, and
sha1_check() to test a blob (or string) against a set of possible
sha1s. read_file() uses the file-loading code from applypatch so it
can read MTD partitions as well.
This is the start of better integration between applypatch and the
rest of edify.
b/2361316 - VZW Issue PP628: Continuous reset to Droid logo:
framework-res.apk update failed (CR LIBtt59130)
Change-Id: Ibd038074749a4d515de1f115c498c6c589ee91e5
-rw-r--r-- | Android.mk | 1 | ||||
-rw-r--r-- | applypatch/Android.mk | 59 | ||||
-rw-r--r-- | applypatch/applypatch.c | 900 | ||||
-rw-r--r-- | applypatch/applypatch.h | 70 | ||||
-rwxr-xr-x | applypatch/applypatch.sh | 345 | ||||
-rw-r--r-- | applypatch/bsdiff.c | 410 | ||||
-rw-r--r-- | applypatch/bspatch.c | 252 | ||||
-rw-r--r-- | applypatch/freecache.c | 172 | ||||
-rw-r--r-- | applypatch/imgdiff.c | 1010 | ||||
-rw-r--r-- | applypatch/imgdiff.h | 30 | ||||
-rwxr-xr-x | applypatch/imgdiff_test.sh | 118 | ||||
-rw-r--r-- | applypatch/imgpatch.c | 364 | ||||
-rw-r--r-- | applypatch/main.c | 60 | ||||
-rw-r--r-- | applypatch/testdata/new.file | bin | 0 -> 1388877 bytes | |||
-rw-r--r-- | applypatch/testdata/old.file | bin | 0 -> 1348051 bytes | |||
-rw-r--r-- | applypatch/testdata/patch.bsdiff | bin | 0 -> 57476 bytes | |||
-rw-r--r-- | applypatch/utils.c | 62 | ||||
-rw-r--r-- | applypatch/utils.h | 30 | ||||
-rw-r--r-- | edify/expr.c | 161 | ||||
-rw-r--r-- | edify/expr.h | 65 | ||||
-rw-r--r-- | updater/install.c | 236 | ||||
-rw-r--r-- | updater/updater.c | 6 |
22 files changed, 4239 insertions, 112 deletions
@@ -65,6 +65,7 @@ include $(commands_recovery_local_path)/mtdutils/Android.mk include $(commands_recovery_local_path)/tools/Android.mk include $(commands_recovery_local_path)/edify/Android.mk include $(commands_recovery_local_path)/updater/Android.mk +include $(commands_recovery_local_path)/applypatch/Android.mk commands_recovery_local_path := endif # TARGET_ARCH == arm diff --git a/applypatch/Android.mk b/applypatch/Android.mk new file mode 100644 index 0000000..d20d6c8 --- /dev/null +++ b/applypatch/Android.mk @@ -0,0 +1,59 @@ +# Copyright (C) 2008 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ifneq ($(TARGET_SIMULATOR),true) + +LOCAL_PATH := $(call my-dir) +include $(CLEAR_VARS) + +LOCAL_SRC_FILES := applypatch.c bspatch.c freecache.c imgpatch.c utils.c +LOCAL_MODULE := libapplypatch +LOCAL_MODULE_TAGS := eng +LOCAL_C_INCLUDES += external/bzip2 external/zlib bootable/recovery +LOCAL_STATIC_LIBRARIES += libmtdutils libmincrypt libbz libz + +include $(BUILD_STATIC_LIBRARY) + +include $(CLEAR_VARS) + +LOCAL_SRC_FILES := main.c +LOCAL_MODULE := applypatch +LOCAL_STATIC_LIBRARIES += libapplypatch libmtdutils libmincrypt libbz +LOCAL_SHARED_LIBRARIES += libz libcutils libstdc++ libc + +include $(BUILD_EXECUTABLE) + +include $(CLEAR_VARS) + +LOCAL_SRC_FILES := main.c +LOCAL_MODULE := applypatch_static +LOCAL_FORCE_STATIC_EXECUTABLE := true +LOCAL_MODULE_TAGS := eng +LOCAL_STATIC_LIBRARIES += libapplypatch libmtdutils libmincrypt libbz +LOCAL_STATIC_LIBRARIES += libz libcutils libstdc++ libc + +include $(BUILD_EXECUTABLE) + +include $(CLEAR_VARS) + +LOCAL_SRC_FILES := imgdiff.c utils.c bsdiff.c +LOCAL_MODULE := imgdiff +LOCAL_FORCE_STATIC_EXECUTABLE := true +LOCAL_MODULE_TAGS := eng +LOCAL_C_INCLUDES += external/zlib external/bzip2 +LOCAL_STATIC_LIBRARIES += libz libbz + +include $(BUILD_HOST_EXECUTABLE) + +endif # !TARGET_SIMULATOR diff --git a/applypatch/applypatch.c b/applypatch/applypatch.c new file mode 100644 index 0000000..daf3729 --- /dev/null +++ b/applypatch/applypatch.c @@ -0,0 +1,900 @@ +/* + * Copyright (C) 2008 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <errno.h> +#include <libgen.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <sys/stat.h> +#include <sys/statfs.h> +#include <sys/types.h> +#include <fcntl.h> +#include <unistd.h> + +#include "mincrypt/sha.h" +#include "applypatch.h" +#include "mtdutils/mtdutils.h" + +int SaveFileContents(const char* filename, FileContents file); +int LoadMTDContents(const char* filename, FileContents* file); +int ParseSha1(const char* str, uint8_t* digest); +ssize_t FileSink(unsigned char* data, ssize_t len, void* token); + +static int mtd_partitions_scanned = 0; + +// Read a file into memory; store it and its associated metadata in +// *file. Return 0 on success. +int LoadFileContents(const char* filename, FileContents* file) { + file->data = NULL; + + // A special 'filename' beginning with "MTD:" means to load the + // contents of an MTD partition. + if (strncmp(filename, "MTD:", 4) == 0) { + return LoadMTDContents(filename, file); + } + + if (stat(filename, &file->st) != 0) { + printf("failed to stat \"%s\": %s\n", filename, strerror(errno)); + return -1; + } + + file->size = file->st.st_size; + file->data = malloc(file->size); + + FILE* f = fopen(filename, "rb"); + if (f == NULL) { + printf("failed to open \"%s\": %s\n", filename, strerror(errno)); + free(file->data); + file->data = NULL; + return -1; + } + + size_t bytes_read = fread(file->data, 1, file->size, f); + if (bytes_read != file->size) { + printf("short read of \"%s\" (%d bytes of %d)\n", + filename, bytes_read, file->size); + free(file->data); + file->data = NULL; + return -1; + } + fclose(f); + + SHA(file->data, file->size, file->sha1); + return 0; +} + +static size_t* size_array; +// comparison function for qsort()ing an int array of indexes into +// size_array[]. +static int compare_size_indices(const void* a, const void* b) { + int aa = *(int*)a; + int bb = *(int*)b; + if (size_array[aa] < size_array[bb]) { + return -1; + } else if (size_array[aa] > size_array[bb]) { + return 1; + } else { + return 0; + } +} + +void FreeFileContents(FileContents* file) { + if (file) free(file->data); + free(file); +} + +// Load the contents of an MTD partition into the provided +// FileContents. filename should be a string of the form +// "MTD:<partition_name>:<size_1>:<sha1_1>:<size_2>:<sha1_2>:...". +// The smallest size_n bytes for which that prefix of the mtd contents +// has the corresponding sha1 hash will be loaded. It is acceptable +// for a size value to be repeated with different sha1s. Will return +// 0 on success. +// +// This complexity is needed because if an OTA installation is +// interrupted, the partition might contain either the source or the +// target data, which might be of different lengths. We need to know +// the length in order to read from MTD (there is no "end-of-file" +// marker), so the caller must specify the possible lengths and the +// hash of the data, and we'll do the load expecting to find one of +// those hashes. +int LoadMTDContents(const char* filename, FileContents* file) { + char* copy = strdup(filename); + const char* magic = strtok(copy, ":"); + if (strcmp(magic, "MTD") != 0) { + printf("LoadMTDContents called with bad filename (%s)\n", + filename); + return -1; + } + const char* partition = strtok(NULL, ":"); + + int i; + int colons = 0; + for (i = 0; filename[i] != '\0'; ++i) { + if (filename[i] == ':') { + ++colons; + } + } + if (colons < 3 || colons%2 == 0) { + printf("LoadMTDContents called with bad filename (%s)\n", + filename); + } + + int pairs = (colons-1)/2; // # of (size,sha1) pairs in filename + int* index = malloc(pairs * sizeof(int)); + size_t* size = malloc(pairs * sizeof(size_t)); + char** sha1sum = malloc(pairs * sizeof(char*)); + + for (i = 0; i < pairs; ++i) { + const char* size_str = strtok(NULL, ":"); + size[i] = strtol(size_str, NULL, 10); + if (size[i] == 0) { + printf("LoadMTDContents called with bad size (%s)\n", filename); + return -1; + } + sha1sum[i] = strtok(NULL, ":"); + index[i] = i; + } + + // sort the index[] array so it indexes the pairs in order of + // increasing size. + size_array = size; + qsort(index, pairs, sizeof(int), compare_size_indices); + + if (!mtd_partitions_scanned) { + mtd_scan_partitions(); + mtd_partitions_scanned = 1; + } + + const MtdPartition* mtd = mtd_find_partition_by_name(partition); + if (mtd == NULL) { + printf("mtd partition \"%s\" not found (loading %s)\n", + partition, filename); + return -1; + } + + MtdReadContext* ctx = mtd_read_partition(mtd); + if (ctx == NULL) { + printf("failed to initialize read of mtd partition \"%s\"\n", + partition); + return -1; + } + + SHA_CTX sha_ctx; + SHA_init(&sha_ctx); + uint8_t parsed_sha[SHA_DIGEST_SIZE]; + + // allocate enough memory to hold the largest size. + file->data = malloc(size[index[pairs-1]]); + char* p = (char*)file->data; + file->size = 0; // # bytes read so far + + for (i = 0; i < pairs; ++i) { + // Read enough additional bytes to get us up to the next size + // (again, we're trying the possibilities in order of increasing + // size). + size_t next = size[index[i]] - file->size; + size_t read = 0; + if (next > 0) { + read = mtd_read_data(ctx, p, next); + if (next != read) { + printf("short read (%d bytes of %d) for partition \"%s\"\n", + read, next, partition); + free(file->data); + file->data = NULL; + return -1; + } + SHA_update(&sha_ctx, p, read); + file->size += read; + } + + // Duplicate the SHA context and finalize the duplicate so we can + // check it against this pair's expected hash. + SHA_CTX temp_ctx; + memcpy(&temp_ctx, &sha_ctx, sizeof(SHA_CTX)); + const uint8_t* sha_so_far = SHA_final(&temp_ctx); + + if (ParseSha1(sha1sum[index[i]], parsed_sha) != 0) { + printf("failed to parse sha1 %s in %s\n", + sha1sum[index[i]], filename); + free(file->data); + file->data = NULL; + return -1; + } + + if (memcmp(sha_so_far, parsed_sha, SHA_DIGEST_SIZE) == 0) { + // we have a match. stop reading the partition; we'll return + // the data we've read so far. + printf("mtd read matched size %d sha %s\n", + size[index[i]], sha1sum[index[i]]); + break; + } + + p += read; + } + + mtd_read_close(ctx); + + if (i == pairs) { + // Ran off the end of the list of (size,sha1) pairs without + // finding a match. + printf("contents of MTD partition \"%s\" didn't match %s\n", + partition, filename); + free(file->data); + file->data = NULL; + return -1; + } + + const uint8_t* sha_final = SHA_final(&sha_ctx); + for (i = 0; i < SHA_DIGEST_SIZE; ++i) { + file->sha1[i] = sha_final[i]; + } + + // Fake some stat() info. + file->st.st_mode = 0644; + file->st.st_uid = 0; + file->st.st_gid = 0; + + free(copy); + free(index); + free(size); + free(sha1sum); + + return 0; +} + + +// Save the contents of the given FileContents object under the given +// filename. Return 0 on success. +int SaveFileContents(const char* filename, FileContents file) { + int fd = open(filename, O_WRONLY | O_CREAT | O_TRUNC); + if (fd < 0) { + printf("failed to open \"%s\" for write: %s\n", + filename, strerror(errno)); + return -1; + } + + size_t bytes_written = FileSink(file.data, file.size, &fd); + if (bytes_written != file.size) { + printf("short write of \"%s\" (%d bytes of %d) (%s)\n", + filename, bytes_written, file.size, strerror(errno)); + close(fd); + return -1; + } + fsync(fd); + close(fd); + + if (chmod(filename, file.st.st_mode) != 0) { + printf("chmod of \"%s\" failed: %s\n", filename, strerror(errno)); + return -1; + } + if (chown(filename, file.st.st_uid, file.st.st_gid) != 0) { + printf("chown of \"%s\" failed: %s\n", filename, strerror(errno)); + return -1; + } + + return 0; +} + +// Write a memory buffer to target_mtd partition, a string of the form +// "MTD:<partition>[:...]". Return 0 on success. +int WriteToMTDPartition(unsigned char* data, size_t len, + const char* target_mtd) { + char* partition = strchr(target_mtd, ':'); + if (partition == NULL) { + printf("bad MTD target name \"%s\"\n", target_mtd); + return -1; + } + ++partition; + // Trim off anything after a colon, eg "MTD:boot:blah:blah:blah...". + // We want just the partition name "boot". + partition = strdup(partition); + char* end = strchr(partition, ':'); + if (end != NULL) + *end = '\0'; + + if (!mtd_partitions_scanned) { + mtd_scan_partitions(); + mtd_partitions_scanned = 1; + } + + const MtdPartition* mtd = mtd_find_partition_by_name(partition); + if (mtd == NULL) { + printf("mtd partition \"%s\" not found for writing\n", partition); + return -1; + } + + MtdWriteContext* ctx = mtd_write_partition(mtd); + if (ctx == NULL) { + printf("failed to init mtd partition \"%s\" for writing\n", + partition); + return -1; + } + + size_t written = mtd_write_data(ctx, (char*)data, len); + if (written != len) { + printf("only wrote %d of %d bytes to MTD %s\n", + written, len, partition); + mtd_write_close(ctx); + return -1; + } + + if (mtd_erase_blocks(ctx, -1) < 0) { + printf("error finishing mtd write of %s\n", partition); + mtd_write_close(ctx); + return -1; + } + + if (mtd_write_close(ctx)) { + printf("error closing mtd write of %s\n", partition); + return -1; + } + + free(partition); + return 0; +} + + +// Take a string 'str' of 40 hex digits and parse it into the 20 +// byte array 'digest'. 'str' may contain only the digest or be of +// the form "<digest>:<anything>". Return 0 on success, -1 on any +// error. +int ParseSha1(const char* str, uint8_t* digest) { + int i; + const char* ps = str; + uint8_t* pd = digest; + for (i = 0; i < SHA_DIGEST_SIZE * 2; ++i, ++ps) { + int digit; + if (*ps >= '0' && *ps <= '9') { + digit = *ps - '0'; + } else if (*ps >= 'a' && *ps <= 'f') { + digit = *ps - 'a' + 10; + } else if (*ps >= 'A' && *ps <= 'F') { + digit = *ps - 'A' + 10; + } else { + return -1; + } + if (i % 2 == 0) { + *pd = digit << 4; + } else { + *pd |= digit; + ++pd; + } + } + if (*ps != '\0' && *ps != ':') return -1; + return 0; +} + +// Parse arguments (which should be of the form "<sha1>" or +// "<sha1>:<filename>" into the array *patches, returning the number +// of Patch objects in *num_patches. Return 0 on success. +int ParseShaArgs(int argc, char** argv, Patch** patches, int* num_patches) { + *num_patches = argc; + *patches = malloc(*num_patches * sizeof(Patch)); + + int i; + for (i = 0; i < *num_patches; ++i) { + if (ParseSha1(argv[i], (*patches)[i].sha1) != 0) { + printf("failed to parse sha1 \"%s\"\n", argv[i]); + return -1; + } + if (argv[i][SHA_DIGEST_SIZE*2] == '\0') { + (*patches)[i].patch_filename = NULL; + } else if (argv[i][SHA_DIGEST_SIZE*2] == ':') { + (*patches)[i].patch_filename = argv[i] + (SHA_DIGEST_SIZE*2+1); + } else { + printf("failed to parse filename \"%s\"\n", argv[i]); + return -1; + } + } + + return 0; +} + +// Search an array of Patch objects for one matching the given sha1. +// Return the Patch object on success, or NULL if no match is found. +const Patch* FindMatchingPatch(uint8_t* sha1, Patch* patches, int num_patches) { + int i; + for (i = 0; i < num_patches; ++i) { + if (memcmp(patches[i].sha1, sha1, SHA_DIGEST_SIZE) == 0) { + return patches+i; + } + } + return NULL; +} + +// Returns 0 if the contents of the file (argv[2]) or the cached file +// match any of the sha1's on the command line (argv[3:]). Returns +// nonzero otherwise. +int CheckMode(int argc, char** argv) { + if (argc < 3) { + printf("no filename given\n"); + return 2; + } + + int num_patches; + Patch* patches; + if (ParseShaArgs(argc-3, argv+3, &patches, &num_patches) != 0) { return 1; } + + FileContents file; + file.data = NULL; + + // It's okay to specify no sha1s; the check will pass if the + // LoadFileContents is successful. (Useful for reading MTD + // partitions, where the filename encodes the sha1s; no need to + // check them twice.) + if (LoadFileContents(argv[2], &file) != 0 || + (num_patches > 0 && + FindMatchingPatch(file.sha1, patches, num_patches) == NULL)) { + printf("file \"%s\" doesn't have any of expected " + "sha1 sums; checking cache\n", argv[2]); + + free(file.data); + + // If the source file is missing or corrupted, it might be because + // we were killed in the middle of patching it. A copy of it + // should have been made in CACHE_TEMP_SOURCE. If that file + // exists and matches the sha1 we're looking for, the check still + // passes. + + if (LoadFileContents(CACHE_TEMP_SOURCE, &file) != 0) { + printf("failed to load cache file\n"); + return 1; + } + + if (FindMatchingPatch(file.sha1, patches, num_patches) == NULL) { + printf("cache bits don't match any sha1 for \"%s\"\n", + argv[2]); + return 1; + } + } + + free(file.data); + return 0; +} + +int ShowLicenses() { + ShowBSDiffLicense(); + return 0; +} + +ssize_t FileSink(unsigned char* data, ssize_t len, void* token) { + int fd = *(int *)token; + ssize_t done = 0; + ssize_t wrote; + while (done < (ssize_t) len) { + wrote = write(fd, data+done, len-done); + if (wrote <= 0) { + printf("error writing %d bytes: %s\n", (int)(len-done), strerror(errno)); + return done; + } + done += wrote; + } + printf("wrote %d bytes to output\n", (int)done); + return done; +} + +typedef struct { + unsigned char* buffer; + ssize_t size; + ssize_t pos; +} MemorySinkInfo; + +ssize_t MemorySink(unsigned char* data, ssize_t len, void* token) { + MemorySinkInfo* msi = (MemorySinkInfo*)token; + if (msi->size - msi->pos < len) { + return -1; + } + memcpy(msi->buffer + msi->pos, data, len); + msi->pos += len; + return len; +} + +// Return the amount of free space (in bytes) on the filesystem +// containing filename. filename must exist. Return -1 on error. +size_t FreeSpaceForFile(const char* filename) { + struct statfs sf; + if (statfs(filename, &sf) != 0) { + printf("failed to statfs %s: %s\n", filename, strerror(errno)); + return -1; + } + return sf.f_bsize * sf.f_bfree; +} + +// This program applies binary patches to files in a way that is safe +// (the original file is not touched until we have the desired +// replacement for it) and idempotent (it's okay to run this program +// multiple times). +// +// - if the sha1 hash of <tgt-file> is <tgt-sha1>, does nothing and exits +// successfully. +// +// - otherwise, if the sha1 hash of <src-file> is <src-sha1>, applies the +// bsdiff <patch> to <src-file> to produce a new file (the type of patch +// is automatically detected from the file header). If that new +// file has sha1 hash <tgt-sha1>, moves it to replace <tgt-file>, and +// exits successfully. Note that if <src-file> and <tgt-file> are +// not the same, <src-file> is NOT deleted on success. <tgt-file> +// may be the string "-" to mean "the same as src-file". +// +// - otherwise, or if any error is encountered, exits with non-zero +// status. +// +// <src-file> (or <file> in check mode) may refer to an MTD partition +// to read the source data. See the comments for the +// LoadMTDContents() function above for the format of such a filename. +// +// +// As you might guess from the arguments, this function used to be +// main(); it was split out this way so applypatch could be built as a +// static library and linked into other executables as well. In the +// future only the library form will exist; we will not need to build +// this as a standalone executable. +// +// The arguments to this function are just the command-line of the +// standalone executable: +// +// <src-file> <tgt-file> <tgt-sha1> <tgt-size> [<src-sha1>:<patch> ...] +// to apply a patch. Returns 0 on success, 1 on failure. +// +// "-c" <file> [<sha1> ...] +// to check a file's contents against zero or more sha1s. Returns +// 0 if it matches any of them, 1 if it doesn't. +// +// "-s" <bytes> +// returns 0 if enough free space is available on /cache; 1 if it +// does not. +// +// "-l" +// shows open-source license information and returns 0. +// +// This function returns 2 if the arguments are not understood (in the +// standalone executable, this causes the usage message to be +// printed). +// +// TODO: make the interface more sensible for use as a library. + +int applypatch(int argc, char** argv) { + if (argc < 2) { + return 2; + } + + if (strncmp(argv[1], "-l", 3) == 0) { + return ShowLicenses(); + } + + if (strncmp(argv[1], "-c", 3) == 0) { + return CheckMode(argc, argv); + } + + if (strncmp(argv[1], "-s", 3) == 0) { + if (argc != 3) { + return 2; + } + size_t bytes = strtol(argv[2], NULL, 10); + if (MakeFreeSpaceOnCache(bytes) < 0) { + printf("unable to make %ld bytes available on /cache\n", (long)bytes); + return 1; + } else { + return 0; + } + } + + uint8_t target_sha1[SHA_DIGEST_SIZE]; + + const char* source_filename = argv[1]; + const char* target_filename = argv[2]; + if (target_filename[0] == '-' && + target_filename[1] == '\0') { + target_filename = source_filename; + } + + printf("\napplying patch to %s\n", source_filename); + + if (ParseSha1(argv[3], target_sha1) != 0) { + printf("failed to parse tgt-sha1 \"%s\"\n", argv[3]); + return 1; + } + + unsigned long target_size = strtoul(argv[4], NULL, 0); + + int num_patches; + Patch* patches; + if (ParseShaArgs(argc-5, argv+5, &patches, &num_patches) < 0) { return 1; } + + FileContents copy_file; + FileContents source_file; + const char* source_patch_filename = NULL; + const char* copy_patch_filename = NULL; + int made_copy = 0; + + // We try to load the target file into the source_file object. + if (LoadFileContents(target_filename, &source_file) == 0) { + if (memcmp(source_file.sha1, target_sha1, SHA_DIGEST_SIZE) == 0) { + // The early-exit case: the patch was already applied, this file + // has the desired hash, nothing for us to do. + printf("\"%s\" is already target; no patch needed\n", + target_filename); + return 0; + } + } + + if (source_file.data == NULL || + (target_filename != source_filename && + strcmp(target_filename, source_filename) != 0)) { + // Need to load the source file: either we failed to load the + // target file, or we did but it's different from the source file. + free(source_file.data); + LoadFileContents(source_filename, &source_file); + } + + if (source_file.data != NULL) { + const Patch* to_use = + FindMatchingPatch(source_file.sha1, patches, num_patches); + if (to_use != NULL) { + source_patch_filename = to_use->patch_filename; + } + } + + if (source_patch_filename == NULL) { + free(source_file.data); + printf("source file is bad; trying copy\n"); + + if (LoadFileContents(CACHE_TEMP_SOURCE, ©_file) < 0) { + // fail. + printf("failed to read copy file\n"); + return 1; + } + + const Patch* to_use = + FindMatchingPatch(copy_file.sha1, patches, num_patches); + if (to_use != NULL) { + copy_patch_filename = to_use->patch_filename; + } + + if (copy_patch_filename == NULL) { + // fail. + printf("copy file doesn't match source SHA-1s either\n"); + return 1; + } + } + + int retry = 1; + SHA_CTX ctx; + int output; + MemorySinkInfo msi; + FileContents* source_to_use; + char* outname; + + // assume that target_filename (eg "/system/app/Foo.apk") is located + // on the same filesystem as its top-level directory ("/system"). + // We need something that exists for calling statfs(). + char target_fs[strlen(target_filename)+1]; + char* slash = strchr(target_filename+1, '/'); + if (slash != NULL) { + int count = slash - target_filename; + strncpy(target_fs, target_filename, count); + target_fs[count] = '\0'; + } else { + strcpy(target_fs, target_filename); + } + + do { + // Is there enough room in the target filesystem to hold the patched + // file? + + if (strncmp(target_filename, "MTD:", 4) == 0) { + // If the target is an MTD partition, we're actually going to + // write the output to /tmp and then copy it to the partition. + // statfs() always returns 0 blocks free for /tmp, so instead + // we'll just assume that /tmp has enough space to hold the file. + + // We still write the original source to cache, in case the MTD + // write is interrupted. + if (MakeFreeSpaceOnCache(source_file.size) < 0) { + printf("not enough free space on /cache\n"); + return 1; + } + if (SaveFileContents(CACHE_TEMP_SOURCE, source_file) < 0) { + printf("failed to back up source file\n"); + return 1; + } + made_copy = 1; + retry = 0; + } else { + int enough_space = 0; + if (retry > 0) { + size_t free_space = FreeSpaceForFile(target_fs); + int enough_space = + (free_space > (target_size * 3 / 2)); // 50% margin of error + printf("target %ld bytes; free space %ld bytes; retry %d; enough %d\n", + (long)target_size, (long)free_space, retry, enough_space); + } + + if (!enough_space) { + retry = 0; + } + + if (!enough_space && source_patch_filename != NULL) { + // Using the original source, but not enough free space. First + // copy the source file to cache, then delete it from the original + // location. + + if (strncmp(source_filename, "MTD:", 4) == 0) { + // It's impossible to free space on the target filesystem by + // deleting the source if the source is an MTD partition. If + // we're ever in a state where we need to do this, fail. + printf("not enough free space for target but source is MTD\n"); + return 1; + } + + if (MakeFreeSpaceOnCache(source_file.size) < 0) { + printf("not enough free space on /cache\n"); + return 1; + } + + if (SaveFileContents(CACHE_TEMP_SOURCE, source_file) < 0) { + printf("failed to back up source file\n"); + return 1; + } + made_copy = 1; + unlink(source_filename); + + size_t free_space = FreeSpaceForFile(target_fs); + printf("(now %ld bytes free for target)\n", (long)free_space); + } + } + + const char* patch_filename; + if (source_patch_filename != NULL) { + source_to_use = &source_file; + patch_filename = source_patch_filename; + } else { + source_to_use = ©_file; + patch_filename = copy_patch_filename; + } + + SinkFn sink = NULL; + void* token = NULL; + output = -1; + outname = NULL; + if (strncmp(target_filename, "MTD:", 4) == 0) { + // We store the decoded output in memory. + msi.buffer = malloc(target_size); + if (msi.buffer == NULL) { + printf("failed to alloc %ld bytes for output\n", + (long)target_size); + return 1; + } + msi.pos = 0; + msi.size = target_size; + sink = MemorySink; + token = &msi; + } else { + // We write the decoded output to "<tgt-file>.patch". + outname = (char*)malloc(strlen(target_filename) + 10); + strcpy(outname, target_filename); + strcat(outname, ".patch"); + + output = open(outname, O_WRONLY | O_CREAT | O_TRUNC); + if (output < 0) { + printf("failed to open output file %s: %s\n", + outname, strerror(errno)); + return 1; + } + sink = FileSink; + token = &output; + } + +#define MAX_HEADER_LENGTH 8 + unsigned char header[MAX_HEADER_LENGTH]; + FILE* patchf = fopen(patch_filename, "rb"); + if (patchf == NULL) { + printf("failed to open patch file %s: %s\n", + patch_filename, strerror(errno)); + return 1; + } + int header_bytes_read = fread(header, 1, MAX_HEADER_LENGTH, patchf); + fclose(patchf); + + SHA_init(&ctx); + + int result; + + if (header_bytes_read >= 4 && + header[0] == 0xd6 && header[1] == 0xc3 && + header[2] == 0xc4 && header[3] == 0) { + // xdelta3 patches begin "VCD" (with the high bits set) followed + // by a zero byte (the version number). + printf("error: xdelta3 patches no longer supported\n"); + return 1; + } else if (header_bytes_read >= 8 && + memcmp(header, "BSDIFF40", 8) == 0) { + result = ApplyBSDiffPatch(source_to_use->data, source_to_use->size, + patch_filename, 0, sink, token, &ctx); + } else if (header_bytes_read >= 8 && + memcmp(header, "IMGDIFF", 7) == 0 && + (header[7] == '1' || header[7] == '2')) { + result = ApplyImagePatch(source_to_use->data, source_to_use->size, + patch_filename, sink, token, &ctx); + } else { + printf("Unknown patch file format\n"); + return 1; + } + + if (output >= 0) { + fsync(output); + close(output); + } + + if (result != 0) { + if (retry == 0) { + printf("applying patch failed\n"); + return result != 0; + } else { + printf("applying patch failed; retrying\n"); + } + if (outname != NULL) { + unlink(outname); + } + } else { + // succeeded; no need to retry + break; + } + } while (retry-- > 0); + + const uint8_t* current_target_sha1 = SHA_final(&ctx); + if (memcmp(current_target_sha1, target_sha1, SHA_DIGEST_SIZE) != 0) { + printf("patch did not produce expected sha1\n"); + return 1; + } + + if (output < 0) { + // Copy the temp file to the MTD partition. + if (WriteToMTDPartition(msi.buffer, msi.pos, target_filename) != 0) { + printf("write of patched data to %s failed\n", target_filename); + return 1; + } + free(msi.buffer); + } else { + // Give the .patch file the same owner, group, and mode of the + // original source file. + if (chmod(outname, source_to_use->st.st_mode) != 0) { + printf("chmod of \"%s\" failed: %s\n", outname, strerror(errno)); + return 1; + } + if (chown(outname, source_to_use->st.st_uid, + source_to_use->st.st_gid) != 0) { + printf("chown of \"%s\" failed: %s\n", outname, strerror(errno)); + return 1; + } + + // Finally, rename the .patch file to replace the target file. + if (rename(outname, target_filename) != 0) { + printf("rename of .patch to \"%s\" failed: %s\n", + target_filename, strerror(errno)); + return 1; + } + } + + // If this run of applypatch created the copy, and we're here, we + // can delete it. + if (made_copy) unlink(CACHE_TEMP_SOURCE); + + // Success! + return 0; +} diff --git a/applypatch/applypatch.h b/applypatch/applypatch.h new file mode 100644 index 0000000..3cb8021 --- /dev/null +++ b/applypatch/applypatch.h @@ -0,0 +1,70 @@ +/* + * Copyright (C) 2008 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef _APPLYPATCH_H +#define _APPLYPATCH_H + +#include <sys/stat.h> +#include "mincrypt/sha.h" + +typedef struct _Patch { + uint8_t sha1[SHA_DIGEST_SIZE]; + const char* patch_filename; +} Patch; + +typedef struct _FileContents { + uint8_t sha1[SHA_DIGEST_SIZE]; + unsigned char* data; + ssize_t size; + struct stat st; +} FileContents; + +// When there isn't enough room on the target filesystem to hold the +// patched version of the file, we copy the original here and delete +// it to free up space. If the expected source file doesn't exist, or +// is corrupted, we look to see if this file contains the bits we want +// and use it as the source instead. +#define CACHE_TEMP_SOURCE "/cache/saved.file" + +typedef ssize_t (*SinkFn)(unsigned char*, ssize_t, void*); + +// applypatch.c +size_t FreeSpaceForFile(const char* filename); +int applypatch(int argc, char** argv); + +// Read a file into memory; store it and its associated metadata in +// *file. Return 0 on success. +int LoadFileContents(const char* filename, FileContents* file); +void FreeFileContents(FileContents* file); + +// bsdiff.c +void ShowBSDiffLicense(); +int ApplyBSDiffPatch(const unsigned char* old_data, ssize_t old_size, + const char* patch_filename, ssize_t offset, + SinkFn sink, void* token, SHA_CTX* ctx); +int ApplyBSDiffPatchMem(const unsigned char* old_data, ssize_t old_size, + const char* patch_filename, ssize_t patch_offset, + unsigned char** new_data, ssize_t* new_size); + +// imgpatch.c +int ApplyImagePatch(const unsigned char* old_data, ssize_t old_size, + const char* patch_filename, + SinkFn sink, void* token, SHA_CTX* ctx); + +// freecache.c +int MakeFreeSpaceOnCache(size_t bytes_needed); + +#endif diff --git a/applypatch/applypatch.sh b/applypatch/applypatch.sh new file mode 100755 index 0000000..88f3025 --- /dev/null +++ b/applypatch/applypatch.sh @@ -0,0 +1,345 @@ +#!/bin/bash +# +# A test suite for applypatch. Run in a client where you have done +# envsetup, choosecombo, etc. +# +# DO NOT RUN THIS ON A DEVICE YOU CARE ABOUT. It will mess up your +# system partition. +# +# +# TODO: find some way to get this run regularly along with the rest of +# the tests. + +EMULATOR_PORT=5580 +DATA_DIR=$ANDROID_BUILD_TOP/build/tools/applypatch/testdata + +# This must be the filename that applypatch uses for its copies. +CACHE_TEMP_SOURCE=/cache/saved.file + +# Put all binaries and files here. We use /cache because it's a +# temporary filesystem in the emulator; it's created fresh each time +# the emulator starts. +WORK_DIR=/system + +# partition that WORK_DIR is located on, without the leading slash +WORK_FS=system + +# set to 0 to use a device instead +USE_EMULATOR=1 + +# ------------------------ + +tmpdir=$(mktemp -d) + +if [ "$USE_EMULATOR" == 1 ]; then + emulator -wipe-data -noaudio -no-window -port $EMULATOR_PORT & + pid_emulator=$! + ADB="adb -s emulator-$EMULATOR_PORT " +else + ADB="adb -d " +fi + +echo "waiting to connect to device" +$ADB wait-for-device +echo "device is available" +$ADB remount +# free up enough space on the system partition for the test to run. +$ADB shell rm -r /system/media + +# run a command on the device; exit with the exit status of the device +# command. +run_command() { + $ADB shell "$@" \; echo \$? | awk '{if (b) {print a}; a=$0; b=1} END {exit a}' +} + +testname() { + echo + echo "$1"... + testname="$1" +} + +fail() { + echo + echo FAIL: $testname + echo + [ "$open_pid" == "" ] || kill $open_pid + [ "$pid_emulator" == "" ] || kill $pid_emulator + exit 1 +} + +sha1() { + sha1sum $1 | awk '{print $1}' +} + +free_space() { + run_command df | awk "/$1/ {print gensub(/K/, \"\", \"g\", \$6)}" +} + +cleanup() { + # not necessary if we're about to kill the emulator, but nice for + # running on real devices or already-running emulators. + testname "removing test files" + run_command rm $WORK_DIR/bloat.dat + run_command rm $WORK_DIR/old.file + run_command rm $WORK_DIR/patch.bsdiff + run_command rm $WORK_DIR/applypatch + run_command rm $CACHE_TEMP_SOURCE + run_command rm /cache/bloat*.dat + + [ "$pid_emulator" == "" ] || kill $pid_emulator + + rm -rf $tmpdir +} + +cleanup + +$ADB push $ANDROID_PRODUCT_OUT/system/bin/applypatch $WORK_DIR/applypatch + +BAD1_SHA1=$(printf "%040x" $RANDOM) +BAD2_SHA1=$(printf "%040x" $RANDOM) +OLD_SHA1=$(sha1 $DATA_DIR/old.file) +NEW_SHA1=$(sha1 $DATA_DIR/new.file) +NEW_SIZE=$(stat -c %s $DATA_DIR/new.file) + +# --------------- basic execution ---------------------- + +testname "usage message" +run_command $WORK_DIR/applypatch && fail + +testname "display license" +run_command $WORK_DIR/applypatch -l | grep -q -i copyright || fail + + +# --------------- check mode ---------------------- + +$ADB push $DATA_DIR/old.file $WORK_DIR + +testname "check mode single" +run_command $WORK_DIR/applypatch -c $WORK_DIR/old.file $OLD_SHA1 || fail + +testname "check mode multiple" +run_command $WORK_DIR/applypatch -c $WORK_DIR/old.file $BAD1_SHA1 $OLD_SHA1 $BAD2_SHA1|| fail + +testname "check mode failure" +run_command $WORK_DIR/applypatch -c $WORK_DIR/old.file $BAD2_SHA1 $BAD1_SHA1 && fail + +$ADB push $DATA_DIR/old.file $CACHE_TEMP_SOURCE +# put some junk in the old file +run_command dd if=/dev/urandom of=$WORK_DIR/old.file count=100 bs=1024 || fail + +testname "check mode cache (corrupted) single" +run_command $WORK_DIR/applypatch -c $WORK_DIR/old.file $OLD_SHA1 || fail + +testname "check mode cache (corrupted) multiple" +run_command $WORK_DIR/applypatch -c $WORK_DIR/old.file $BAD1_SHA1 $OLD_SHA1 $BAD2_SHA1|| fail + +testname "check mode cache (corrupted) failure" +run_command $WORK_DIR/applypatch -c $WORK_DIR/old.file $BAD2_SHA1 $BAD1_SHA1 && fail + +# remove the old file entirely +run_command rm $WORK_DIR/old.file + +testname "check mode cache (missing) single" +run_command $WORK_DIR/applypatch -c $WORK_DIR/old.file $OLD_SHA1 || fail + +testname "check mode cache (missing) multiple" +run_command $WORK_DIR/applypatch -c $WORK_DIR/old.file $BAD1_SHA1 $OLD_SHA1 $BAD2_SHA1|| fail + +testname "check mode cache (missing) failure" +run_command $WORK_DIR/applypatch -c $WORK_DIR/old.file $BAD2_SHA1 $BAD1_SHA1 && fail + + +# --------------- apply patch ---------------------- + +$ADB push $DATA_DIR/old.file $WORK_DIR +$ADB push $DATA_DIR/patch.bsdiff $WORK_DIR + +# Check that the partition has enough space to apply the patch without +# copying. If it doesn't, we'll be testing the low-space condition +# when we intend to test the not-low-space condition. +testname "apply patches (with enough space)" +free_kb=$(free_space $WORK_FS) +echo "${free_kb}kb free on /$WORK_FS." +if (( free_kb * 1024 < NEW_SIZE * 3 / 2 )); then + echo "Not enough space on /$WORK_FS to patch test file." + echo + echo "This doesn't mean that applypatch is necessarily broken;" + echo "just that /$WORK_FS doesn't have enough free space to" + echo "properly run this test." + exit 1 +fi + +testname "apply bsdiff patch" +run_command $WORK_DIR/applypatch $WORK_DIR/old.file - $NEW_SHA1 $NEW_SIZE $BAD1_SHA1:$WORK_DIR/foo $OLD_SHA1:$WORK_DIR/patch.bsdiff || fail +$ADB pull $WORK_DIR/old.file $tmpdir/patched +diff -q $DATA_DIR/new.file $tmpdir/patched || fail + +testname "reapply bsdiff patch" +run_command $WORK_DIR/applypatch $WORK_DIR/old.file - $NEW_SHA1 $NEW_SIZE $BAD1_SHA1:$WORK_DIR/foo $OLD_SHA1:$WORK_DIR/patch.bsdiff || fail +$ADB pull $WORK_DIR/old.file $tmpdir/patched +diff -q $DATA_DIR/new.file $tmpdir/patched || fail + + +# --------------- apply patch in new location ---------------------- + +$ADB push $DATA_DIR/old.file $WORK_DIR +$ADB push $DATA_DIR/patch.bsdiff $WORK_DIR + +# Check that the partition has enough space to apply the patch without +# copying. If it doesn't, we'll be testing the low-space condition +# when we intend to test the not-low-space condition. +testname "apply patch to new location (with enough space)" +free_kb=$(free_space $WORK_FS) +echo "${free_kb}kb free on /$WORK_FS." +if (( free_kb * 1024 < NEW_SIZE * 3 / 2 )); then + echo "Not enough space on /$WORK_FS to patch test file." + echo + echo "This doesn't mean that applypatch is necessarily broken;" + echo "just that /$WORK_FS doesn't have enough free space to" + echo "properly run this test." + exit 1 +fi + +run_command rm $WORK_DIR/new.file +run_command rm $CACHE_TEMP_SOURCE + +testname "apply bsdiff patch to new location" +run_command $WORK_DIR/applypatch $WORK_DIR/old.file $WORK_DIR/new.file $NEW_SHA1 $NEW_SIZE $BAD1_SHA1:$WORK_DIR/foo $OLD_SHA1:$WORK_DIR/patch.bsdiff || fail +$ADB pull $WORK_DIR/new.file $tmpdir/patched +diff -q $DATA_DIR/new.file $tmpdir/patched || fail + +testname "reapply bsdiff patch to new location" +run_command $WORK_DIR/applypatch $WORK_DIR/old.file $WORK_DIR/new.file $NEW_SHA1 $NEW_SIZE $BAD1_SHA1:$WORK_DIR/foo $OLD_SHA1:$WORK_DIR/patch.bsdiff || fail +$ADB pull $WORK_DIR/new.file $tmpdir/patched +diff -q $DATA_DIR/new.file $tmpdir/patched || fail + +$ADB push $DATA_DIR/old.file $CACHE_TEMP_SOURCE +# put some junk in the old file +run_command dd if=/dev/urandom of=$WORK_DIR/old.file count=100 bs=1024 || fail + +testname "apply bsdiff patch to new location with corrupted source" +run_command $WORK_DIR/applypatch $WORK_DIR/old.file $WORK_DIR/new.file $NEW_SHA1 $NEW_SIZE $OLD_SHA1:$WORK_DIR/patch.bsdiff $BAD1_SHA1:$WORK_DIR/foo || fail +$ADB pull $WORK_DIR/new.file $tmpdir/patched +diff -q $DATA_DIR/new.file $tmpdir/patched || fail + +# put some junk in the cache copy, too +run_command dd if=/dev/urandom of=$CACHE_TEMP_SOURCE count=100 bs=1024 || fail + +run_command rm $WORK_DIR/new.file +testname "apply bsdiff patch to new location with corrupted source and copy (no new file)" +run_command $WORK_DIR/applypatch $WORK_DIR/old.file $WORK_DIR/new.file $NEW_SHA1 $NEW_SIZE $OLD_SHA1:$WORK_DIR/patch.bsdiff $BAD1_SHA1:$WORK_DIR/foo && fail + +# put some junk in the new file +run_command dd if=/dev/urandom of=$WORK_DIR/new.file count=100 bs=1024 || fail + +testname "apply bsdiff patch to new location with corrupted source and copy (bad new file)" +run_command $WORK_DIR/applypatch $WORK_DIR/old.file $WORK_DIR/new.file $NEW_SHA1 $NEW_SIZE $OLD_SHA1:$WORK_DIR/patch.bsdiff $BAD1_SHA1:$WORK_DIR/foo && fail + +# --------------- apply patch with low space on /system ---------------------- + +$ADB push $DATA_DIR/old.file $WORK_DIR +$ADB push $DATA_DIR/patch.bsdiff $WORK_DIR + +free_kb=$(free_space $WORK_FS) +echo "${free_kb}kb free on /$WORK_FS; we'll soon fix that." +echo run_command dd if=/dev/zero of=$WORK_DIR/bloat.dat count=$((free_kb-512)) bs=1024 || fail +run_command dd if=/dev/zero of=$WORK_DIR/bloat.dat count=$((free_kb-512)) bs=1024 || fail +free_kb=$(free_space $WORK_FS) +echo "${free_kb}kb free on /$WORK_FS now." + +testname "apply bsdiff patch with low space" +run_command $WORK_DIR/applypatch $WORK_DIR/old.file - $NEW_SHA1 $NEW_SIZE $BAD1_SHA1:$WORK_DIR/foo $OLD_SHA1:$WORK_DIR/patch.bsdiff || fail +$ADB pull $WORK_DIR/old.file $tmpdir/patched +diff -q $DATA_DIR/new.file $tmpdir/patched || fail + +testname "reapply bsdiff patch with low space" +run_command $WORK_DIR/applypatch $WORK_DIR/old.file - $NEW_SHA1 $NEW_SIZE $BAD1_SHA1:$WORK_DIR/foo $OLD_SHA1:$WORK_DIR/patch.bsdiff || fail +$ADB pull $WORK_DIR/old.file $tmpdir/patched +diff -q $DATA_DIR/new.file $tmpdir/patched || fail + +# --------------- apply patch with low space on /system and /cache ---------------------- + +$ADB push $DATA_DIR/old.file $WORK_DIR +$ADB push $DATA_DIR/patch.bsdiff $WORK_DIR + +free_kb=$(free_space $WORK_FS) +echo "${free_kb}kb free on /$WORK_FS" + +run_command mkdir /cache/subdir +run_command 'echo > /cache/subdir/a.file' +run_command 'echo > /cache/a.file' +run_command mkdir /cache/recovery /cache/recovery/otatest +run_command 'echo > /cache/recovery/otatest/b.file' +run_command "echo > $CACHE_TEMP_SOURCE" +free_kb=$(free_space cache) +echo "${free_kb}kb free on /cache; we'll soon fix that." +run_command dd if=/dev/zero of=/cache/bloat_small.dat count=128 bs=1024 || fail +run_command dd if=/dev/zero of=/cache/bloat_large.dat count=$((free_kb-640)) bs=1024 || fail +free_kb=$(free_space cache) +echo "${free_kb}kb free on /cache now." + +testname "apply bsdiff patch with low space, full cache, can't delete enough" +$ADB shell 'cat >> /cache/bloat_large.dat' & open_pid=$! +echo "open_pid is $open_pid" + +# size check should fail even though it deletes some stuff +run_command $WORK_DIR/applypatch -s $NEW_SIZE && fail +run_command ls /cache/bloat_small.dat && fail # was deleted +run_command ls /cache/a.file && fail # was deleted +run_command ls /cache/recovery/otatest/b.file && fail # was deleted +run_command ls /cache/bloat_large.dat || fail # wasn't deleted because it was open +run_command ls /cache/subdir/a.file || fail # wasn't deleted because it's in a subdir +run_command ls $CACHE_TEMP_SOURCE || fail # wasn't deleted because it's the source file copy + +# should fail; not enough files can be deleted +run_command $WORK_DIR/applypatch $WORK_DIR/old.file - $NEW_SHA1 $NEW_SIZE $BAD1_SHA1:$WORK_DIR/foo $OLD_SHA1:$WORK_DIR/patch.bsdiff && fail +run_command ls /cache/bloat_large.dat || fail # wasn't deleted because it was open +run_command ls /cache/subdir/a.file || fail # wasn't deleted because it's in a subdir +run_command ls $CACHE_TEMP_SOURCE || fail # wasn't deleted because it's the source file copy + +kill $open_pid # /cache/bloat_large.dat is no longer open + +testname "apply bsdiff patch with low space, full cache, can delete enough" + +# should succeed after deleting /cache/bloat_large.dat +run_command $WORK_DIR/applypatch -s $NEW_SIZE || fail +run_command ls /cache/bloat_large.dat && fail # was deleted +run_command ls /cache/subdir/a.file || fail # still wasn't deleted because it's in a subdir +run_command ls $CACHE_TEMP_SOURCE || fail # wasn't deleted because it's the source file copy + +# should succeed +run_command $WORK_DIR/applypatch $WORK_DIR/old.file - $NEW_SHA1 $NEW_SIZE $BAD1_SHA1:$WORK_DIR/foo $OLD_SHA1:$WORK_DIR/patch.bsdiff || fail +$ADB pull $WORK_DIR/old.file $tmpdir/patched +diff -q $DATA_DIR/new.file $tmpdir/patched || fail +run_command ls /cache/subdir/a.file || fail # still wasn't deleted because it's in a subdir +run_command ls $CACHE_TEMP_SOURCE && fail # was deleted because patching overwrote it, then deleted it + +# --------------- apply patch from cache ---------------------- + +$ADB push $DATA_DIR/old.file $CACHE_TEMP_SOURCE +# put some junk in the old file +run_command dd if=/dev/urandom of=$WORK_DIR/old.file count=100 bs=1024 || fail + +testname "apply bsdiff patch from cache (corrupted source) with low space" +run_command $WORK_DIR/applypatch $WORK_DIR/old.file - $NEW_SHA1 $NEW_SIZE $BAD1_SHA1:$WORK_DIR/foo $OLD_SHA1:$WORK_DIR/patch.bsdiff || fail +$ADB pull $WORK_DIR/old.file $tmpdir/patched +diff -q $DATA_DIR/new.file $tmpdir/patched || fail + +$ADB push $DATA_DIR/old.file $CACHE_TEMP_SOURCE +# remove the old file entirely +run_command rm $WORK_DIR/old.file + +testname "apply bsdiff patch from cache (missing source) with low space" +run_command $WORK_DIR/applypatch $WORK_DIR/old.file - $NEW_SHA1 $NEW_SIZE $BAD1_SHA1:$WORK_DIR/foo $OLD_SHA1:$WORK_DIR/patch.bsdiff || fail +$ADB pull $WORK_DIR/old.file $tmpdir/patched +diff -q $DATA_DIR/new.file $tmpdir/patched || fail + + +# --------------- cleanup ---------------------- + +cleanup + +echo +echo PASS +echo + diff --git a/applypatch/bsdiff.c b/applypatch/bsdiff.c new file mode 100644 index 0000000..b6d342b --- /dev/null +++ b/applypatch/bsdiff.c @@ -0,0 +1,410 @@ +/* + * Copyright (C) 2009 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Most of this code comes from bsdiff.c from the bsdiff-4.3 + * distribution, which is: + */ + +/*- + * Copyright 2003-2005 Colin Percival + * All rights reserved + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted providing that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING + * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include <sys/types.h> + +#include <bzlib.h> +#include <err.h> +#include <fcntl.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <unistd.h> + +#define MIN(x,y) (((x)<(y)) ? (x) : (y)) + +static void split(off_t *I,off_t *V,off_t start,off_t len,off_t h) +{ + off_t i,j,k,x,tmp,jj,kk; + + if(len<16) { + for(k=start;k<start+len;k+=j) { + j=1;x=V[I[k]+h]; + for(i=1;k+i<start+len;i++) { + if(V[I[k+i]+h]<x) { + x=V[I[k+i]+h]; + j=0; + }; + if(V[I[k+i]+h]==x) { + tmp=I[k+j];I[k+j]=I[k+i];I[k+i]=tmp; + j++; + }; + }; + for(i=0;i<j;i++) V[I[k+i]]=k+j-1; + if(j==1) I[k]=-1; + }; + return; + }; + + x=V[I[start+len/2]+h]; + jj=0;kk=0; + for(i=start;i<start+len;i++) { + if(V[I[i]+h]<x) jj++; + if(V[I[i]+h]==x) kk++; + }; + jj+=start;kk+=jj; + + i=start;j=0;k=0; + while(i<jj) { + if(V[I[i]+h]<x) { + i++; + } else if(V[I[i]+h]==x) { + tmp=I[i];I[i]=I[jj+j];I[jj+j]=tmp; + j++; + } else { + tmp=I[i];I[i]=I[kk+k];I[kk+k]=tmp; + k++; + }; + }; + + while(jj+j<kk) { + if(V[I[jj+j]+h]==x) { + j++; + } else { + tmp=I[jj+j];I[jj+j]=I[kk+k];I[kk+k]=tmp; + k++; + }; + }; + + if(jj>start) split(I,V,start,jj-start,h); + + for(i=0;i<kk-jj;i++) V[I[jj+i]]=kk-1; + if(jj==kk-1) I[jj]=-1; + + if(start+len>kk) split(I,V,kk,start+len-kk,h); +} + +static void qsufsort(off_t *I,off_t *V,u_char *old,off_t oldsize) +{ + off_t buckets[256]; + off_t i,h,len; + + for(i=0;i<256;i++) buckets[i]=0; + for(i=0;i<oldsize;i++) buckets[old[i]]++; + for(i=1;i<256;i++) buckets[i]+=buckets[i-1]; + for(i=255;i>0;i--) buckets[i]=buckets[i-1]; + buckets[0]=0; + + for(i=0;i<oldsize;i++) I[++buckets[old[i]]]=i; + I[0]=oldsize; + for(i=0;i<oldsize;i++) V[i]=buckets[old[i]]; + V[oldsize]=0; + for(i=1;i<256;i++) if(buckets[i]==buckets[i-1]+1) I[buckets[i]]=-1; + I[0]=-1; + + for(h=1;I[0]!=-(oldsize+1);h+=h) { + len=0; + for(i=0;i<oldsize+1;) { + if(I[i]<0) { + len-=I[i]; + i-=I[i]; + } else { + if(len) I[i-len]=-len; + len=V[I[i]]+1-i; + split(I,V,i,len,h); + i+=len; + len=0; + }; + }; + if(len) I[i-len]=-len; + }; + + for(i=0;i<oldsize+1;i++) I[V[i]]=i; +} + +static off_t matchlen(u_char *old,off_t oldsize,u_char *new,off_t newsize) +{ + off_t i; + + for(i=0;(i<oldsize)&&(i<newsize);i++) + if(old[i]!=new[i]) break; + + return i; +} + +static off_t search(off_t *I,u_char *old,off_t oldsize, + u_char *new,off_t newsize,off_t st,off_t en,off_t *pos) +{ + off_t x,y; + + if(en-st<2) { + x=matchlen(old+I[st],oldsize-I[st],new,newsize); + y=matchlen(old+I[en],oldsize-I[en],new,newsize); + + if(x>y) { + *pos=I[st]; + return x; + } else { + *pos=I[en]; + return y; + } + }; + + x=st+(en-st)/2; + if(memcmp(old+I[x],new,MIN(oldsize-I[x],newsize))<0) { + return search(I,old,oldsize,new,newsize,x,en,pos); + } else { + return search(I,old,oldsize,new,newsize,st,x,pos); + }; +} + +static void offtout(off_t x,u_char *buf) +{ + off_t y; + + if(x<0) y=-x; else y=x; + + buf[0]=y%256;y-=buf[0]; + y=y/256;buf[1]=y%256;y-=buf[1]; + y=y/256;buf[2]=y%256;y-=buf[2]; + y=y/256;buf[3]=y%256;y-=buf[3]; + y=y/256;buf[4]=y%256;y-=buf[4]; + y=y/256;buf[5]=y%256;y-=buf[5]; + y=y/256;buf[6]=y%256;y-=buf[6]; + y=y/256;buf[7]=y%256; + + if(x<0) buf[7]|=0x80; +} + +// This is main() from bsdiff.c, with the following changes: +// +// - old, oldsize, new, newsize are arguments; we don't load this +// data from files. old and new are owned by the caller; we +// don't free them at the end. +// +// - the "I" block of memory is owned by the caller, who passes a +// pointer to *I, which can be NULL. This way if we call +// bsdiff() multiple times with the same 'old' data, we only do +// the qsufsort() step the first time. +// +int bsdiff(u_char* old, off_t oldsize, off_t** IP, u_char* new, off_t newsize, + const char* patch_filename) +{ + int fd; + off_t *I; + off_t scan,pos,len; + off_t lastscan,lastpos,lastoffset; + off_t oldscore,scsc; + off_t s,Sf,lenf,Sb,lenb; + off_t overlap,Ss,lens; + off_t i; + off_t dblen,eblen; + u_char *db,*eb; + u_char buf[8]; + u_char header[32]; + FILE * pf; + BZFILE * pfbz2; + int bz2err; + + if (*IP == NULL) { + off_t* V; + *IP = malloc((oldsize+1) * sizeof(off_t)); + V = malloc((oldsize+1) * sizeof(off_t)); + qsufsort(*IP, V, old, oldsize); + free(V); + } + I = *IP; + + if(((db=malloc(newsize+1))==NULL) || + ((eb=malloc(newsize+1))==NULL)) err(1,NULL); + dblen=0; + eblen=0; + + /* Create the patch file */ + if ((pf = fopen(patch_filename, "w")) == NULL) + err(1, "%s", patch_filename); + + /* Header is + 0 8 "BSDIFF40" + 8 8 length of bzip2ed ctrl block + 16 8 length of bzip2ed diff block + 24 8 length of new file */ + /* File is + 0 32 Header + 32 ?? Bzip2ed ctrl block + ?? ?? Bzip2ed diff block + ?? ?? Bzip2ed extra block */ + memcpy(header,"BSDIFF40",8); + offtout(0, header + 8); + offtout(0, header + 16); + offtout(newsize, header + 24); + if (fwrite(header, 32, 1, pf) != 1) + err(1, "fwrite(%s)", patch_filename); + + /* Compute the differences, writing ctrl as we go */ + if ((pfbz2 = BZ2_bzWriteOpen(&bz2err, pf, 9, 0, 0)) == NULL) + errx(1, "BZ2_bzWriteOpen, bz2err = %d", bz2err); + scan=0;len=0; + lastscan=0;lastpos=0;lastoffset=0; + while(scan<newsize) { + oldscore=0; + + for(scsc=scan+=len;scan<newsize;scan++) { + len=search(I,old,oldsize,new+scan,newsize-scan, + 0,oldsize,&pos); + + for(;scsc<scan+len;scsc++) + if((scsc+lastoffset<oldsize) && + (old[scsc+lastoffset] == new[scsc])) + oldscore++; + + if(((len==oldscore) && (len!=0)) || + (len>oldscore+8)) break; + + if((scan+lastoffset<oldsize) && + (old[scan+lastoffset] == new[scan])) + oldscore--; + }; + + if((len!=oldscore) || (scan==newsize)) { + s=0;Sf=0;lenf=0; + for(i=0;(lastscan+i<scan)&&(lastpos+i<oldsize);) { + if(old[lastpos+i]==new[lastscan+i]) s++; + i++; + if(s*2-i>Sf*2-lenf) { Sf=s; lenf=i; }; + }; + + lenb=0; + if(scan<newsize) { + s=0;Sb=0; + for(i=1;(scan>=lastscan+i)&&(pos>=i);i++) { + if(old[pos-i]==new[scan-i]) s++; + if(s*2-i>Sb*2-lenb) { Sb=s; lenb=i; }; + }; + }; + + if(lastscan+lenf>scan-lenb) { + overlap=(lastscan+lenf)-(scan-lenb); + s=0;Ss=0;lens=0; + for(i=0;i<overlap;i++) { + if(new[lastscan+lenf-overlap+i]== + old[lastpos+lenf-overlap+i]) s++; + if(new[scan-lenb+i]== + old[pos-lenb+i]) s--; + if(s>Ss) { Ss=s; lens=i+1; }; + }; + + lenf+=lens-overlap; + lenb-=lens; + }; + + for(i=0;i<lenf;i++) + db[dblen+i]=new[lastscan+i]-old[lastpos+i]; + for(i=0;i<(scan-lenb)-(lastscan+lenf);i++) + eb[eblen+i]=new[lastscan+lenf+i]; + + dblen+=lenf; + eblen+=(scan-lenb)-(lastscan+lenf); + + offtout(lenf,buf); + BZ2_bzWrite(&bz2err, pfbz2, buf, 8); + if (bz2err != BZ_OK) + errx(1, "BZ2_bzWrite, bz2err = %d", bz2err); + + offtout((scan-lenb)-(lastscan+lenf),buf); + BZ2_bzWrite(&bz2err, pfbz2, buf, 8); + if (bz2err != BZ_OK) + errx(1, "BZ2_bzWrite, bz2err = %d", bz2err); + + offtout((pos-lenb)-(lastpos+lenf),buf); + BZ2_bzWrite(&bz2err, pfbz2, buf, 8); + if (bz2err != BZ_OK) + errx(1, "BZ2_bzWrite, bz2err = %d", bz2err); + + lastscan=scan-lenb; + lastpos=pos-lenb; + lastoffset=pos-scan; + }; + }; + BZ2_bzWriteClose(&bz2err, pfbz2, 0, NULL, NULL); + if (bz2err != BZ_OK) + errx(1, "BZ2_bzWriteClose, bz2err = %d", bz2err); + + /* Compute size of compressed ctrl data */ + if ((len = ftello(pf)) == -1) + err(1, "ftello"); + offtout(len-32, header + 8); + + /* Write compressed diff data */ + if ((pfbz2 = BZ2_bzWriteOpen(&bz2err, pf, 9, 0, 0)) == NULL) + errx(1, "BZ2_bzWriteOpen, bz2err = %d", bz2err); + BZ2_bzWrite(&bz2err, pfbz2, db, dblen); + if (bz2err != BZ_OK) + errx(1, "BZ2_bzWrite, bz2err = %d", bz2err); + BZ2_bzWriteClose(&bz2err, pfbz2, 0, NULL, NULL); + if (bz2err != BZ_OK) + errx(1, "BZ2_bzWriteClose, bz2err = %d", bz2err); + + /* Compute size of compressed diff data */ + if ((newsize = ftello(pf)) == -1) + err(1, "ftello"); + offtout(newsize - len, header + 16); + + /* Write compressed extra data */ + if ((pfbz2 = BZ2_bzWriteOpen(&bz2err, pf, 9, 0, 0)) == NULL) + errx(1, "BZ2_bzWriteOpen, bz2err = %d", bz2err); + BZ2_bzWrite(&bz2err, pfbz2, eb, eblen); + if (bz2err != BZ_OK) + errx(1, "BZ2_bzWrite, bz2err = %d", bz2err); + BZ2_bzWriteClose(&bz2err, pfbz2, 0, NULL, NULL); + if (bz2err != BZ_OK) + errx(1, "BZ2_bzWriteClose, bz2err = %d", bz2err); + + /* Seek to the beginning, write the header, and close the file */ + if (fseeko(pf, 0, SEEK_SET)) + err(1, "fseeko"); + if (fwrite(header, 32, 1, pf) != 1) + err(1, "fwrite(%s)", patch_filename); + if (fclose(pf)) + err(1, "fclose"); + + /* Free the memory we used */ + free(db); + free(eb); + + return 0; +} diff --git a/applypatch/bspatch.c b/applypatch/bspatch.c new file mode 100644 index 0000000..d5cd617 --- /dev/null +++ b/applypatch/bspatch.c @@ -0,0 +1,252 @@ +/* + * Copyright (C) 2008 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file is a nearly line-for-line copy of bspatch.c from the +// bsdiff-4.3 distribution; the primary differences being how the +// input and output data are read and the error handling. Running +// applypatch with the -l option will display the bsdiff license +// notice. + +#include <stdio.h> +#include <sys/stat.h> +#include <errno.h> +#include <unistd.h> +#include <string.h> + +#include <bzlib.h> + +#include "mincrypt/sha.h" +#include "applypatch.h" + +void ShowBSDiffLicense() { + puts("The bsdiff library used herein is:\n" + "\n" + "Copyright 2003-2005 Colin Percival\n" + "All rights reserved\n" + "\n" + "Redistribution and use in source and binary forms, with or without\n" + "modification, are permitted providing that the following conditions\n" + "are met:\n" + "1. Redistributions of source code must retain the above copyright\n" + " notice, this list of conditions and the following disclaimer.\n" + "2. Redistributions in binary form must reproduce the above copyright\n" + " notice, this list of conditions and the following disclaimer in the\n" + " documentation and/or other materials provided with the distribution.\n" + "\n" + "THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR\n" + "IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\n" + "WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n" + "ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY\n" + "DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL\n" + "DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS\n" + "OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)\n" + "HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,\n" + "STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING\n" + "IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE\n" + "POSSIBILITY OF SUCH DAMAGE.\n" + "\n------------------\n\n" + "This program uses Julian R Seward's \"libbzip2\" library, available\n" + "from http://www.bzip.org/.\n" + ); +} + +static off_t offtin(u_char *buf) +{ + off_t y; + + y=buf[7]&0x7F; + y=y*256;y+=buf[6]; + y=y*256;y+=buf[5]; + y=y*256;y+=buf[4]; + y=y*256;y+=buf[3]; + y=y*256;y+=buf[2]; + y=y*256;y+=buf[1]; + y=y*256;y+=buf[0]; + + if(buf[7]&0x80) y=-y; + + return y; +} + + +int ApplyBSDiffPatch(const unsigned char* old_data, ssize_t old_size, + const char* patch_filename, ssize_t patch_offset, + SinkFn sink, void* token, SHA_CTX* ctx) { + + unsigned char* new_data; + ssize_t new_size; + if (ApplyBSDiffPatchMem(old_data, old_size, patch_filename, patch_offset, + &new_data, &new_size) != 0) { + return -1; + } + + if (sink(new_data, new_size, token) < new_size) { + fprintf(stderr, "short write of output: %d (%s)\n", errno, strerror(errno)); + return 1; + } + if (ctx) { + SHA_update(ctx, new_data, new_size); + } + free(new_data); + + return 0; +} + +int ApplyBSDiffPatchMem(const unsigned char* old_data, ssize_t old_size, + const char* patch_filename, ssize_t patch_offset, + unsigned char** new_data, ssize_t* new_size) { + + FILE* f; + if ((f = fopen(patch_filename, "rb")) == NULL) { + fprintf(stderr, "failed to open patch file\n"); + return 1; + } + + // File format: + // 0 8 "BSDIFF40" + // 8 8 X + // 16 8 Y + // 24 8 sizeof(newfile) + // 32 X bzip2(control block) + // 32+X Y bzip2(diff block) + // 32+X+Y ??? bzip2(extra block) + // with control block a set of triples (x,y,z) meaning "add x bytes + // from oldfile to x bytes from the diff block; copy y bytes from the + // extra block; seek forwards in oldfile by z bytes". + + fseek(f, patch_offset, SEEK_SET); + + unsigned char header[32]; + if (fread(header, 1, 32, f) < 32) { + fprintf(stderr, "failed to read patch file header\n"); + return 1; + } + + if (memcmp(header, "BSDIFF40", 8) != 0) { + fprintf(stderr, "corrupt bsdiff patch file header (magic number)\n"); + return 1; + } + + ssize_t ctrl_len, data_len; + ctrl_len = offtin(header+8); + data_len = offtin(header+16); + *new_size = offtin(header+24); + + if (ctrl_len < 0 || data_len < 0 || *new_size < 0) { + fprintf(stderr, "corrupt patch file header (data lengths)\n"); + return 1; + } + + fclose(f); + + int bzerr; + +#define OPEN_AT(f, bzf, offset) \ + FILE* f; \ + BZFILE* bzf; \ + if ((f = fopen(patch_filename, "rb")) == NULL) { \ + fprintf(stderr, "failed to open patch file\n"); \ + return 1; \ + } \ + if (fseeko(f, offset+patch_offset, SEEK_SET)) { \ + fprintf(stderr, "failed to seek in patch file\n"); \ + return 1; \ + } \ + if ((bzf = BZ2_bzReadOpen(&bzerr, f, 0, 0, NULL, 0)) == NULL) { \ + fprintf(stderr, "failed to bzReadOpen in patch file (%d)\n", bzerr); \ + return 1; \ + } + + OPEN_AT(cpf, cpfbz2, 32); + OPEN_AT(dpf, dpfbz2, 32+ctrl_len); + OPEN_AT(epf, epfbz2, 32+ctrl_len+data_len); + +#undef OPEN_AT + + *new_data = malloc(*new_size); + if (*new_data == NULL) { + fprintf(stderr, "failed to allocate %d bytes of memory for output file\n", + (int)*new_size); + return 1; + } + + off_t oldpos = 0, newpos = 0; + off_t ctrl[3]; + off_t len_read; + int i; + unsigned char buf[8]; + while (newpos < *new_size) { + // Read control data + for (i = 0; i < 3; ++i) { + len_read = BZ2_bzRead(&bzerr, cpfbz2, buf, 8); + if (len_read < 8 || !(bzerr == BZ_OK || bzerr == BZ_STREAM_END)) { + fprintf(stderr, "corrupt patch (read control)\n"); + return 1; + } + ctrl[i] = offtin(buf); + } + + // Sanity check + if (newpos + ctrl[0] > *new_size) { + fprintf(stderr, "corrupt patch (new file overrun)\n"); + return 1; + } + + // Read diff string + len_read = BZ2_bzRead(&bzerr, dpfbz2, *new_data + newpos, ctrl[0]); + if (len_read < ctrl[0] || !(bzerr == BZ_OK || bzerr == BZ_STREAM_END)) { + fprintf(stderr, "corrupt patch (read diff)\n"); + return 1; + } + + // Add old data to diff string + for (i = 0; i < ctrl[0]; ++i) { + if ((oldpos+i >= 0) && (oldpos+i < old_size)) { + (*new_data)[newpos+i] += old_data[oldpos+i]; + } + } + + // Adjust pointers + newpos += ctrl[0]; + oldpos += ctrl[0]; + + // Sanity check + if (newpos + ctrl[1] > *new_size) { + fprintf(stderr, "corrupt patch (new file overrun)\n"); + return 1; + } + + // Read extra string + len_read = BZ2_bzRead(&bzerr, epfbz2, *new_data + newpos, ctrl[1]); + if (len_read < ctrl[1] || !(bzerr == BZ_OK || bzerr == BZ_STREAM_END)) { + fprintf(stderr, "corrupt patch (read extra)\n"); + return 1; + } + + // Adjust pointers + newpos += ctrl[1]; + oldpos += ctrl[2]; + } + + BZ2_bzReadClose(&bzerr, cpfbz2); + BZ2_bzReadClose(&bzerr, dpfbz2); + BZ2_bzReadClose(&bzerr, epfbz2); + fclose(cpf); + fclose(dpf); + fclose(epf); + + return 0; +} diff --git a/applypatch/freecache.c b/applypatch/freecache.c new file mode 100644 index 0000000..9827fda --- /dev/null +++ b/applypatch/freecache.c @@ -0,0 +1,172 @@ +#include <errno.h> +#include <libgen.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <sys/stat.h> +#include <sys/statfs.h> +#include <unistd.h> +#include <dirent.h> +#include <ctype.h> + +#include "applypatch.h" + +static int EliminateOpenFiles(char** files, int file_count) { + DIR* d; + struct dirent* de; + d = opendir("/proc"); + if (d == NULL) { + printf("error opening /proc: %s\n", strerror(errno)); + return -1; + } + while ((de = readdir(d)) != 0) { + int i; + for (i = 0; de->d_name[i] != '\0' && isdigit(de->d_name[i]); ++i); + if (de->d_name[i]) continue; + + // de->d_name[i] is numeric + + char path[FILENAME_MAX]; + strcpy(path, "/proc/"); + strcat(path, de->d_name); + strcat(path, "/fd/"); + + DIR* fdd; + struct dirent* fdde; + fdd = opendir(path); + if (fdd == NULL) { + printf("error opening %s: %s\n", path, strerror(errno)); + continue; + } + while ((fdde = readdir(fdd)) != 0) { + char fd_path[FILENAME_MAX]; + char link[FILENAME_MAX]; + strcpy(fd_path, path); + strcat(fd_path, fdde->d_name); + + int count; + count = readlink(fd_path, link, sizeof(link)-1); + if (count >= 0) { + link[count] = '\0'; + + // This is inefficient, but it should only matter if there are + // lots of files in /cache, and lots of them are open (neither + // of which should be true, especially in recovery). + if (strncmp(link, "/cache/", 7) == 0) { + int j; + for (j = 0; j < file_count; ++j) { + if (files[j] && strcmp(files[j], link) == 0) { + printf("%s is open by %s\n", link, de->d_name); + free(files[j]); + files[j] = NULL; + } + } + } + } + } + closedir(fdd); + } + closedir(d); + + return 0; +} + +int FindExpendableFiles(char*** names, int* entries) { + DIR* d; + struct dirent* de; + int size = 32; + *entries = 0; + *names = malloc(size * sizeof(char*)); + + char path[FILENAME_MAX]; + + // We're allowed to delete unopened regular files in any of these + // directories. + const char* dirs[2] = {"/cache", "/cache/recovery/otatest"}; + + unsigned int i; + for (i = 0; i < sizeof(dirs)/sizeof(dirs[0]); ++i) { + d = opendir(dirs[i]); + if (d == NULL) { + printf("error opening %s: %s\n", dirs[i], strerror(errno)); + continue; + } + + // Look for regular files in the directory (not in any subdirectories). + while ((de = readdir(d)) != 0) { + strcpy(path, dirs[i]); + strcat(path, "/"); + strcat(path, de->d_name); + + // We can't delete CACHE_TEMP_SOURCE; if it's there we might have + // restarted during installation and could be depending on it to + // be there. + if (strcmp(path, CACHE_TEMP_SOURCE) == 0) continue; + + struct stat st; + if (stat(path, &st) == 0 && S_ISREG(st.st_mode)) { + if (*entries >= size) { + size *= 2; + *names = realloc(*names, size * sizeof(char*)); + } + (*names)[(*entries)++] = strdup(path); + } + } + + closedir(d); + } + + printf("%d regular files in deletable directories\n", *entries); + + if (EliminateOpenFiles(*names, *entries) < 0) { + return -1; + } + + return 0; +} + +int MakeFreeSpaceOnCache(size_t bytes_needed) { + size_t free_now = FreeSpaceForFile("/cache"); + printf("%ld bytes free on /cache (%ld needed)\n", + (long)free_now, (long)bytes_needed); + + if (free_now >= bytes_needed) { + return 0; + } + + char** names; + int entries; + + if (FindExpendableFiles(&names, &entries) < 0) { + return -1; + } + + if (entries == 0) { + // nothing we can delete to free up space! + printf("no files can be deleted to free space on /cache\n"); + return -1; + } + + // We could try to be smarter about which files to delete: the + // biggest ones? the smallest ones that will free up enough space? + // the oldest? the newest? + // + // Instead, we'll be dumb. + + int i; + for (i = 0; i < entries && free_now < bytes_needed; ++i) { + if (names[i]) { + unlink(names[i]); + free_now = FreeSpaceForFile("/cache"); + printf("deleted %s; now %ld bytes free\n", names[i], (long)free_now); + free(names[i]); + } + } + + for (; i < entries; ++i) { + free(names[i]); + } + free(names); + + return (free_now >= bytes_needed) ? 0 : -1; +} diff --git a/applypatch/imgdiff.c b/applypatch/imgdiff.c new file mode 100644 index 0000000..6b9ebee --- /dev/null +++ b/applypatch/imgdiff.c @@ -0,0 +1,1010 @@ +/* + * Copyright (C) 2009 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * This program constructs binary patches for images -- such as boot.img + * and recovery.img -- that consist primarily of large chunks of gzipped + * data interspersed with uncompressed data. Doing a naive bsdiff of + * these files is not useful because small changes in the data lead to + * large changes in the compressed bitstream; bsdiff patches of gzipped + * data are typically as large as the data itself. + * + * To patch these usefully, we break the source and target images up into + * chunks of two types: "normal" and "gzip". Normal chunks are simply + * patched using a plain bsdiff. Gzip chunks are first expanded, then a + * bsdiff is applied to the uncompressed data, then the patched data is + * gzipped using the same encoder parameters. Patched chunks are + * concatenated together to create the output file; the output image + * should be *exactly* the same series of bytes as the target image used + * originally to generate the patch. + * + * To work well with this tool, the gzipped sections of the target + * image must have been generated using the same deflate encoder that + * is available in applypatch, namely, the one in the zlib library. + * In practice this means that images should be compressed using the + * "minigzip" tool included in the zlib distribution, not the GNU gzip + * program. + * + * An "imgdiff" patch consists of a header describing the chunk structure + * of the file and any encoding parameters needed for the gzipped + * chunks, followed by N bsdiff patches, one per chunk. + * + * For a diff to be generated, the source and target images must have the + * same "chunk" structure: that is, the same number of gzipped and normal + * chunks in the same order. Android boot and recovery images currently + * consist of five chunks: a small normal header, a gzipped kernel, a + * small normal section, a gzipped ramdisk, and finally a small normal + * footer. + * + * Caveats: we locate gzipped sections within the source and target + * images by searching for the byte sequence 1f8b0800: 1f8b is the gzip + * magic number; 08 specifies the "deflate" encoding [the only encoding + * supported by the gzip standard]; and 00 is the flags byte. We do not + * currently support any extra header fields (which would be indicated by + * a nonzero flags byte). We also don't handle the case when that byte + * sequence appears spuriously in the file. (Note that it would have to + * occur spuriously within a normal chunk to be a problem.) + * + * + * The imgdiff patch header looks like this: + * + * "IMGDIFF1" (8) [magic number and version] + * chunk count (4) + * for each chunk: + * chunk type (4) [CHUNK_{NORMAL, GZIP, DEFLATE, RAW}] + * if chunk type == CHUNK_NORMAL: + * source start (8) + * source len (8) + * bsdiff patch offset (8) [from start of patch file] + * if chunk type == CHUNK_GZIP: (version 1 only) + * source start (8) + * source len (8) + * bsdiff patch offset (8) [from start of patch file] + * source expanded len (8) [size of uncompressed source] + * target expected len (8) [size of uncompressed target] + * gzip level (4) + * method (4) + * windowBits (4) + * memLevel (4) + * strategy (4) + * gzip header len (4) + * gzip header (gzip header len) + * gzip footer (8) + * if chunk type == CHUNK_DEFLATE: (version 2 only) + * source start (8) + * source len (8) + * bsdiff patch offset (8) [from start of patch file] + * source expanded len (8) [size of uncompressed source] + * target expected len (8) [size of uncompressed target] + * gzip level (4) + * method (4) + * windowBits (4) + * memLevel (4) + * strategy (4) + * if chunk type == RAW: (version 2 only) + * target len (4) + * data (target len) + * + * All integers are little-endian. "source start" and "source len" + * specify the section of the input image that comprises this chunk, + * including the gzip header and footer for gzip chunks. "source + * expanded len" is the size of the uncompressed source data. "target + * expected len" is the size of the uncompressed data after applying + * the bsdiff patch. The next five parameters specify the zlib + * parameters to be used when compressing the patched data, and the + * next three specify the header and footer to be wrapped around the + * compressed data to create the output chunk (so that header contents + * like the timestamp are recreated exactly). + * + * After the header there are 'chunk count' bsdiff patches; the offset + * of each from the beginning of the file is specified in the header. + */ + +#include <errno.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <sys/stat.h> +#include <unistd.h> +#include <sys/types.h> + +#include "zlib.h" +#include "imgdiff.h" +#include "utils.h" + +typedef struct { + int type; // CHUNK_NORMAL, CHUNK_DEFLATE + size_t start; // offset of chunk in original image file + + size_t len; + unsigned char* data; // data to be patched (uncompressed, for deflate chunks) + + size_t source_start; + size_t source_len; + + off_t* I; // used by bsdiff + + // --- for CHUNK_DEFLATE chunks only: --- + + // original (compressed) deflate data + size_t deflate_len; + unsigned char* deflate_data; + + char* filename; // used for zip entries + + // deflate encoder parameters + int level, method, windowBits, memLevel, strategy; + + size_t source_uncompressed_len; +} ImageChunk; + +typedef struct { + int data_offset; + int deflate_len; + int uncomp_len; + char* filename; +} ZipFileEntry; + +static int fileentry_compare(const void* a, const void* b) { + int ao = ((ZipFileEntry*)a)->data_offset; + int bo = ((ZipFileEntry*)b)->data_offset; + if (ao < bo) { + return -1; + } else if (ao > bo) { + return 1; + } else { + return 0; + } +} + +// from bsdiff.c +int bsdiff(u_char* old, off_t oldsize, off_t** IP, u_char* new, off_t newsize, + const char* patch_filename); + +unsigned char* ReadZip(const char* filename, + int* num_chunks, ImageChunk** chunks, + int include_pseudo_chunk) { + struct stat st; + if (stat(filename, &st) != 0) { + printf("failed to stat \"%s\": %s\n", filename, strerror(errno)); + return NULL; + } + + unsigned char* img = malloc(st.st_size); + FILE* f = fopen(filename, "rb"); + if (fread(img, 1, st.st_size, f) != st.st_size) { + printf("failed to read \"%s\" %s\n", filename, strerror(errno)); + fclose(f); + return NULL; + } + fclose(f); + + // look for the end-of-central-directory record. + + int i; + for (i = st.st_size-20; i >= 0 && i > st.st_size - 65600; --i) { + if (img[i] == 0x50 && img[i+1] == 0x4b && + img[i+2] == 0x05 && img[i+3] == 0x06) { + break; + } + } + // double-check: this archive consists of a single "disk" + if (!(img[i+4] == 0 && img[i+5] == 0 && img[i+6] == 0 && img[i+7] == 0)) { + printf("can't process multi-disk archive\n"); + return NULL; + } + + int cdcount = Read2(img+i+8); + int cdoffset = Read4(img+i+16); + + ZipFileEntry* temp_entries = malloc(cdcount * sizeof(ZipFileEntry)); + int entrycount = 0; + + unsigned char* cd = img+cdoffset; + for (i = 0; i < cdcount; ++i) { + if (!(cd[0] == 0x50 && cd[1] == 0x4b && cd[2] == 0x01 && cd[3] == 0x02)) { + printf("bad central directory entry %d\n", i); + return NULL; + } + + int clen = Read4(cd+20); // compressed len + int ulen = Read4(cd+24); // uncompressed len + int nlen = Read2(cd+28); // filename len + int xlen = Read2(cd+30); // extra field len + int mlen = Read2(cd+32); // file comment len + int hoffset = Read4(cd+42); // local header offset + + char* filename = malloc(nlen+1); + memcpy(filename, cd+46, nlen); + filename[nlen] = '\0'; + + int method = Read2(cd+10); + + cd += 46 + nlen + xlen + mlen; + + if (method != 8) { // 8 == deflate + free(filename); + continue; + } + + unsigned char* lh = img + hoffset; + + if (!(lh[0] == 0x50 && lh[1] == 0x4b && lh[2] == 0x03 && lh[3] == 0x04)) { + printf("bad local file header entry %d\n", i); + return NULL; + } + + if (Read2(lh+26) != nlen || memcmp(lh+30, filename, nlen) != 0) { + printf("central dir filename doesn't match local header\n"); + return NULL; + } + + xlen = Read2(lh+28); // extra field len; might be different from CD entry? + + temp_entries[entrycount].data_offset = hoffset+30+nlen+xlen; + temp_entries[entrycount].deflate_len = clen; + temp_entries[entrycount].uncomp_len = ulen; + temp_entries[entrycount].filename = filename; + ++entrycount; + } + + qsort(temp_entries, entrycount, sizeof(ZipFileEntry), fileentry_compare); + +#if 0 + printf("found %d deflated entries\n", entrycount); + for (i = 0; i < entrycount; ++i) { + printf("off %10d len %10d unlen %10d %p %s\n", + temp_entries[i].data_offset, + temp_entries[i].deflate_len, + temp_entries[i].uncomp_len, + temp_entries[i].filename, + temp_entries[i].filename); + } +#endif + + *num_chunks = 0; + *chunks = malloc((entrycount*2+2) * sizeof(ImageChunk)); + ImageChunk* curr = *chunks; + + if (include_pseudo_chunk) { + curr->type = CHUNK_NORMAL; + curr->start = 0; + curr->len = st.st_size; + curr->data = img; + curr->filename = NULL; + curr->I = NULL; + ++curr; + ++*num_chunks; + } + + int pos = 0; + int nextentry = 0; + + while (pos < st.st_size) { + if (nextentry < entrycount && pos == temp_entries[nextentry].data_offset) { + curr->type = CHUNK_DEFLATE; + curr->start = pos; + curr->deflate_len = temp_entries[nextentry].deflate_len; + curr->deflate_data = img + pos; + curr->filename = temp_entries[nextentry].filename; + curr->I = NULL; + + curr->len = temp_entries[nextentry].uncomp_len; + curr->data = malloc(curr->len); + + z_stream strm; + strm.zalloc = Z_NULL; + strm.zfree = Z_NULL; + strm.opaque = Z_NULL; + strm.avail_in = curr->deflate_len; + strm.next_in = curr->deflate_data; + + // -15 means we are decoding a 'raw' deflate stream; zlib will + // not expect zlib headers. + int ret = inflateInit2(&strm, -15); + + strm.avail_out = curr->len; + strm.next_out = curr->data; + ret = inflate(&strm, Z_NO_FLUSH); + if (ret != Z_STREAM_END) { + printf("failed to inflate \"%s\"; %d\n", curr->filename, ret); + return NULL; + } + + inflateEnd(&strm); + + pos += curr->deflate_len; + ++nextentry; + ++*num_chunks; + ++curr; + continue; + } + + // use a normal chunk to take all the data up to the start of the + // next deflate section. + + curr->type = CHUNK_NORMAL; + curr->start = pos; + if (nextentry < entrycount) { + curr->len = temp_entries[nextentry].data_offset - pos; + } else { + curr->len = st.st_size - pos; + } + curr->data = img + pos; + curr->filename = NULL; + curr->I = NULL; + pos += curr->len; + + ++*num_chunks; + ++curr; + } + + free(temp_entries); + return img; +} + +/* + * Read the given file and break it up into chunks, putting the number + * of chunks and their info in *num_chunks and **chunks, + * respectively. Returns a malloc'd block of memory containing the + * contents of the file; various pointers in the output chunk array + * will point into this block of memory. The caller should free the + * return value when done with all the chunks. Returns NULL on + * failure. + */ +unsigned char* ReadImage(const char* filename, + int* num_chunks, ImageChunk** chunks) { + struct stat st; + if (stat(filename, &st) != 0) { + printf("failed to stat \"%s\": %s\n", filename, strerror(errno)); + return NULL; + } + + unsigned char* img = malloc(st.st_size + 4); + FILE* f = fopen(filename, "rb"); + if (fread(img, 1, st.st_size, f) != st.st_size) { + printf("failed to read \"%s\" %s\n", filename, strerror(errno)); + fclose(f); + return NULL; + } + fclose(f); + + // append 4 zero bytes to the data so we can always search for the + // four-byte string 1f8b0800 starting at any point in the actual + // file data, without special-casing the end of the data. + memset(img+st.st_size, 0, 4); + + size_t pos = 0; + + *num_chunks = 0; + *chunks = NULL; + + while (pos < st.st_size) { + unsigned char* p = img+pos; + + if (st.st_size - pos >= 4 && + p[0] == 0x1f && p[1] == 0x8b && + p[2] == 0x08 && // deflate compression + p[3] == 0x00) { // no header flags + // 'pos' is the offset of the start of a gzip chunk. + + *num_chunks += 3; + *chunks = realloc(*chunks, *num_chunks * sizeof(ImageChunk)); + ImageChunk* curr = *chunks + (*num_chunks-3); + + // create a normal chunk for the header. + curr->start = pos; + curr->type = CHUNK_NORMAL; + curr->len = GZIP_HEADER_LEN; + curr->data = p; + curr->I = NULL; + + pos += curr->len; + p += curr->len; + ++curr; + + curr->type = CHUNK_DEFLATE; + curr->filename = NULL; + curr->I = NULL; + + // We must decompress this chunk in order to discover where it + // ends, and so we can put the uncompressed data and its length + // into curr->data and curr->len. + + size_t allocated = 32768; + curr->len = 0; + curr->data = malloc(allocated); + curr->start = pos; + curr->deflate_data = p; + + z_stream strm; + strm.zalloc = Z_NULL; + strm.zfree = Z_NULL; + strm.opaque = Z_NULL; + strm.avail_in = st.st_size - pos; + strm.next_in = p; + + // -15 means we are decoding a 'raw' deflate stream; zlib will + // not expect zlib headers. + int ret = inflateInit2(&strm, -15); + + do { + strm.avail_out = allocated - curr->len; + strm.next_out = curr->data + curr->len; + ret = inflate(&strm, Z_NO_FLUSH); + curr->len = allocated - strm.avail_out; + if (strm.avail_out == 0) { + allocated *= 2; + curr->data = realloc(curr->data, allocated); + } + } while (ret != Z_STREAM_END); + + curr->deflate_len = st.st_size - strm.avail_in - pos; + inflateEnd(&strm); + pos += curr->deflate_len; + p += curr->deflate_len; + ++curr; + + // create a normal chunk for the footer + + curr->type = CHUNK_NORMAL; + curr->start = pos; + curr->len = GZIP_FOOTER_LEN; + curr->data = img+pos; + curr->I = NULL; + + pos += curr->len; + p += curr->len; + ++curr; + + // The footer (that we just skipped over) contains the size of + // the uncompressed data. Double-check to make sure that it + // matches the size of the data we got when we actually did + // the decompression. + size_t footer_size = Read4(p-4); + if (footer_size != curr[-2].len) { + printf("Error: footer size %d != decompressed size %d\n", + footer_size, curr[-2].len); + free(img); + return NULL; + } + } else { + // Reallocate the list for every chunk; we expect the number of + // chunks to be small (5 for typical boot and recovery images). + ++*num_chunks; + *chunks = realloc(*chunks, *num_chunks * sizeof(ImageChunk)); + ImageChunk* curr = *chunks + (*num_chunks-1); + curr->start = pos; + curr->I = NULL; + + // 'pos' is not the offset of the start of a gzip chunk, so scan + // forward until we find a gzip header. + curr->type = CHUNK_NORMAL; + curr->data = p; + + for (curr->len = 0; curr->len < (st.st_size - pos); ++curr->len) { + if (p[curr->len] == 0x1f && + p[curr->len+1] == 0x8b && + p[curr->len+2] == 0x08 && + p[curr->len+3] == 0x00) { + break; + } + } + pos += curr->len; + } + } + + return img; +} + +#define BUFFER_SIZE 32768 + +/* + * Takes the uncompressed data stored in the chunk, compresses it + * using the zlib parameters stored in the chunk, and checks that it + * matches exactly the compressed data we started with (also stored in + * the chunk). Return 0 on success. + */ +int TryReconstruction(ImageChunk* chunk, unsigned char* out) { + size_t p = 0; + +#if 0 + printf("trying %d %d %d %d %d\n", + chunk->level, chunk->method, chunk->windowBits, + chunk->memLevel, chunk->strategy); +#endif + + z_stream strm; + strm.zalloc = Z_NULL; + strm.zfree = Z_NULL; + strm.opaque = Z_NULL; + strm.avail_in = chunk->len; + strm.next_in = chunk->data; + int ret; + ret = deflateInit2(&strm, chunk->level, chunk->method, chunk->windowBits, + chunk->memLevel, chunk->strategy); + do { + strm.avail_out = BUFFER_SIZE; + strm.next_out = out; + ret = deflate(&strm, Z_FINISH); + size_t have = BUFFER_SIZE - strm.avail_out; + + if (memcmp(out, chunk->deflate_data+p, have) != 0) { + // mismatch; data isn't the same. + deflateEnd(&strm); + return -1; + } + p += have; + } while (ret != Z_STREAM_END); + deflateEnd(&strm); + if (p != chunk->deflate_len) { + // mismatch; ran out of data before we should have. + return -1; + } + return 0; +} + +/* + * Verify that we can reproduce exactly the same compressed data that + * we started with. Sets the level, method, windowBits, memLevel, and + * strategy fields in the chunk to the encoding parameters needed to + * produce the right output. Returns 0 on success. + */ +int ReconstructDeflateChunk(ImageChunk* chunk) { + if (chunk->type != CHUNK_DEFLATE) { + printf("attempt to reconstruct non-deflate chunk\n"); + return -1; + } + + size_t p = 0; + unsigned char* out = malloc(BUFFER_SIZE); + + // We only check two combinations of encoder parameters: level 6 + // (the default) and level 9 (the maximum). + for (chunk->level = 6; chunk->level <= 9; chunk->level += 3) { + chunk->windowBits = -15; // 32kb window; negative to indicate a raw stream. + chunk->memLevel = 8; // the default value. + chunk->method = Z_DEFLATED; + chunk->strategy = Z_DEFAULT_STRATEGY; + + if (TryReconstruction(chunk, out) == 0) { + free(out); + return 0; + } + } + + free(out); + return -1; +} + +/* + * Given source and target chunks, compute a bsdiff patch between them + * by running bsdiff in a subprocess. Return the patch data, placing + * its length in *size. Return NULL on failure. We expect the bsdiff + * program to be in the path. + */ +unsigned char* MakePatch(ImageChunk* src, ImageChunk* tgt, size_t* size) { + if (tgt->type == CHUNK_NORMAL) { + if (tgt->len <= 160) { + tgt->type = CHUNK_RAW; + *size = tgt->len; + return tgt->data; + } + } + + char ptemp[] = "/tmp/imgdiff-patch-XXXXXX"; + mkstemp(ptemp); + + int r = bsdiff(src->data, src->len, &(src->I), tgt->data, tgt->len, ptemp); + if (r != 0) { + printf("bsdiff() failed: %d\n", r); + return NULL; + } + + struct stat st; + if (stat(ptemp, &st) != 0) { + printf("failed to stat patch file %s: %s\n", + ptemp, strerror(errno)); + return NULL; + } + + unsigned char* data = malloc(st.st_size); + + if (tgt->type == CHUNK_NORMAL && tgt->len <= st.st_size) { + unlink(ptemp); + + tgt->type = CHUNK_RAW; + *size = tgt->len; + return tgt->data; + } + + *size = st.st_size; + + FILE* f = fopen(ptemp, "rb"); + if (f == NULL) { + printf("failed to open patch %s: %s\n", ptemp, strerror(errno)); + return NULL; + } + if (fread(data, 1, st.st_size, f) != st.st_size) { + printf("failed to read patch %s: %s\n", ptemp, strerror(errno)); + return NULL; + } + fclose(f); + + unlink(ptemp); + + tgt->source_start = src->start; + switch (tgt->type) { + case CHUNK_NORMAL: + tgt->source_len = src->len; + break; + case CHUNK_DEFLATE: + tgt->source_len = src->deflate_len; + tgt->source_uncompressed_len = src->len; + break; + } + + return data; +} + +/* + * Cause a gzip chunk to be treated as a normal chunk (ie, as a blob + * of uninterpreted data). The resulting patch will likely be about + * as big as the target file, but it lets us handle the case of images + * where some gzip chunks are reconstructible but others aren't (by + * treating the ones that aren't as normal chunks). + */ +void ChangeDeflateChunkToNormal(ImageChunk* ch) { + if (ch->type != CHUNK_DEFLATE) return; + ch->type = CHUNK_NORMAL; + free(ch->data); + ch->data = ch->deflate_data; + ch->len = ch->deflate_len; +} + +/* + * Return true if the data in the chunk is identical (including the + * compressed representation, for gzip chunks). + */ +int AreChunksEqual(ImageChunk* a, ImageChunk* b) { + if (a->type != b->type) return 0; + + switch (a->type) { + case CHUNK_NORMAL: + return a->len == b->len && memcmp(a->data, b->data, a->len) == 0; + + case CHUNK_DEFLATE: + return a->deflate_len == b->deflate_len && + memcmp(a->deflate_data, b->deflate_data, a->deflate_len) == 0; + + default: + printf("unknown chunk type %d\n", a->type); + return 0; + } +} + +/* + * Look for runs of adjacent normal chunks and compress them down into + * a single chunk. (Such runs can be produced when deflate chunks are + * changed to normal chunks.) + */ +void MergeAdjacentNormalChunks(ImageChunk* chunks, int* num_chunks) { + int out = 0; + int in_start = 0, in_end; + while (in_start < *num_chunks) { + if (chunks[in_start].type != CHUNK_NORMAL) { + in_end = in_start+1; + } else { + // in_start is a normal chunk. Look for a run of normal chunks + // that constitute a solid block of data (ie, each chunk begins + // where the previous one ended). + for (in_end = in_start+1; + in_end < *num_chunks && chunks[in_end].type == CHUNK_NORMAL && + (chunks[in_end].start == + chunks[in_end-1].start + chunks[in_end-1].len && + chunks[in_end].data == + chunks[in_end-1].data + chunks[in_end-1].len); + ++in_end); + } + + if (in_end == in_start+1) { +#if 0 + printf("chunk %d is now %d\n", in_start, out); +#endif + if (out != in_start) { + memcpy(chunks+out, chunks+in_start, sizeof(ImageChunk)); + } + } else { +#if 0 + printf("collapse normal chunks %d-%d into %d\n", in_start, in_end-1, out); +#endif + + // Merge chunks [in_start, in_end-1] into one chunk. Since the + // data member of each chunk is just a pointer into an in-memory + // copy of the file, this can be done without recopying (the + // output chunk has the first chunk's start location and data + // pointer, and length equal to the sum of the input chunk + // lengths). + chunks[out].type = CHUNK_NORMAL; + chunks[out].start = chunks[in_start].start; + chunks[out].data = chunks[in_start].data; + chunks[out].len = chunks[in_end-1].len + + (chunks[in_end-1].start - chunks[in_start].start); + } + + ++out; + in_start = in_end; + } + *num_chunks = out; +} + +ImageChunk* FindChunkByName(const char* name, + ImageChunk* chunks, int num_chunks) { + int i; + for (i = 0; i < num_chunks; ++i) { + if (chunks[i].type == CHUNK_DEFLATE && chunks[i].filename && + strcmp(name, chunks[i].filename) == 0) { + return chunks+i; + } + } + return NULL; +} + +void DumpChunks(ImageChunk* chunks, int num_chunks) { + int i; + for (i = 0; i < num_chunks; ++i) { + printf("chunk %d: type %d start %d len %d\n", + i, chunks[i].type, chunks[i].start, chunks[i].len); + } +} + +int main(int argc, char** argv) { + if (argc != 4 && argc != 5) { + usage: + printf("usage: %s [-z] <src-img> <tgt-img> <patch-file>\n", + argv[0]); + return 2; + } + + int zip_mode = 0; + + if (strcmp(argv[1], "-z") == 0) { + zip_mode = 1; + --argc; + ++argv; + } + + + int num_src_chunks; + ImageChunk* src_chunks; + int num_tgt_chunks; + ImageChunk* tgt_chunks; + int i; + + if (zip_mode) { + if (ReadZip(argv[1], &num_src_chunks, &src_chunks, 1) == NULL) { + printf("failed to break apart source zip file\n"); + return 1; + } + if (ReadZip(argv[2], &num_tgt_chunks, &tgt_chunks, 0) == NULL) { + printf("failed to break apart target zip file\n"); + return 1; + } + } else { + if (ReadImage(argv[1], &num_src_chunks, &src_chunks) == NULL) { + printf("failed to break apart source image\n"); + return 1; + } + if (ReadImage(argv[2], &num_tgt_chunks, &tgt_chunks) == NULL) { + printf("failed to break apart target image\n"); + return 1; + } + + // Verify that the source and target images have the same chunk + // structure (ie, the same sequence of deflate and normal chunks). + + if (!zip_mode) { + // Merge the gzip header and footer in with any adjacent + // normal chunks. + MergeAdjacentNormalChunks(tgt_chunks, &num_tgt_chunks); + MergeAdjacentNormalChunks(src_chunks, &num_src_chunks); + } + + if (num_src_chunks != num_tgt_chunks) { + printf("source and target don't have same number of chunks!\n"); + printf("source chunks:\n"); + DumpChunks(src_chunks, num_src_chunks); + printf("target chunks:\n"); + DumpChunks(tgt_chunks, num_tgt_chunks); + return 1; + } + for (i = 0; i < num_src_chunks; ++i) { + if (src_chunks[i].type != tgt_chunks[i].type) { + printf("source and target don't have same chunk " + "structure! (chunk %d)\n", i); + printf("source chunks:\n"); + DumpChunks(src_chunks, num_src_chunks); + printf("target chunks:\n"); + DumpChunks(tgt_chunks, num_tgt_chunks); + return 1; + } + } + } + + for (i = 0; i < num_tgt_chunks; ++i) { + if (tgt_chunks[i].type == CHUNK_DEFLATE) { + // Confirm that given the uncompressed chunk data in the target, we + // can recompress it and get exactly the same bits as are in the + // input target image. If this fails, treat the chunk as a normal + // non-deflated chunk. + if (ReconstructDeflateChunk(tgt_chunks+i) < 0) { + printf("failed to reconstruct target deflate chunk %d [%s]; " + "treating as normal\n", i, tgt_chunks[i].filename); + ChangeDeflateChunkToNormal(tgt_chunks+i); + if (zip_mode) { + ImageChunk* src = FindChunkByName(tgt_chunks[i].filename, src_chunks, num_src_chunks); + if (src) { + ChangeDeflateChunkToNormal(src); + } + } else { + ChangeDeflateChunkToNormal(src_chunks+i); + } + continue; + } + + // If two deflate chunks are identical (eg, the kernel has not + // changed between two builds), treat them as normal chunks. + // This makes applypatch much faster -- it can apply a trivial + // patch to the compressed data, rather than uncompressing and + // recompressing to apply the trivial patch to the uncompressed + // data. + ImageChunk* src; + if (zip_mode) { + src = FindChunkByName(tgt_chunks[i].filename, src_chunks, num_src_chunks); + } else { + src = src_chunks+i; + } + + if (src == NULL || AreChunksEqual(tgt_chunks+i, src)) { + ChangeDeflateChunkToNormal(tgt_chunks+i); + if (src) { + ChangeDeflateChunkToNormal(src); + } + } + } + } + + // Merging neighboring normal chunks. + if (zip_mode) { + // For zips, we only need to do this to the target: deflated + // chunks are matched via filename, and normal chunks are patched + // using the entire source file as the source. + MergeAdjacentNormalChunks(tgt_chunks, &num_tgt_chunks); + } else { + // For images, we need to maintain the parallel structure of the + // chunk lists, so do the merging in both the source and target + // lists. + MergeAdjacentNormalChunks(tgt_chunks, &num_tgt_chunks); + MergeAdjacentNormalChunks(src_chunks, &num_src_chunks); + if (num_src_chunks != num_tgt_chunks) { + // This shouldn't happen. + printf("merging normal chunks went awry\n"); + return 1; + } + } + + // Compute bsdiff patches for each chunk's data (the uncompressed + // data, in the case of deflate chunks). + + printf("Construct patches for %d chunks...\n", num_tgt_chunks); + unsigned char** patch_data = malloc(num_tgt_chunks * sizeof(unsigned char*)); + size_t* patch_size = malloc(num_tgt_chunks * sizeof(size_t)); + for (i = 0; i < num_tgt_chunks; ++i) { + if (zip_mode) { + ImageChunk* src; + if (tgt_chunks[i].type == CHUNK_DEFLATE && + (src = FindChunkByName(tgt_chunks[i].filename, src_chunks, + num_src_chunks))) { + patch_data[i] = MakePatch(src, tgt_chunks+i, patch_size+i); + } else { + patch_data[i] = MakePatch(src_chunks, tgt_chunks+i, patch_size+i); + } + } else { + patch_data[i] = MakePatch(src_chunks+i, tgt_chunks+i, patch_size+i); + } + printf("patch %3d is %d bytes (of %d)\n", + i, patch_size[i], tgt_chunks[i].source_len); + } + + // Figure out how big the imgdiff file header is going to be, so + // that we can correctly compute the offset of each bsdiff patch + // within the file. + + size_t total_header_size = 12; + for (i = 0; i < num_tgt_chunks; ++i) { + total_header_size += 4; + switch (tgt_chunks[i].type) { + case CHUNK_NORMAL: + total_header_size += 8*3; + break; + case CHUNK_DEFLATE: + total_header_size += 8*5 + 4*5; + break; + case CHUNK_RAW: + total_header_size += 4 + patch_size[i]; + break; + } + } + + size_t offset = total_header_size; + + FILE* f = fopen(argv[3], "wb"); + + // Write out the headers. + + fwrite("IMGDIFF2", 1, 8, f); + Write4(num_tgt_chunks, f); + for (i = 0; i < num_tgt_chunks; ++i) { + Write4(tgt_chunks[i].type, f); + + switch (tgt_chunks[i].type) { + case CHUNK_NORMAL: + printf("chunk %3d: normal (%10d, %10d) %10d\n", i, + tgt_chunks[i].start, tgt_chunks[i].len, patch_size[i]); + Write8(tgt_chunks[i].source_start, f); + Write8(tgt_chunks[i].source_len, f); + Write8(offset, f); + offset += patch_size[i]; + break; + + case CHUNK_DEFLATE: + printf("chunk %3d: deflate (%10d, %10d) %10d %s\n", i, + tgt_chunks[i].start, tgt_chunks[i].deflate_len, patch_size[i], + tgt_chunks[i].filename); + Write8(tgt_chunks[i].source_start, f); + Write8(tgt_chunks[i].source_len, f); + Write8(offset, f); + Write8(tgt_chunks[i].source_uncompressed_len, f); + Write8(tgt_chunks[i].len, f); + Write4(tgt_chunks[i].level, f); + Write4(tgt_chunks[i].method, f); + Write4(tgt_chunks[i].windowBits, f); + Write4(tgt_chunks[i].memLevel, f); + Write4(tgt_chunks[i].strategy, f); + offset += patch_size[i]; + break; + + case CHUNK_RAW: + printf("chunk %3d: raw (%10d, %10d)\n", i, + tgt_chunks[i].start, tgt_chunks[i].len); + Write4(patch_size[i], f); + fwrite(patch_data[i], 1, patch_size[i], f); + break; + } + } + + // Append each chunk's bsdiff patch, in order. + + for (i = 0; i < num_tgt_chunks; ++i) { + if (tgt_chunks[i].type != CHUNK_RAW) { + fwrite(patch_data[i], 1, patch_size[i], f); + } + } + + fclose(f); + + return 0; +} diff --git a/applypatch/imgdiff.h b/applypatch/imgdiff.h new file mode 100644 index 0000000..f2069b4 --- /dev/null +++ b/applypatch/imgdiff.h @@ -0,0 +1,30 @@ +/* + * Copyright (C) 2009 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Image patch chunk types +#define CHUNK_NORMAL 0 +#define CHUNK_GZIP 1 // version 1 only +#define CHUNK_DEFLATE 2 // version 2 only +#define CHUNK_RAW 3 // version 2 only + +// The gzip header size is actually variable, but we currently don't +// support gzipped data with any of the optional fields, so for now it +// will always be ten bytes. See RFC 1952 for the definition of the +// gzip format. +#define GZIP_HEADER_LEN 10 + +// The gzip footer size really is fixed. +#define GZIP_FOOTER_LEN 8 diff --git a/applypatch/imgdiff_test.sh b/applypatch/imgdiff_test.sh new file mode 100755 index 0000000..dcdb922 --- /dev/null +++ b/applypatch/imgdiff_test.sh @@ -0,0 +1,118 @@ +#!/bin/bash +# +# A script for testing imgdiff/applypatch. It takes two full OTA +# packages as arguments. It generates (on the host) patches for all +# the zip/jar/apk files they have in common, as well as boot and +# recovery images. It then applies the patches on the device (or +# emulator) and checks that the resulting file is correct. + +EMULATOR_PORT=5580 + +# set to 0 to use a device instead +USE_EMULATOR=0 + +# where on the device to do all the patching. +WORK_DIR=/data/local/tmp + +START_OTA_PACKAGE=$1 +END_OTA_PACKAGE=$2 + +# ------------------------ + +tmpdir=$(mktemp -d) + +if [ "$USE_EMULATOR" == 1 ]; then + emulator -wipe-data -noaudio -no-window -port $EMULATOR_PORT & + pid_emulator=$! + ADB="adb -s emulator-$EMULATOR_PORT " +else + ADB="adb -d " +fi + +echo "waiting to connect to device" +$ADB wait-for-device + +# run a command on the device; exit with the exit status of the device +# command. +run_command() { + $ADB shell "$@" \; echo \$? | awk '{if (b) {print a}; a=$0; b=1} END {exit a}' +} + +testname() { + echo + echo "$1"... + testname="$1" +} + +fail() { + echo + echo FAIL: $testname + echo + [ "$open_pid" == "" ] || kill $open_pid + [ "$pid_emulator" == "" ] || kill $pid_emulator + exit 1 +} + +sha1() { + sha1sum $1 | awk '{print $1}' +} + +size() { + stat -c %s $1 | tr -d '\n' +} + +cleanup() { + # not necessary if we're about to kill the emulator, but nice for + # running on real devices or already-running emulators. + testname "removing test files" + run_command rm $WORK_DIR/applypatch + run_command rm $WORK_DIR/source + run_command rm $WORK_DIR/target + run_command rm $WORK_DIR/patch + + [ "$pid_emulator" == "" ] || kill $pid_emulator + + rm -rf $tmpdir +} + +$ADB push $ANDROID_PRODUCT_OUT/system/bin/applypatch $WORK_DIR/applypatch + +patch_and_apply() { + local fn=$1 + shift + + unzip -p $START_OTA_PACKAGE $fn > $tmpdir/source + unzip -p $END_OTA_PACKAGE $fn > $tmpdir/target + imgdiff "$@" $tmpdir/source $tmpdir/target $tmpdir/patch + bsdiff $tmpdir/source $tmpdir/target $tmpdir/patch.bs + echo "patch for $fn is $(size $tmpdir/patch) [of $(size $tmpdir/target)] ($(size $tmpdir/patch.bs) with bsdiff)" + echo "$fn $(size $tmpdir/patch) of $(size $tmpdir/target) bsdiff $(size $tmpdir/patch.bs)" >> /tmp/stats.txt + $ADB push $tmpdir/source $WORK_DIR/source || fail "source push failed" + run_command rm /data/local/tmp/target + $ADB push $tmpdir/patch $WORK_DIR/patch || fail "patch push failed" + run_command /data/local/tmp/applypatch /data/local/tmp/source \ + /data/local/tmp/target $(sha1 $tmpdir/target) $(size $tmpdir/target) \ + $(sha1 $tmpdir/source):/data/local/tmp/patch \ + || fail "applypatch of $fn failed" + $ADB pull /data/local/tmp/target $tmpdir/result + diff -q $tmpdir/target $tmpdir/result || fail "patch output not correct!" +} + +# --------------- basic execution ---------------------- + +for i in $((zipinfo -1 $START_OTA_PACKAGE; zipinfo -1 $END_OTA_PACKAGE) | \ + sort | uniq -d | egrep -e '[.](apk|jar|zip)$'); do + patch_and_apply $i -z +done +patch_and_apply boot.img +patch_and_apply system/recovery.img + + +# --------------- cleanup ---------------------- + +cleanup + +echo +echo PASS +echo + diff --git a/applypatch/imgpatch.c b/applypatch/imgpatch.c new file mode 100644 index 0000000..5322817 --- /dev/null +++ b/applypatch/imgpatch.c @@ -0,0 +1,364 @@ +/* + * Copyright (C) 2009 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// See imgdiff.c in this directory for a description of the patch file +// format. + +#include <stdio.h> +#include <sys/stat.h> +#include <errno.h> +#include <unistd.h> +#include <string.h> + +#include "zlib.h" +#include "mincrypt/sha.h" +#include "applypatch.h" +#include "imgdiff.h" +#include "utils.h" + +/* + * Apply the patch given in 'patch_filename' to the source data given + * by (old_data, old_size). Write the patched output to the 'output' + * file, and update the SHA context with the output data as well. + * Return 0 on success. + */ +int ApplyImagePatch(const unsigned char* old_data, ssize_t old_size, + const char* patch_filename, + SinkFn sink, void* token, SHA_CTX* ctx) { + FILE* f; + if ((f = fopen(patch_filename, "rb")) == NULL) { + printf("failed to open patch file\n"); + return -1; + } + + unsigned char header[12]; + if (fread(header, 1, 12, f) != 12) { + printf("failed to read patch file header\n"); + return -1; + } + + // IMGDIFF1 uses CHUNK_NORMAL and CHUNK_GZIP. + // IMGDIFF2 uses CHUNK_NORMAL, CHUNK_DEFLATE, and CHUNK_RAW. + if (memcmp(header, "IMGDIFF", 7) != 0 || + (header[7] != '1' && header[7] != '2')) { + printf("corrupt patch file header (magic number)\n"); + return -1; + } + + int num_chunks = Read4(header+8); + + int i; + for (i = 0; i < num_chunks; ++i) { + // each chunk's header record starts with 4 bytes. + unsigned char chunk[4]; + if (fread(chunk, 1, 4, f) != 4) { + printf("failed to read chunk %d record\n", i); + return -1; + } + + int type = Read4(chunk); + + if (type == CHUNK_NORMAL) { + unsigned char normal_header[24]; + if (fread(normal_header, 1, 24, f) != 24) { + printf("failed to read chunk %d normal header data\n", i); + return -1; + } + + size_t src_start = Read8(normal_header); + size_t src_len = Read8(normal_header+8); + size_t patch_offset = Read8(normal_header+16); + + printf("CHUNK %d: normal patch offset %d\n", i, patch_offset); + + ApplyBSDiffPatch(old_data + src_start, src_len, + patch_filename, patch_offset, + sink, token, ctx); + } else if (type == CHUNK_GZIP) { + // This branch is basically a duplicate of the CHUNK_DEFLATE + // branch, with a bit of extra processing for the gzip header + // and footer. I've avoided factoring the common code out since + // this branch will just be deleted when we drop support for + // IMGDIFF1. + + // gzip chunks have an additional 64 + gzip_header_len + 8 bytes + // in their chunk header. + unsigned char* gzip = malloc(64); + if (fread(gzip, 1, 64, f) != 64) { + printf("failed to read chunk %d initial gzip header data\n", + i); + return -1; + } + size_t gzip_header_len = Read4(gzip+60); + gzip = realloc(gzip, 64 + gzip_header_len + 8); + if (fread(gzip+64, 1, gzip_header_len+8, f) != gzip_header_len+8) { + printf("failed to read chunk %d remaining gzip header data\n", + i); + return -1; + } + + size_t src_start = Read8(gzip); + size_t src_len = Read8(gzip+8); + size_t patch_offset = Read8(gzip+16); + + size_t expanded_len = Read8(gzip+24); + size_t target_len = Read8(gzip+32); + int gz_level = Read4(gzip+40); + int gz_method = Read4(gzip+44); + int gz_windowBits = Read4(gzip+48); + int gz_memLevel = Read4(gzip+52); + int gz_strategy = Read4(gzip+56); + + printf("CHUNK %d: gzip patch offset %d\n", i, patch_offset); + + // Decompress the source data; the chunk header tells us exactly + // how big we expect it to be when decompressed. + + unsigned char* expanded_source = malloc(expanded_len); + if (expanded_source == NULL) { + printf("failed to allocate %d bytes for expanded_source\n", + expanded_len); + return -1; + } + + z_stream strm; + strm.zalloc = Z_NULL; + strm.zfree = Z_NULL; + strm.opaque = Z_NULL; + strm.avail_in = src_len - (gzip_header_len + 8); + strm.next_in = (unsigned char*)(old_data + src_start + gzip_header_len); + strm.avail_out = expanded_len; + strm.next_out = expanded_source; + + int ret; + ret = inflateInit2(&strm, -15); + if (ret != Z_OK) { + printf("failed to init source inflation: %d\n", ret); + return -1; + } + + // Because we've provided enough room to accommodate the output + // data, we expect one call to inflate() to suffice. + ret = inflate(&strm, Z_SYNC_FLUSH); + if (ret != Z_STREAM_END) { + printf("source inflation returned %d\n", ret); + return -1; + } + // We should have filled the output buffer exactly. + if (strm.avail_out != 0) { + printf("source inflation short by %d bytes\n", strm.avail_out); + return -1; + } + inflateEnd(&strm); + + // Next, apply the bsdiff patch (in memory) to the uncompressed + // data. + unsigned char* uncompressed_target_data; + ssize_t uncompressed_target_size; + if (ApplyBSDiffPatchMem(expanded_source, expanded_len, + patch_filename, patch_offset, + &uncompressed_target_data, + &uncompressed_target_size) != 0) { + return -1; + } + + // Now compress the target data and append it to the output. + + // start with the gzip header. + sink(gzip+64, gzip_header_len, token); + SHA_update(ctx, gzip+64, gzip_header_len); + + // we're done with the expanded_source data buffer, so we'll + // reuse that memory to receive the output of deflate. + unsigned char* temp_data = expanded_source; + ssize_t temp_size = expanded_len; + if (temp_size < 32768) { + // ... unless the buffer is too small, in which case we'll + // allocate a fresh one. + free(temp_data); + temp_data = malloc(32768); + temp_size = 32768; + } + + // now the deflate stream + strm.zalloc = Z_NULL; + strm.zfree = Z_NULL; + strm.opaque = Z_NULL; + strm.avail_in = uncompressed_target_size; + strm.next_in = uncompressed_target_data; + ret = deflateInit2(&strm, gz_level, gz_method, gz_windowBits, + gz_memLevel, gz_strategy); + do { + strm.avail_out = temp_size; + strm.next_out = temp_data; + ret = deflate(&strm, Z_FINISH); + size_t have = temp_size - strm.avail_out; + + if (sink(temp_data, have, token) != have) { + printf("failed to write %d compressed bytes to output\n", + have); + return -1; + } + SHA_update(ctx, temp_data, have); + } while (ret != Z_STREAM_END); + deflateEnd(&strm); + + // lastly, the gzip footer. + sink(gzip+64+gzip_header_len, 8, token); + SHA_update(ctx, gzip+64+gzip_header_len, 8); + + free(temp_data); + free(uncompressed_target_data); + free(gzip); + } else if (type == CHUNK_RAW) { + unsigned char raw_header[4]; + if (fread(raw_header, 1, 4, f) != 4) { + printf("failed to read chunk %d raw header data\n", i); + return -1; + } + + size_t data_len = Read4(raw_header); + + printf("CHUNK %d: raw data %d\n", i, data_len); + + unsigned char* temp = malloc(data_len); + if (fread(temp, 1, data_len, f) != data_len) { + printf("failed to read chunk %d raw data\n", i); + return -1; + } + SHA_update(ctx, temp, data_len); + if (sink(temp, data_len, token) != data_len) { + printf("failed to write chunk %d raw data\n", i); + return -1; + } + } else if (type == CHUNK_DEFLATE) { + // deflate chunks have an additional 60 bytes in their chunk header. + unsigned char deflate_header[60]; + if (fread(deflate_header, 1, 60, f) != 60) { + printf("failed to read chunk %d deflate header data\n", i); + return -1; + } + + size_t src_start = Read8(deflate_header); + size_t src_len = Read8(deflate_header+8); + size_t patch_offset = Read8(deflate_header+16); + size_t expanded_len = Read8(deflate_header+24); + size_t target_len = Read8(deflate_header+32); + int level = Read4(deflate_header+40); + int method = Read4(deflate_header+44); + int windowBits = Read4(deflate_header+48); + int memLevel = Read4(deflate_header+52); + int strategy = Read4(deflate_header+56); + + printf("CHUNK %d: deflate patch offset %d\n", i, patch_offset); + + // Decompress the source data; the chunk header tells us exactly + // how big we expect it to be when decompressed. + + unsigned char* expanded_source = malloc(expanded_len); + if (expanded_source == NULL) { + printf("failed to allocate %d bytes for expanded_source\n", + expanded_len); + return -1; + } + + z_stream strm; + strm.zalloc = Z_NULL; + strm.zfree = Z_NULL; + strm.opaque = Z_NULL; + strm.avail_in = src_len; + strm.next_in = (unsigned char*)(old_data + src_start); + strm.avail_out = expanded_len; + strm.next_out = expanded_source; + + int ret; + ret = inflateInit2(&strm, -15); + if (ret != Z_OK) { + printf("failed to init source inflation: %d\n", ret); + return -1; + } + + // Because we've provided enough room to accommodate the output + // data, we expect one call to inflate() to suffice. + ret = inflate(&strm, Z_SYNC_FLUSH); + if (ret != Z_STREAM_END) { + printf("source inflation returned %d\n", ret); + return -1; + } + // We should have filled the output buffer exactly. + if (strm.avail_out != 0) { + printf("source inflation short by %d bytes\n", strm.avail_out); + return -1; + } + inflateEnd(&strm); + + // Next, apply the bsdiff patch (in memory) to the uncompressed + // data. + unsigned char* uncompressed_target_data; + ssize_t uncompressed_target_size; + if (ApplyBSDiffPatchMem(expanded_source, expanded_len, + patch_filename, patch_offset, + &uncompressed_target_data, + &uncompressed_target_size) != 0) { + return -1; + } + + // Now compress the target data and append it to the output. + + // we're done with the expanded_source data buffer, so we'll + // reuse that memory to receive the output of deflate. + unsigned char* temp_data = expanded_source; + ssize_t temp_size = expanded_len; + if (temp_size < 32768) { + // ... unless the buffer is too small, in which case we'll + // allocate a fresh one. + free(temp_data); + temp_data = malloc(32768); + temp_size = 32768; + } + + // now the deflate stream + strm.zalloc = Z_NULL; + strm.zfree = Z_NULL; + strm.opaque = Z_NULL; + strm.avail_in = uncompressed_target_size; + strm.next_in = uncompressed_target_data; + ret = deflateInit2(&strm, level, method, windowBits, memLevel, strategy); + do { + strm.avail_out = temp_size; + strm.next_out = temp_data; + ret = deflate(&strm, Z_FINISH); + size_t have = temp_size - strm.avail_out; + + if (sink(temp_data, have, token) != have) { + printf("failed to write %d compressed bytes to output\n", + have); + return -1; + } + SHA_update(ctx, temp_data, have); + } while (ret != Z_STREAM_END); + deflateEnd(&strm); + + free(temp_data); + free(uncompressed_target_data); + } else { + printf("patch chunk %d is unknown type %d\n", i, type); + return -1; + } + } + + return 0; +} diff --git a/applypatch/main.c b/applypatch/main.c new file mode 100644 index 0000000..e08f5c1 --- /dev/null +++ b/applypatch/main.c @@ -0,0 +1,60 @@ +/* + * Copyright (C) 2009 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <stdio.h> + +extern int applypatch(int argc, char** argv); + +// This program applies binary patches to files in a way that is safe +// (the original file is not touched until we have the desired +// replacement for it) and idempotent (it's okay to run this program +// multiple times). +// +// - if the sha1 hash of <tgt-file> is <tgt-sha1>, does nothing and exits +// successfully. +// +// - otherwise, if the sha1 hash of <src-file> is <src-sha1>, applies the +// bsdiff <patch> to <src-file> to produce a new file (the type of patch +// is automatically detected from the file header). If that new +// file has sha1 hash <tgt-sha1>, moves it to replace <tgt-file>, and +// exits successfully. Note that if <src-file> and <tgt-file> are +// not the same, <src-file> is NOT deleted on success. <tgt-file> +// may be the string "-" to mean "the same as src-file". +// +// - otherwise, or if any error is encountered, exits with non-zero +// status. +// +// <src-file> (or <file> in check mode) may refer to an MTD partition +// to read the source data. See the comments for the +// LoadMTDContents() function above for the format of such a filename. + +int main(int argc, char** argv) { + int result = applypatch(argc, argv); + if (result == 2) { + printf( + "usage: %s <src-file> <tgt-file> <tgt-sha1> <tgt-size> " + "[<src-sha1>:<patch> ...]\n" + " or %s -c <file> [<sha1> ...]\n" + " or %s -s <bytes>\n" + " or %s -l\n" + "\n" + "Filenames may be of the form\n" + " MTD:<partition>:<len_1>:<sha1_1>:<len_2>:<sha1_2>:...\n" + "to specify reading from or writing to an MTD partition.\n\n", + argv[0], argv[0], argv[0], argv[0]); + } + return result; +} diff --git a/applypatch/testdata/new.file b/applypatch/testdata/new.file Binary files differnew file mode 100644 index 0000000..cdeb8fd --- /dev/null +++ b/applypatch/testdata/new.file diff --git a/applypatch/testdata/old.file b/applypatch/testdata/old.file Binary files differnew file mode 100644 index 0000000..166c873 --- /dev/null +++ b/applypatch/testdata/old.file diff --git a/applypatch/testdata/patch.bsdiff b/applypatch/testdata/patch.bsdiff Binary files differnew file mode 100644 index 0000000..b78d385 --- /dev/null +++ b/applypatch/testdata/patch.bsdiff diff --git a/applypatch/utils.c b/applypatch/utils.c new file mode 100644 index 0000000..912229b --- /dev/null +++ b/applypatch/utils.c @@ -0,0 +1,62 @@ +/* + * Copyright (C) 2009 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <stdio.h> + +#include "utils.h" + +/** Write a 4-byte value to f in little-endian order. */ +void Write4(int value, FILE* f) { + fputc(value & 0xff, f); + fputc((value >> 8) & 0xff, f); + fputc((value >> 16) & 0xff, f); + fputc((value >> 24) & 0xff, f); +} + +/** Write an 8-byte value to f in little-endian order. */ +void Write8(long long value, FILE* f) { + fputc(value & 0xff, f); + fputc((value >> 8) & 0xff, f); + fputc((value >> 16) & 0xff, f); + fputc((value >> 24) & 0xff, f); + fputc((value >> 32) & 0xff, f); + fputc((value >> 40) & 0xff, f); + fputc((value >> 48) & 0xff, f); + fputc((value >> 56) & 0xff, f); +} + +int Read2(unsigned char* p) { + return (int)(((unsigned int)p[1] << 8) | + (unsigned int)p[0]); +} + +int Read4(unsigned char* p) { + return (int)(((unsigned int)p[3] << 24) | + ((unsigned int)p[2] << 16) | + ((unsigned int)p[1] << 8) | + (unsigned int)p[0]); +} + +long long Read8(unsigned char* p) { + return (long long)(((unsigned long long)p[7] << 56) | + ((unsigned long long)p[6] << 48) | + ((unsigned long long)p[5] << 40) | + ((unsigned long long)p[4] << 32) | + ((unsigned long long)p[3] << 24) | + ((unsigned long long)p[2] << 16) | + ((unsigned long long)p[1] << 8) | + (unsigned long long)p[0]); +} diff --git a/applypatch/utils.h b/applypatch/utils.h new file mode 100644 index 0000000..d6d6f1d --- /dev/null +++ b/applypatch/utils.h @@ -0,0 +1,30 @@ +/* + * Copyright (C) 2009 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef _BUILD_TOOLS_APPLYPATCH_UTILS_H +#define _BUILD_TOOLS_APPLYPATCH_UTILS_H + +#include <stdio.h> + +// Read and write little-endian values of various sizes. + +void Write4(int value, FILE* f); +void Write8(long long value, FILE* f); +int Read2(unsigned char* p); +int Read4(unsigned char* p); +long long Read8(unsigned char* p); + +#endif // _BUILD_TOOLS_APPLYPATCH_UTILS_H diff --git a/edify/expr.c b/edify/expr.c index df3c1ab..7a5b2fb 100644 --- a/edify/expr.c +++ b/edify/expr.c @@ -33,12 +33,39 @@ int BooleanString(const char* s) { } char* Evaluate(State* state, Expr* expr) { + Value* v = expr->fn(expr->name, state, expr->argc, expr->argv); + if (v == NULL) return NULL; + if (v->type != VAL_STRING) { + ErrorAbort(state, "expecting string, got value type %d", v->type); + FreeValue(v); + return NULL; + } + char* result = v->data; + free(v); + return result; +} + +Value* EvaluateValue(State* state, Expr* expr) { return expr->fn(expr->name, state, expr->argc, expr->argv); } -char* ConcatFn(const char* name, State* state, int argc, Expr* argv[]) { +Value* StringValue(char* str) { + Value* v = malloc(sizeof(Value)); + v->type = VAL_STRING; + v->size = strlen(str); + v->data = str; + return v; +} + +void FreeValue(Value* v) { + if (v == NULL) return; + free(v->data); + free(v); +} + +Value* ConcatFn(const char* name, State* state, int argc, Expr* argv[]) { if (argc == 0) { - return strdup(""); + return StringValue(strdup("")); } char** strings = malloc(argc * sizeof(char*)); int i; @@ -68,10 +95,10 @@ char* ConcatFn(const char* name, State* state, int argc, Expr* argv[]) { free(strings[i]); } free(strings); - return result; + return StringValue(result); } -char* IfElseFn(const char* name, State* state, int argc, Expr* argv[]) { +Value* IfElseFn(const char* name, State* state, int argc, Expr* argv[]) { if (argc != 2 && argc != 3) { free(state->errmsg); state->errmsg = strdup("ifelse expects 2 or 3 arguments"); @@ -84,18 +111,18 @@ char* IfElseFn(const char* name, State* state, int argc, Expr* argv[]) { if (BooleanString(cond) == true) { free(cond); - return Evaluate(state, argv[1]); + return EvaluateValue(state, argv[1]); } else { if (argc == 3) { free(cond); - return Evaluate(state, argv[2]); + return EvaluateValue(state, argv[2]); } else { - return cond; + return StringValue(cond); } } } -char* AbortFn(const char* name, State* state, int argc, Expr* argv[]) { +Value* AbortFn(const char* name, State* state, int argc, Expr* argv[]) { char* msg = NULL; if (argc > 0) { msg = Evaluate(state, argv[0]); @@ -109,7 +136,7 @@ char* AbortFn(const char* name, State* state, int argc, Expr* argv[]) { return NULL; } -char* AssertFn(const char* name, State* state, int argc, Expr* argv[]) { +Value* AssertFn(const char* name, State* state, int argc, Expr* argv[]) { int i; for (i = 0; i < argc; ++i) { char* v = Evaluate(state, argv[i]); @@ -131,20 +158,20 @@ char* AssertFn(const char* name, State* state, int argc, Expr* argv[]) { return NULL; } } - return strdup(""); + return StringValue(strdup("")); } -char* SleepFn(const char* name, State* state, int argc, Expr* argv[]) { +Value* SleepFn(const char* name, State* state, int argc, Expr* argv[]) { char* val = Evaluate(state, argv[0]); if (val == NULL) { return NULL; } int v = strtol(val, NULL, 10); sleep(v); - return val; + return StringValue(val); } -char* StdoutFn(const char* name, State* state, int argc, Expr* argv[]) { +Value* StdoutFn(const char* name, State* state, int argc, Expr* argv[]) { int i; for (i = 0; i < argc; ++i) { char* v = Evaluate(state, argv[i]); @@ -154,48 +181,44 @@ char* StdoutFn(const char* name, State* state, int argc, Expr* argv[]) { fputs(v, stdout); free(v); } - return strdup(""); + return StringValue(strdup("")); } -char* LogicalAndFn(const char* name, State* state, +Value* LogicalAndFn(const char* name, State* state, int argc, Expr* argv[]) { char* left = Evaluate(state, argv[0]); if (left == NULL) return NULL; if (BooleanString(left) == true) { free(left); - return Evaluate(state, argv[1]); + return EvaluateValue(state, argv[1]); } else { - return left; + return StringValue(left); } } -char* LogicalOrFn(const char* name, State* state, - int argc, Expr* argv[]) { +Value* LogicalOrFn(const char* name, State* state, + int argc, Expr* argv[]) { char* left = Evaluate(state, argv[0]); if (left == NULL) return NULL; if (BooleanString(left) == false) { free(left); - return Evaluate(state, argv[1]); + return EvaluateValue(state, argv[1]); } else { - return left; + return StringValue(left); } } -char* LogicalNotFn(const char* name, State* state, - int argc, Expr* argv[]) { +Value* LogicalNotFn(const char* name, State* state, + int argc, Expr* argv[]) { char* val = Evaluate(state, argv[0]); if (val == NULL) return NULL; bool bv = BooleanString(val); free(val); - if (bv) { - return strdup(""); - } else { - return strdup("t"); - } + return StringValue(strdup(bv ? "" : "t")); } -char* SubstringFn(const char* name, State* state, - int argc, Expr* argv[]) { +Value* SubstringFn(const char* name, State* state, + int argc, Expr* argv[]) { char* needle = Evaluate(state, argv[0]); if (needle == NULL) return NULL; char* haystack = Evaluate(state, argv[1]); @@ -207,10 +230,10 @@ char* SubstringFn(const char* name, State* state, char* result = strdup(strstr(haystack, needle) ? "t" : ""); free(needle); free(haystack); - return result; + return StringValue(result); } -char* EqualityFn(const char* name, State* state, int argc, Expr* argv[]) { +Value* EqualityFn(const char* name, State* state, int argc, Expr* argv[]) { char* left = Evaluate(state, argv[0]); if (left == NULL) return NULL; char* right = Evaluate(state, argv[1]); @@ -222,10 +245,10 @@ char* EqualityFn(const char* name, State* state, int argc, Expr* argv[]) { char* result = strdup(strcmp(left, right) == 0 ? "t" : ""); free(left); free(right); - return result; + return StringValue(result); } -char* InequalityFn(const char* name, State* state, int argc, Expr* argv[]) { +Value* InequalityFn(const char* name, State* state, int argc, Expr* argv[]) { char* left = Evaluate(state, argv[0]); if (left == NULL) return NULL; char* right = Evaluate(state, argv[1]); @@ -237,17 +260,17 @@ char* InequalityFn(const char* name, State* state, int argc, Expr* argv[]) { char* result = strdup(strcmp(left, right) != 0 ? "t" : ""); free(left); free(right); - return result; + return StringValue(result); } -char* SequenceFn(const char* name, State* state, int argc, Expr* argv[]) { - char* left = Evaluate(state, argv[0]); +Value* SequenceFn(const char* name, State* state, int argc, Expr* argv[]) { + Value* left = EvaluateValue(state, argv[0]); if (left == NULL) return NULL; - free(left); - return Evaluate(state, argv[1]); + FreeValue(left); + return EvaluateValue(state, argv[1]); } -char* LessThanIntFn(const char* name, State* state, int argc, Expr* argv[]) { +Value* LessThanIntFn(const char* name, State* state, int argc, Expr* argv[]) { if (argc != 2) { free(state->errmsg); state->errmsg = strdup("less_than_int expects 2 arguments"); @@ -278,10 +301,11 @@ char* LessThanIntFn(const char* name, State* state, int argc, Expr* argv[]) { done: free(left); free(right); - return strdup(result ? "t" : ""); + return StringValue(strdup(result ? "t" : "")); } -char* GreaterThanIntFn(const char* name, State* state, int argc, Expr* argv[]) { +Value* GreaterThanIntFn(const char* name, State* state, + int argc, Expr* argv[]) { if (argc != 2) { free(state->errmsg); state->errmsg = strdup("greater_than_int expects 2 arguments"); @@ -295,8 +319,8 @@ char* GreaterThanIntFn(const char* name, State* state, int argc, Expr* argv[]) { return LessThanIntFn(name, state, 2, temp); } -char* Literal(const char* name, State* state, int argc, Expr* argv[]) { - return strdup(name); +Value* Literal(const char* name, State* state, int argc, Expr* argv[]) { + return StringValue(strdup(name)); } Expr* Build(Function fn, YYLTYPE loc, int count, ...) { @@ -400,6 +424,32 @@ int ReadArgs(State* state, Expr* argv[], int count, ...) { return 0; } +// Evaluate the expressions in argv, giving 'count' Value* (the ... is +// zero or more Value** to put them in). If any expression evaluates +// to NULL, free the rest and return -1. Return 0 on success. +int ReadValueArgs(State* state, Expr* argv[], int count, ...) { + Value** args = malloc(count * sizeof(Value*)); + va_list v; + va_start(v, count); + int i; + for (i = 0; i < count; ++i) { + args[i] = EvaluateValue(state, argv[i]); + if (args[i] == NULL) { + va_end(v); + int j; + for (j = 0; j < i; ++j) { + FreeValue(args[j]); + } + free(args); + return -1; + } + *(va_arg(v, Value**)) = args[i]; + } + va_end(v); + free(args); + return 0; +} + // Evaluate the expressions in argv, returning an array of char* // results. If any evaluate to NULL, free the rest and return NULL. // The caller is responsible for freeing the returned array and the @@ -421,9 +471,30 @@ char** ReadVarArgs(State* state, int argc, Expr* argv[]) { return args; } +// Evaluate the expressions in argv, returning an array of Value* +// results. If any evaluate to NULL, free the rest and return NULL. +// The caller is responsible for freeing the returned array and the +// Values it contains. +Value** ReadValueVarArgs(State* state, int argc, Expr* argv[]) { + Value** args = (Value**)malloc(argc * sizeof(Value*)); + int i = 0; + for (i = 0; i < argc; ++i) { + args[i] = EvaluateValue(state, argv[i]); + if (args[i] == NULL) { + int j; + for (j = 0; j < i; ++j) { + FreeValue(args[j]); + } + free(args); + return NULL; + } + } + return args; +} + // Use printf-style arguments to compose an error message to put into // *state. Returns NULL. -char* ErrorAbort(State* state, char* format, ...) { +Value* ErrorAbort(State* state, char* format, ...) { char* buffer = malloc(4096); va_list v; va_start(v, format); diff --git a/edify/expr.h b/edify/expr.h index d2e7392..1462531 100644 --- a/edify/expr.h +++ b/edify/expr.h @@ -39,8 +39,17 @@ typedef struct { char* errmsg; } State; -typedef char* (*Function)(const char* name, State* state, - int argc, Expr* argv[]); +#define VAL_STRING 1 // data will be NULL-terminated; size doesn't count null +#define VAL_BLOB 2 + +typedef struct { + int type; + ssize_t size; + char* data; +} Value; + +typedef Value* (*Function)(const char* name, State* state, + int argc, Expr* argv[]); struct Expr { Function fn; @@ -50,31 +59,41 @@ struct Expr { int start, end; }; +// Take one of the Expr*s passed to the function as an argument, +// evaluate it, return the resulting Value. The caller takes +// ownership of the returned Value. +Value* EvaluateValue(State* state, Expr* expr); + +// Take one of the Expr*s passed to the function as an argument, +// evaluate it, assert that it is a string, and return the resulting +// char*. The caller takes ownership of the returned char*. This is +// a convenience function for older functions that want to deal only +// with strings. char* Evaluate(State* state, Expr* expr); // Glue to make an Expr out of a literal. -char* Literal(const char* name, State* state, int argc, Expr* argv[]); +Value* Literal(const char* name, State* state, int argc, Expr* argv[]); // Functions corresponding to various syntactic sugar operators. // ("concat" is also available as a builtin function, to concatenate // more than two strings.) -char* ConcatFn(const char* name, State* state, int argc, Expr* argv[]); -char* LogicalAndFn(const char* name, State* state, int argc, Expr* argv[]); -char* LogicalOrFn(const char* name, State* state, int argc, Expr* argv[]); -char* LogicalNotFn(const char* name, State* state, int argc, Expr* argv[]); -char* SubstringFn(const char* name, State* state, int argc, Expr* argv[]); -char* EqualityFn(const char* name, State* state, int argc, Expr* argv[]); -char* InequalityFn(const char* name, State* state, int argc, Expr* argv[]); -char* SequenceFn(const char* name, State* state, int argc, Expr* argv[]); +Value* ConcatFn(const char* name, State* state, int argc, Expr* argv[]); +Value* LogicalAndFn(const char* name, State* state, int argc, Expr* argv[]); +Value* LogicalOrFn(const char* name, State* state, int argc, Expr* argv[]); +Value* LogicalNotFn(const char* name, State* state, int argc, Expr* argv[]); +Value* SubstringFn(const char* name, State* state, int argc, Expr* argv[]); +Value* EqualityFn(const char* name, State* state, int argc, Expr* argv[]); +Value* InequalityFn(const char* name, State* state, int argc, Expr* argv[]); +Value* SequenceFn(const char* name, State* state, int argc, Expr* argv[]); // Convenience function for building expressions with a fixed number // of arguments. Expr* Build(Function fn, YYLTYPE loc, int count, ...); // Global builtins, registered by RegisterBuiltins(). -char* IfElseFn(const char* name, State* state, int argc, Expr* argv[]); -char* AssertFn(const char* name, State* state, int argc, Expr* argv[]); -char* AbortFn(const char* name, State* state, int argc, Expr* argv[]); +Value* IfElseFn(const char* name, State* state, int argc, Expr* argv[]); +Value* AssertFn(const char* name, State* state, int argc, Expr* argv[]); +Value* AbortFn(const char* name, State* state, int argc, Expr* argv[]); // For setting and getting the global error string (when returning @@ -112,15 +131,31 @@ Function FindFunction(const char* name); // to NULL, free the rest and return -1. Return 0 on success. int ReadArgs(State* state, Expr* argv[], int count, ...); +// Evaluate the expressions in argv, giving 'count' Value* (the ... is +// zero or more Value** to put them in). If any expression evaluates +// to NULL, free the rest and return -1. Return 0 on success. +int ReadValueArgs(State* state, Expr* argv[], int count, ...); + // Evaluate the expressions in argv, returning an array of char* // results. If any evaluate to NULL, free the rest and return NULL. // The caller is responsible for freeing the returned array and the // strings it contains. char** ReadVarArgs(State* state, int argc, Expr* argv[]); +// Evaluate the expressions in argv, returning an array of Value* +// results. If any evaluate to NULL, free the rest and return NULL. +// The caller is responsible for freeing the returned array and the +// Values it contains. +Value** ReadValueVarArgs(State* state, int argc, Expr* argv[]); + // Use printf-style arguments to compose an error message to put into // *state. Returns NULL. -char* ErrorAbort(State* state, char* format, ...); +Value* ErrorAbort(State* state, char* format, ...); + +// Wrap a string into a Value, taking ownership of the string. +Value* StringValue(char* str); +// Free a Value object. +void FreeValue(Value* v); #endif // _EXPRESSION_H diff --git a/updater/install.c b/updater/install.c index 852b393..2ffb384 100644 --- a/updater/install.c +++ b/updater/install.c @@ -29,17 +29,18 @@ #include "cutils/misc.h" #include "cutils/properties.h" #include "edify/expr.h" +#include "mincrypt/sha.h" #include "minzip/DirUtil.h" #include "mtdutils/mounts.h" #include "mtdutils/mtdutils.h" #include "updater.h" - +#include "applypatch/applypatch.h" // mount(type, location, mount_point) // // what: type="MTD" location="<partition>" to mount a yaffs2 filesystem // type="vfat" location="/dev/block/<whatever>" to mount a device -char* MountFn(const char* name, State* state, int argc, Expr* argv[]) { +Value* MountFn(const char* name, State* state, int argc, Expr* argv[]) { char* result = NULL; if (argc != 3) { return ErrorAbort(state, "%s() expects 3 args, got %d", name, argc); @@ -98,12 +99,12 @@ done: free(type); free(location); if (result != mount_point) free(mount_point); - return result; + return StringValue(result); } // is_mounted(mount_point) -char* IsMountedFn(const char* name, State* state, int argc, Expr* argv[]) { +Value* IsMountedFn(const char* name, State* state, int argc, Expr* argv[]) { char* result = NULL; if (argc != 1) { return ErrorAbort(state, "%s() expects 1 arg, got %d", name, argc); @@ -127,11 +128,11 @@ char* IsMountedFn(const char* name, State* state, int argc, Expr* argv[]) { done: if (result != mount_point) free(mount_point); - return result; + return StringValue(result); } -char* UnmountFn(const char* name, State* state, int argc, Expr* argv[]) { +Value* UnmountFn(const char* name, State* state, int argc, Expr* argv[]) { char* result = NULL; if (argc != 1) { return ErrorAbort(state, "%s() expects 1 arg, got %d", name, argc); @@ -157,14 +158,14 @@ char* UnmountFn(const char* name, State* state, int argc, Expr* argv[]) { done: if (result != mount_point) free(mount_point); - return result; + return StringValue(result); } // format(type, location) // // type="MTD" location=partition -char* FormatFn(const char* name, State* state, int argc, Expr* argv[]) { +Value* FormatFn(const char* name, State* state, int argc, Expr* argv[]) { char* result = NULL; if (argc != 2) { return ErrorAbort(state, "%s() expects 2 args, got %d", name, argc); @@ -218,11 +219,11 @@ char* FormatFn(const char* name, State* state, int argc, Expr* argv[]) { done: free(type); if (result != location) free(location); - return result; + return StringValue(result); } -char* DeleteFn(const char* name, State* state, int argc, Expr* argv[]) { +Value* DeleteFn(const char* name, State* state, int argc, Expr* argv[]) { char** paths = malloc(argc * sizeof(char*)); int i; for (i = 0; i < argc; ++i) { @@ -249,11 +250,11 @@ char* DeleteFn(const char* name, State* state, int argc, Expr* argv[]) { char buffer[10]; sprintf(buffer, "%d", success); - return strdup(buffer); + return StringValue(strdup(buffer)); } -char* ShowProgressFn(const char* name, State* state, int argc, Expr* argv[]) { +Value* ShowProgressFn(const char* name, State* state, int argc, Expr* argv[]) { if (argc != 2) { return ErrorAbort(state, "%s() expects 2 args, got %d", name, argc); } @@ -270,10 +271,10 @@ char* ShowProgressFn(const char* name, State* state, int argc, Expr* argv[]) { fprintf(ui->cmd_pipe, "progress %f %d\n", frac, sec); free(sec_str); - return frac_str; + return StringValue(frac_str); } -char* SetProgressFn(const char* name, State* state, int argc, Expr* argv[]) { +Value* SetProgressFn(const char* name, State* state, int argc, Expr* argv[]) { if (argc != 1) { return ErrorAbort(state, "%s() expects 1 arg, got %d", name, argc); } @@ -287,11 +288,11 @@ char* SetProgressFn(const char* name, State* state, int argc, Expr* argv[]) { UpdaterInfo* ui = (UpdaterInfo*)(state->cookie); fprintf(ui->cmd_pipe, "set_progress %f\n", frac); - return frac_str; + return StringValue(frac_str); } // package_extract_dir(package_path, destination_path) -char* PackageExtractDirFn(const char* name, State* state, +Value* PackageExtractDirFn(const char* name, State* state, int argc, Expr* argv[]) { if (argc != 2) { return ErrorAbort(state, "%s() expects 2 args, got %d", name, argc); @@ -310,7 +311,7 @@ char* PackageExtractDirFn(const char* name, State* state, NULL, NULL); free(zip_path); free(dest_path); - return strdup(success ? "t" : ""); + return StringValue(strdup(success ? "t" : "")); } @@ -318,9 +319,8 @@ char* PackageExtractDirFn(const char* name, State* state, // or // package_extract_file(package_path) // to return the entire contents of the file as the result of this -// function (the char* returned points to a long giving the length -// followed by that many bytes of data). -char* PackageExtractFileFn(const char* name, State* state, +// function (the char* returned is actually a FileContents*). +Value* PackageExtractFileFn(const char* name, State* state, int argc, Expr* argv[]) { if (argc != 1 && argc != 2) { return ErrorAbort(state, "%s() expects 1 or 2 args, got %d", @@ -353,13 +353,16 @@ char* PackageExtractFileFn(const char* name, State* state, done2: free(zip_path); free(dest_path); - return strdup(success ? "t" : ""); + return StringValue(strdup(success ? "t" : "")); } else { // The one-argument version returns the contents of the file // as the result. char* zip_path; - char* buffer = NULL; + Value* v = malloc(sizeof(Value)); + v->type = VAL_BLOB; + v->size = -1; + v->data = NULL; if (ReadArgs(state, argv, 1, &zip_path) < 0) return NULL; @@ -370,33 +373,32 @@ char* PackageExtractFileFn(const char* name, State* state, goto done1; } - long size = mzGetZipEntryUncompLen(entry); - buffer = malloc(size + sizeof(long)); - if (buffer == NULL) { + v->size = mzGetZipEntryUncompLen(entry); + v->data = malloc(v->size); + if (v->data == NULL) { fprintf(stderr, "%s: failed to allocate %ld bytes for %s\n", - name, size+sizeof(long), zip_path); + name, (long)v->size, zip_path); goto done1; } - *(long *)buffer = size; - success = mzExtractZipEntryToBuffer( - za, entry, (unsigned char *)(buffer + sizeof(long))); + success = mzExtractZipEntryToBuffer(za, entry, + (unsigned char *)v->data); done1: free(zip_path); if (!success) { - free(buffer); - buffer = malloc(sizeof(long)); - *(long *)buffer = -1L; + free(v->data); + v->data = NULL; + v->size = -1; } - return buffer; + return v; } } // symlink target src1 src2 ... // unlinks any previously existing src1, src2, etc before creating symlinks. -char* SymlinkFn(const char* name, State* state, int argc, Expr* argv[]) { +Value* SymlinkFn(const char* name, State* state, int argc, Expr* argv[]) { if (argc == 0) { return ErrorAbort(state, "%s() expects 1+ args, got %d", name, argc); } @@ -425,11 +427,11 @@ char* SymlinkFn(const char* name, State* state, int argc, Expr* argv[]) { free(srcs[i]); } free(srcs); - return strdup(""); + return StringValue(strdup("")); } -char* SetPermFn(const char* name, State* state, int argc, Expr* argv[]) { +Value* SetPermFn(const char* name, State* state, int argc, Expr* argv[]) { char* result = NULL; bool recursive = (strcmp(name, "set_perm_recursive") == 0); @@ -499,11 +501,11 @@ done: } free(args); - return result; + return StringValue(result); } -char* GetPropFn(const char* name, State* state, int argc, Expr* argv[]) { +Value* GetPropFn(const char* name, State* state, int argc, Expr* argv[]) { if (argc != 1) { return ErrorAbort(state, "%s() expects 1 arg, got %d", name, argc); } @@ -515,7 +517,7 @@ char* GetPropFn(const char* name, State* state, int argc, Expr* argv[]) { property_get(key, value, ""); free(key); - return strdup(value); + return StringValue(strdup(value)); } @@ -524,7 +526,7 @@ char* GetPropFn(const char* name, State* state, int argc, Expr* argv[]) { // interprets 'file' as a getprop-style file (key=value pairs, one // per line, # comment lines and blank lines okay), and returns the value // for 'key' (or "" if it isn't defined). -char* FileGetPropFn(const char* name, State* state, int argc, Expr* argv[]) { +Value* FileGetPropFn(const char* name, State* state, int argc, Expr* argv[]) { char* result = NULL; char* buffer = NULL; char* filename; @@ -614,7 +616,7 @@ char* FileGetPropFn(const char* name, State* state, int argc, Expr* argv[]) { free(filename); free(key); free(buffer); - return result; + return StringValue(result); } @@ -627,7 +629,7 @@ static bool write_raw_image_cb(const unsigned char* data, } // write_raw_image(file, partition) -char* WriteRawImageFn(const char* name, State* state, int argc, Expr* argv[]) { +Value* WriteRawImageFn(const char* name, State* state, int argc, Expr* argv[]) { char* result = NULL; char* partition; @@ -700,15 +702,13 @@ char* WriteRawImageFn(const char* name, State* state, int argc, Expr* argv[]) { done: if (result != partition) free(partition); free(filename); - return result; + return StringValue(result); } -extern int applypatch(int argc, char** argv); - // apply_patch(srcfile, tgtfile, tgtsha1, tgtsize, sha1:patch, ...) // apply_patch_check(file, sha1, ...) // apply_patch_space(bytes) -char* ApplyPatchFn(const char* name, State* state, int argc, Expr* argv[]) { +Value* ApplyPatchFn(const char* name, State* state, int argc, Expr* argv[]) { printf("in applypatchfn (%s)\n", name); char* prepend = NULL; @@ -747,13 +747,13 @@ char* ApplyPatchFn(const char* name, State* state, int argc, Expr* argv[]) { free(args); switch (result) { - case 0: return strdup("t"); - case 1: return strdup(""); + case 0: return StringValue(strdup("t")); + case 1: return StringValue(strdup("")); default: return ErrorAbort(state, "applypatch couldn't parse args"); } } -char* UIPrintFn(const char* name, State* state, int argc, Expr* argv[]) { +Value* UIPrintFn(const char* name, State* state, int argc, Expr* argv[]) { char** args = ReadVarArgs(state, argc, argv); if (args == NULL) { return NULL; @@ -782,10 +782,10 @@ char* UIPrintFn(const char* name, State* state, int argc, Expr* argv[]) { } fprintf(((UpdaterInfo*)(state->cookie))->cmd_pipe, "ui_print\n"); - return buffer; + return StringValue(buffer); } -char* RunProgramFn(const char* name, State* state, int argc, Expr* argv[]) { +Value* RunProgramFn(const char* name, State* state, int argc, Expr* argv[]) { if (argc < 1) { return ErrorAbort(state, "%s() expects at least 1 arg", name); } @@ -828,9 +828,138 @@ char* RunProgramFn(const char* name, State* state, int argc, Expr* argv[]) { char buffer[20]; sprintf(buffer, "%d", status); - return strdup(buffer); + return StringValue(strdup(buffer)); } +// Take a string 'str' of 40 hex digits and parse it into the 20 +// byte array 'digest'. 'str' may contain only the digest or be of +// the form "<digest>:<anything>". Return 0 on success, -1 on any +// error. +static int ParseSha1(const char* str, uint8_t* digest) { + int i; + const char* ps = str; + uint8_t* pd = digest; + for (i = 0; i < SHA_DIGEST_SIZE * 2; ++i, ++ps) { + int digit; + if (*ps >= '0' && *ps <= '9') { + digit = *ps - '0'; + } else if (*ps >= 'a' && *ps <= 'f') { + digit = *ps - 'a' + 10; + } else if (*ps >= 'A' && *ps <= 'F') { + digit = *ps - 'A' + 10; + } else { + return -1; + } + if (i % 2 == 0) { + *pd = digit << 4; + } else { + *pd |= digit; + ++pd; + } + } + if (*ps != '\0') return -1; + return 0; +} + +// Take a sha-1 digest and return it as a newly-allocated hex string. +static char* PrintSha1(uint8_t* digest) { + char* buffer = malloc(SHA_DIGEST_SIZE*2 + 1); + int i; + const char* alphabet = "0123456789abcdef"; + for (i = 0; i < SHA_DIGEST_SIZE; ++i) { + buffer[i*2] = alphabet[(digest[i] >> 4) & 0xf]; + buffer[i*2+1] = alphabet[digest[i] & 0xf]; + } + buffer[i*2] = '\0'; + return buffer; +} + +// sha1_check(data) +// to return the sha1 of the data (given in the format returned by +// read_file). +// +// sha1_check(data, sha1_hex, [sha1_hex, ...]) +// returns the sha1 of the file if it matches any of the hex +// strings passed, or "" if it does not equal any of them. +// +Value* Sha1CheckFn(const char* name, State* state, int argc, Expr* argv[]) { + if (argc < 1) { + return ErrorAbort(state, "%s() expects at least 1 arg", name); + } + + Value** args = ReadValueVarArgs(state, argc, argv); + if (args == NULL) { + return NULL; + } + + if (args[0]->size < 0) { + fprintf(stderr, "%s(): no file contents received", name); + return StringValue(strdup("")); + } + uint8_t digest[SHA_DIGEST_SIZE]; + SHA(args[0]->data, args[0]->size, digest); + FreeValue(args[0]); + + if (argc == 1) { + return StringValue(PrintSha1(digest)); + } + + int i; + uint8_t* arg_digest = malloc(SHA_DIGEST_SIZE); + for (i = 1; i < argc; ++i) { + if (args[i]->type != VAL_STRING) { + fprintf(stderr, "%s(): arg %d is not a string; skipping", + name, i); + } else if (ParseSha1(args[i]->data, arg_digest) != 0) { + // Warn about bad args and skip them. + fprintf(stderr, "%s(): error parsing \"%s\" as sha-1; skipping", + name, args[i]->data); + } else if (memcmp(digest, arg_digest, SHA_DIGEST_SIZE) == 0) { + break; + } + FreeValue(args[i]); + } + if (i >= argc) { + // Didn't match any of the hex strings; return false. + return StringValue(strdup("")); + } + // Found a match; free all the remaining arguments and return the + // matched one. + int j; + for (j = i+1; j < argc; ++j) { + FreeValue(args[j]); + } + return args[i]; +} + +// Read a local file and return its contents (the char* returned +// is actually a FileContents*). +Value* ReadFileFn(const char* name, State* state, int argc, Expr* argv[]) { + if (argc != 1) { + return ErrorAbort(state, "%s() expects 1 arg, got %d", name, argc); + } + char* filename; + if (ReadArgs(state, argv, 1, &filename) < 0) return NULL; + + Value* v = malloc(sizeof(Value)); + v->type = VAL_BLOB; + + FileContents fc; + if (LoadFileContents(filename, &fc) != 0) { + ErrorAbort(state, "%s() loading \"%s\" failed: %s", + name, filename, strerror(errno)); + free(filename); + free(v); + free(fc.data); + return NULL; + } + + v->size = fc.size; + v->data = (char*)fc.data; + + free(filename); + return v; +} void RegisterInstallFunctions() { RegisterFunction("mount", MountFn); @@ -855,6 +984,9 @@ void RegisterInstallFunctions() { RegisterFunction("apply_patch_check", ApplyPatchFn); RegisterFunction("apply_patch_space", ApplyPatchFn); + RegisterFunction("read_file", ReadFileFn); + RegisterFunction("sha1_check", Sha1CheckFn); + RegisterFunction("ui_print", UIPrintFn); RegisterFunction("run_program", RunProgramFn); diff --git a/updater/updater.c b/updater/updater.c index 2d16dee..6537a94 100644 --- a/updater/updater.c +++ b/updater/updater.c @@ -33,6 +33,12 @@ #define SCRIPT_NAME "META-INF/com/google/android/updater-script" int main(int argc, char** argv) { + // Various things log information to stdout or stderr more or less + // at random. The log file makes more sense if buffering is + // turned off so things appear in the right order. + setbuf(stdout, NULL); + setbuf(stderr, NULL); + if (argc != 4) { fprintf(stderr, "unexpected number of arguments (%d)\n", argc); return 1; |