Merge pull request #4705 from libgit2/ethomson/apply

Patch (diff) application

6 files changed, 946 insertions, 19 deletions

diff --git a/src/apply.c b/src/apply.c
index 8c7bb6bf3..614baf7a3 100644
--- a/src/apply.c
+++ b/src/apply.c
@@ -9,16 +9,23 @@
 
 #include <assert.h>
 
+#include "git2/apply.h"
 #include "git2/patch.h"
 #include "git2/filter.h"
+#include "git2/blob.h"
+#include "git2/index.h"
+#include "git2/checkout.h"
+#include "git2/repository.h"
 #include "array.h"
 #include "patch.h"
 #include "fileops.h"
 #include "delta.h"
 #include "zstream.h"
+#include "reader.h"
+#include "index.h"
 
 #define apply_err(...) \
-	( giterr_set(GITERR_PATCH, __VA_ARGS__), -1 )
+	( giterr_set(GITERR_PATCH, __VA_ARGS__), GIT_EAPPLYFAIL )
 
 typedef struct {
 	/* The lines that we allocate ourself are allocated out of the pool.
@@ -160,15 +167,36 @@ static int update_hunk(
 	return 0;
 }
 
+typedef struct {
+	git_apply_options opts;
+	size_t skipped_new_lines;
+	size_t skipped_old_lines;
+} apply_hunks_ctx;
+
 static int apply_hunk(
 	patch_image *image,
 	git_patch *patch,
-	git_patch_hunk *hunk)
+	git_patch_hunk *hunk,
+	apply_hunks_ctx *ctx)
 {
 	patch_image preimage = PATCH_IMAGE_INIT, postimage = PATCH_IMAGE_INIT;
 	size_t line_num, i;
 	int error = 0;
 
+	if (ctx->opts.hunk_cb) {
+		error = ctx->opts.hunk_cb(&hunk->hunk, ctx->opts.payload);
+
+		if (error) {
+			if (error > 0) {
+				ctx->skipped_new_lines += hunk->hunk.new_lines;
+				ctx->skipped_old_lines += hunk->hunk.old_lines;
+				error = 0;
+			}
+
+			goto done;
+		}
+	}
+
 	for (i = 0; i < hunk->line_count; i++) {
 		size_t linenum = hunk->line_start + i;
 		git_diff_line *line = git_array_get(patch->lines, linenum);
@@ -191,7 +219,14 @@ static int apply_hunk(
 		}
 	}
 
-	line_num = hunk->hunk.new_start ? hunk->hunk.new_start - 1 : 0;
+	if (hunk->hunk.new_start) {
+		line_num = hunk->hunk.new_start -
+			ctx->skipped_new_lines +
+			ctx->skipped_old_lines -
+			1;
+	} else {
+		line_num = 0;
+	}
 
 	if (!find_hunk_linenum(&line_num, image, &preimage, line_num)) {
 		error = apply_err("hunk at line %d did not apply",
@@ -212,7 +247,8 @@ static int apply_hunks(
 	git_buf *out,
 	const char *source,
 	size_t source_len,
-	git_patch *patch)
+	git_patch *patch,
+	apply_hunks_ctx *ctx)
 {
 	git_patch_hunk *hunk;
 	git_diff_line *line;
@@ -224,7 +260,7 @@ static int apply_hunks(
 		goto done;
 
 	git_array_foreach(patch->hunks, i, hunk) {
-		if ((error = apply_hunk(&image, patch, hunk)) < 0)
+		if ((error = apply_hunk(&image, patch, hunk, ctx)) < 0)
 			goto done;
 	}
 
@@ -332,14 +368,19 @@ int git_apply__patch(
 	unsigned int *mode_out,
 	const char *source,
 	size_t source_len,
-	git_patch *patch)
+	git_patch *patch,
+	const git_apply_options *given_opts)
 {
+	apply_hunks_ctx ctx = { GIT_APPLY_OPTIONS_INIT };
 	char *filename = NULL;
 	unsigned int mode = 0;
 	int error = 0;
 
 	assert(contents_out && filename_out && mode_out && (source || !source_len) && patch);
 
+	if (given_opts)
+		memcpy(&ctx.opts, given_opts, sizeof(git_apply_options));
+
 	*filename_out = NULL;
 	*mode_out = 0;
 
@@ -354,7 +395,7 @@ int git_apply__patch(
 	if (patch->delta->flags & GIT_DIFF_FLAG_BINARY)
 		error = apply_binary(contents_out, source, source_len, patch);
 	else if (patch->hunks.size)
-		error = apply_hunks(contents_out, source, source_len, patch);
+		error = apply_hunks(contents_out, source, source_len, patch, &ctx);
 	else
 		error = git_buf_put(contents_out, source, source_len);
 
@@ -376,3 +417,438 @@ done:
 
 	return error;
 }
+
+static int apply_one(
+	git_repository *repo,
+	git_reader *preimage_reader,
+	git_index *preimage,
+	git_reader *postimage_reader,
+	git_index *postimage,
+	git_diff *diff,
+	git_strmap *removed_paths,
+	size_t i,
+	const git_apply_options *opts)
+{
+	git_patch *patch = NULL;
+	git_buf pre_contents = GIT_BUF_INIT, post_contents = GIT_BUF_INIT;
+	const git_diff_delta *delta;
+	char *filename = NULL;
+	unsigned int mode;
+	git_oid pre_id, post_id;
+	git_filemode_t pre_filemode;
+	git_index_entry pre_entry, post_entry;
+	bool skip_preimage = false;
+	size_t pos;
+	int error;
+
+	if ((error = git_patch_from_diff(&patch, diff, i)) < 0)
+		goto done;
+
+	delta = git_patch_get_delta(patch);
+
+	if (opts->delta_cb) {
+		error = opts->delta_cb(delta, opts->payload);
+
+		if (error) {
+			if (error > 0)
+				error = 0;
+
+			goto done;
+		}
+	}
+
+	/*
+	 * Ensure that the file has not been deleted or renamed if we're
+	 * applying a modification delta.
+	 */
+	if (delta->status != GIT_DELTA_RENAMED &&
+	    delta->status != GIT_DELTA_ADDED) {
+		pos = git_strmap_lookup_index(removed_paths, delta->old_file.path);
+		if (git_strmap_valid_index(removed_paths, pos)) {
+			error = apply_err("path '%s' has been renamed or deleted", delta->old_file.path);
+			goto done;
+		}
+	}
+
+	/*
+	 * We may be applying a second delta to an already seen file.  If so,
+	 * use the already modified data in the postimage instead of the
+	 * content from the index or working directory.  (Don't do this in
+	 * the case of a rename, which must be specified before additional
+	 * deltas since we apply deltas to the target filename.)
+	 */
+	if (delta->status != GIT_DELTA_RENAMED) {
+		if ((error = git_reader_read(&pre_contents, &pre_id, &pre_filemode,
+		    postimage_reader, delta->old_file.path)) == 0) {
+			skip_preimage = true;
+		} else if (error == GIT_ENOTFOUND) {
+			giterr_clear();
+			error = 0;
+		} else {
+			goto done;
+		}
+	}
+
+	if (!skip_preimage && delta->status != GIT_DELTA_ADDED) {
+		error = git_reader_read(&pre_contents, &pre_id, &pre_filemode,
+			preimage_reader, delta->old_file.path);
+
+		/* ENOTFOUND means the preimage was not found; apply failed. */
+		if (error == GIT_ENOTFOUND)
+			error = GIT_EAPPLYFAIL;
+
+		/* When applying to BOTH, the index did not match the workdir. */
+		if (error == GIT_READER_MISMATCH)
+			error = apply_err("%s: does not match index", delta->old_file.path);
+
+		if (error < 0)
+			goto done;
+
+		/*
+		 * We need to populate the preimage data structure with the
+		 * contents that we are using as the preimage for this file.
+		 * This allows us to apply patches to files that have been
+		 * modified in the working directory.  During checkout,
+		 * we will use this expected preimage as the baseline, and
+		 * limit checkout to only the paths affected by patch
+		 * application.  (Without this, we would fail to write the
+		 * postimage contents to any file that had been modified
+		 * from HEAD on-disk, even if the patch application succeeded.)
+		 * Use the contents from the delta where available - some
+		 * fields may not be available, like the old file mode (eg in
+		 * an exact rename situation) so trust the patch parsing to
+		 * validate and use the preimage data in that case.
+		 */
+		if (preimage) {
+			memset(&pre_entry, 0, sizeof(git_index_entry));
+			pre_entry.path = delta->old_file.path;
+			pre_entry.mode = delta->old_file.mode ? delta->old_file.mode : pre_filemode;
+			git_oid_cpy(&pre_entry.id, &pre_id);
+
+			if ((error = git_index_add(preimage, &pre_entry)) < 0)
+				goto done;
+		}
+	}
+
+	if (delta->status != GIT_DELTA_DELETED) {
+		if ((error = git_apply__patch(&post_contents, &filename, &mode,
+				pre_contents.ptr, pre_contents.size, patch, opts)) < 0 ||
+			(error = git_blob_create_frombuffer(&post_id, repo,
+				post_contents.ptr, post_contents.size)) < 0)
+			goto done;
+
+		memset(&post_entry, 0, sizeof(git_index_entry));
+		post_entry.path = filename;
+		post_entry.mode = mode;
+		git_oid_cpy(&post_entry.id, &post_id);
+
+		if ((error = git_index_add(postimage, &post_entry)) < 0)
+			goto done;
+	}
+
+	if (delta->status == GIT_DELTA_RENAMED ||
+	    delta->status == GIT_DELTA_DELETED)
+		git_strmap_insert(removed_paths, delta->old_file.path, (char *)delta->old_file.path, &error);
+
+	if (delta->status == GIT_DELTA_RENAMED ||
+	    delta->status == GIT_DELTA_ADDED)
+		git_strmap_delete(removed_paths, delta->new_file.path);
+
+done:
+	git_buf_dispose(&pre_contents);
+	git_buf_dispose(&post_contents);
+	git__free(filename);
+	git_patch_free(patch);
+
+	return error;
+}
+
+static int apply_deltas(
+	git_repository *repo,
+	git_reader *pre_reader,
+	git_index *preimage,
+	git_reader *post_reader,
+	git_index *postimage,
+	git_diff *diff,
+	const git_apply_options *opts)
+{
+	git_strmap *removed_paths;
+	size_t i;
+	int error;
+
+	if (git_strmap_alloc(&removed_paths) < 0)
+		return -1;
+
+	for (i = 0; i < git_diff_num_deltas(diff); i++) {
+		if ((error = apply_one(repo, pre_reader, preimage, post_reader, postimage, diff, removed_paths, i, opts)) < 0)
+			goto done;
+	}
+
+done:
+	git_strmap_free(removed_paths);
+	return error;
+}
+
+int git_apply_to_tree(
+	git_index **out,
+	git_repository *repo,
+	git_tree *preimage,
+	git_diff *diff,
+	const git_apply_options *given_opts)
+{
+	git_index *postimage = NULL;
+	git_reader *pre_reader = NULL, *post_reader = NULL;
+	git_apply_options opts = GIT_APPLY_OPTIONS_INIT;
+	const git_diff_delta *delta;
+	size_t i;
+	int error = 0;
+
+	assert(out && repo && preimage && diff);
+
+	*out = NULL;
+
+	if (given_opts)
+		memcpy(&opts, given_opts, sizeof(git_apply_options));
+
+	if ((error = git_reader_for_tree(&pre_reader, preimage)) < 0)
+		goto done;
+
+	/*
+	 * put the current tree into the postimage as-is - the diff will
+	 * replace any entries contained therein
+	 */
+	if ((error = git_index_new(&postimage)) < 0 ||
+		(error = git_index_read_tree(postimage, preimage)) < 0 ||
+		(error = git_reader_for_index(&post_reader, repo, postimage)) < 0)
+		goto done;
+
+	/*
+	 * Remove the old paths from the index before applying diffs -
+	 * we need to do a full pass to remove them before adding deltas,
+	 * in order to handle rename situations.
+	 */
+	for (i = 0; i < git_diff_num_deltas(diff); i++) {
+		delta = git_diff_get_delta(diff, i);
+
+		if ((error = git_index_remove(postimage,
+				delta->old_file.path, 0)) < 0)
+			goto done;
+	}
+
+	if ((error = apply_deltas(repo, pre_reader, NULL, post_reader, postimage, diff, &opts)) < 0)
+		goto done;
+
+	*out = postimage;
+
+done:
+	if (error < 0)
+		git_index_free(postimage);
+
+	git_reader_free(pre_reader);
+	git_reader_free(post_reader);
+
+	return error;
+}
+
+static int git_apply__to_workdir(
+	git_repository *repo,
+	git_diff *diff,
+	git_index *preimage,
+	git_index *postimage,
+	git_apply_location_t location,
+	git_apply_options *opts)
+{
+	git_vector paths = GIT_VECTOR_INIT;
+	git_checkout_options checkout_opts = GIT_CHECKOUT_OPTIONS_INIT;
+	const git_diff_delta *delta;
+	size_t i;
+	int error;
+
+	GIT_UNUSED(opts);
+
+	/*
+	 * Limit checkout to the paths affected by the diff; this ensures
+	 * that other modifications in the working directory are unaffected.
+	 */
+	if ((error = git_vector_init(&paths, git_diff_num_deltas(diff), NULL)) < 0)
+		goto done;
+
+	for (i = 0; i < git_diff_num_deltas(diff); i++) {
+		delta = git_diff_get_delta(diff, i);
+
+		if ((error = git_vector_insert(&paths, (void *)delta->old_file.path)) < 0)
+			goto done;
+
+		if (strcmp(delta->old_file.path, delta->new_file.path) &&
+		    (error = git_vector_insert(&paths, (void *)delta->new_file.path)) < 0)
+			goto done;
+	}
+
+	checkout_opts.checkout_strategy |= GIT_CHECKOUT_SAFE;
+	checkout_opts.checkout_strategy |= GIT_CHECKOUT_DISABLE_PATHSPEC_MATCH;
+	checkout_opts.checkout_strategy |= GIT_CHECKOUT_DONT_WRITE_INDEX;
+
+	if (location == GIT_APPLY_LOCATION_WORKDIR)
+		checkout_opts.checkout_strategy |= GIT_CHECKOUT_DONT_UPDATE_INDEX;
+
+	checkout_opts.paths.strings = (char **)paths.contents;
+	checkout_opts.paths.count = paths.length;
+
+	checkout_opts.baseline_index = preimage;
+
+	error = git_checkout_index(repo, postimage, &checkout_opts);
+
+done:
+	git_vector_free(&paths);
+	return error;
+}
+
+static int git_apply__to_index(
+	git_repository *repo,
+	git_diff *diff,
+	git_index *preimage,
+	git_index *postimage,
+	git_apply_options *opts)
+{
+	git_index *index = NULL;
+	const git_diff_delta *delta;
+	const git_index_entry *entry;
+	size_t i;
+	int error;
+
+	GIT_UNUSED(preimage);
+	GIT_UNUSED(opts);
+
+	if ((error = git_repository_index(&index, repo)) < 0)
+		goto done;
+
+	/* Remove deleted (or renamed) paths from the index. */
+	for (i = 0; i < git_diff_num_deltas(diff); i++) {
+		delta = git_diff_get_delta(diff, i);
+
+		if (delta->status == GIT_DELTA_DELETED ||
+		    delta->status == GIT_DELTA_RENAMED) {
+			if ((error = git_index_remove(index, delta->old_file.path, 0)) < 0)
+				goto done;
+		}
+	}
+
+	/* Then add the changes back to the index. */
+	for (i = 0; i < git_index_entrycount(postimage); i++) {
+		entry = git_index_get_byindex(postimage, i);
+
+		if ((error = git_index_add(index, entry)) < 0)
+			goto done;
+	}
+
+done:
+	git_index_free(index);
+	return error;
+}
+
+/*
+ * Handle the three application options ("locations"):
+ *
+ * GIT_APPLY_LOCATION_WORKDIR: the default, emulates `git apply`.
+ * Applies the diff only to the workdir items and ignores the index
+ * entirely.
+ *
+ * GIT_APPLY_LOCATION_INDEX: emulates `git apply --cached`.
+ * Applies the diff only to the index items and ignores the workdir
+ * completely.
+ *
+ * GIT_APPLY_LOCATION_BOTH: emulates `git apply --index`.
+ * Applies the diff to both the index items and the working directory
+ * items.
+ */
+
+int git_apply(
+	git_repository *repo,
+	git_diff *diff,
+	git_apply_location_t location,
+	const git_apply_options *given_opts)
+{
+	git_indexwriter indexwriter = GIT_INDEXWRITER_INIT;
+	git_index *index = NULL, *preimage = NULL, *postimage = NULL;
+	git_reader *pre_reader = NULL, *post_reader = NULL;
+	git_apply_options opts = GIT_APPLY_OPTIONS_INIT;
+	int error = GIT_EINVALID;
+
+	assert(repo && diff);
+
+	GITERR_CHECK_VERSION(
+		given_opts, GIT_APPLY_OPTIONS_VERSION, "git_apply_options");
+
+	if (given_opts)
+		memcpy(&opts, given_opts, sizeof(git_apply_options));
+
+	/*
+	 * by default, we apply a patch directly to the working directory;
+	 * in `--cached` or `--index` mode, we apply to the contents already
+	 * in the index.
+	 */
+	switch (location) {
+	case GIT_APPLY_LOCATION_BOTH:
+		error = git_reader_for_workdir(&pre_reader, repo, true);
+		break;
+	case GIT_APPLY_LOCATION_INDEX:
+		error = git_reader_for_index(&pre_reader, repo, NULL);
+		break;
+	case GIT_APPLY_LOCATION_WORKDIR:
+		error = git_reader_for_workdir(&pre_reader, repo, false);
+		break;
+	default:
+		assert(false);
+	}
+
+	if (error < 0)
+		goto done;
+
+	/*
+	 * Build the preimage and postimage (differences).  Note that
+	 * this is not the complete preimage or postimage, it only
+	 * contains the files affected by the patch.  We want to avoid
+	 * having the full repo index, so we will limit our checkout
+	 * to only write these files that were affected by the diff.
+	 */
+	if ((error = git_index_new(&preimage)) < 0 ||
+	    (error = git_index_new(&postimage)) < 0 ||
+	    (error = git_reader_for_index(&post_reader, repo, postimage)) < 0)
+		goto done;
+
+	if ((error = git_repository_index(&index, repo)) < 0 ||
+	    (error = git_indexwriter_init(&indexwriter, index)) < 0)
+		goto done;
+
+	if ((error = apply_deltas(repo, pre_reader, preimage, post_reader, postimage, diff, &opts)) < 0)
+		goto done;
+
+	switch (location) {
+	case GIT_APPLY_LOCATION_BOTH:
+		error = git_apply__to_workdir(repo, diff, preimage, postimage, location, &opts);
+		break;
+	case GIT_APPLY_LOCATION_INDEX:
+		error = git_apply__to_index(repo, diff, preimage, postimage, &opts);
+		break;
+	case GIT_APPLY_LOCATION_WORKDIR:
+		error = git_apply__to_workdir(repo, diff, preimage, postimage, location, &opts);
+		break;
+	default:
+		assert(false);
+	}
+
+	if (error < 0)
+		goto done;
+
+	error = git_indexwriter_commit(&indexwriter);
+
+done:
+	git_indexwriter_cleanup(&indexwriter);
+	git_index_free(postimage);
+	git_index_free(preimage);
+	git_index_free(index);
+	git_reader_free(pre_reader);
+	git_reader_free(post_reader);
+
+	return error;
+}
diff --git a/src/apply.h b/src/apply.h
index b29460c0b..11ec75637 100644
--- a/src/apply.h
+++ b/src/apply.h
@@ -10,6 +10,7 @@
 #include "common.h"
 
 #include "git2/patch.h"
+#include "git2/apply.h"
 #include "buffer.h"
 
 extern int git_apply__patch(
@@ -18,6 +19,7 @@ extern int git_apply__patch(
 	unsigned int *mode,
 	const char *source,
 	size_t source_len,
-	git_patch *patch);
+	git_patch *patch,
+	const git_apply_options *opts);
 
 #endif
diff --git a/src/iterator.c b/src/iterator.c
index 6e7300af3..40f675957 100644
--- a/src/iterator.c
+++ b/src/iterator.c
@@ -1015,6 +1015,7 @@ typedef struct {
 	struct stat st;
 	size_t path_len;
 	iterator_pathlist_search_t match;
+	git_oid id;
 	char path[GIT_FLEX_ARRAY];
 } filesystem_iterator_entry;
 
@@ -1265,7 +1266,32 @@ GIT_INLINE(bool) filesystem_iterator_is_dot_git(
 	return (len == 4 || path[len - 5] == '/');
 }
 
-static filesystem_iterator_entry *filesystem_iterator_entry_init(
+static int filesystem_iterator_entry_hash(
+	filesystem_iterator *iter,
+	filesystem_iterator_entry *entry)
+{
+	git_buf fullpath = GIT_BUF_INIT;
+	int error;
+
+	if (S_ISDIR(entry->st.st_mode)) {
+		memset(&entry->id, 0, GIT_OID_RAWSZ);
+		return 0;
+	}
+
+	if (iter->base.type == GIT_ITERATOR_TYPE_WORKDIR)
+		return git_repository_hashfile(&entry->id,
+			iter->base.repo, entry->path, GIT_OBJ_BLOB, NULL);
+
+	if (!(error = git_buf_joinpath(&fullpath, iter->root, entry->path)))
+		error = git_odb_hashfile(&entry->id, fullpath.ptr, GIT_OBJ_BLOB);
+
+	git_buf_dispose(&fullpath);
+	return error;
+}
+
+static int filesystem_iterator_entry_init(
+	filesystem_iterator_entry **out,
+	filesystem_iterator *iter,
 	filesystem_iterator_frame *frame,
 	const char *path,
 	size_t path_len,
@@ -1274,15 +1300,19 @@ static filesystem_iterator_entry *filesystem_iterator_entry_init(
 {
 	filesystem_iterator_entry *entry;
 	size_t entry_size;
+	int error = 0;
+
+	*out = NULL;
 
 	/* Make sure to append two bytes, one for the path's null
 	 * termination, one for a possible trailing '/' for folders.
 	 */
-	if (GIT_ADD_SIZET_OVERFLOW(&entry_size,
-			sizeof(filesystem_iterator_entry), path_len) ||
-		GIT_ADD_SIZET_OVERFLOW(&entry_size, entry_size, 2) ||
-		(entry = git_pool_malloc(&frame->entry_pool, entry_size)) == NULL)
-		return NULL;
+	GITERR_CHECK_ALLOC_ADD(&entry_size,
+		sizeof(filesystem_iterator_entry), path_len);
+	GITERR_CHECK_ALLOC_ADD(&entry_size, entry_size, 2);
+
+	entry = git_pool_malloc(&frame->entry_pool, entry_size);
+	GITERR_CHECK_ALLOC(entry);
 
 	entry->path_len = path_len;
 	entry->match = pathlist_match;
@@ -1295,7 +1325,13 @@ static filesystem_iterator_entry *filesystem_iterator_entry_init(
 
 	entry->path[entry->path_len] = '\0';
 
-	return entry;
+	if (iter->base.flags & GIT_ITERATOR_INCLUDE_HASH)
+		error = filesystem_iterator_entry_hash(iter, entry);
+
+	if (!error)
+		*out = entry;
+
+	return error;
 }
 
 static int filesystem_iterator_frame_push(
@@ -1418,9 +1454,9 @@ static int filesystem_iterator_frame_push(
 		else if (dir_expected)
 			continue;
 
-		entry = filesystem_iterator_entry_init(new_frame,
-			path, path_len, &statbuf, pathlist_match);
-		GITERR_CHECK_ALLOC(entry);
+		if ((error = filesystem_iterator_entry_init(&entry,
+			iter, new_frame, path, path_len, &statbuf, pathlist_match)) < 0)
+			goto done;
 
 		git_vector_insert(&new_frame->entries, entry);
 	}
@@ -1460,7 +1496,7 @@ static void filesystem_iterator_set_current(
 	iter->entry.ctime.seconds = entry->st.st_ctime;
 	iter->entry.mtime.seconds = entry->st.st_mtime;
 
-#if defined(GIT_USE_NSEC)	
+#if defined(GIT_USE_NSEC)
 	iter->entry.ctime.nanoseconds = entry->st.st_ctime_nsec;
 	iter->entry.mtime.nanoseconds = entry->st.st_mtime_nsec;
 #else
@@ -1475,6 +1511,9 @@ static void filesystem_iterator_set_current(
 	iter->entry.gid = entry->st.st_gid;
 	iter->entry.file_size = entry->st.st_size;
 
+	if (iter->base.flags & GIT_ITERATOR_INCLUDE_HASH)
+		git_oid_cpy(&iter->entry.id, &entry->id);
+
 	iter->entry.path = entry->path;
 
 	iter->current_is_ignored = GIT_IGNORE_UNCHECKED;
@@ -2259,6 +2298,35 @@ void git_iterator_free(git_iterator *iter)
 	git__free(iter);
 }
 
+int git_iterator_foreach(
+	git_iterator *iterator,
+	git_iterator_foreach_cb cb,
+	void *data)
+{
+	const git_index_entry *iterator_item;
+	int error = 0;
+
+	if ((error = git_iterator_current(&iterator_item, iterator)) < 0)
+		goto done;
+
+	if ((error = cb(iterator_item, data)) != 0)
+		goto done;
+
+	while (true) {
+		if ((error = git_iterator_advance(&iterator_item, iterator)) < 0)
+			goto done;
+
+		if ((error = cb(iterator_item, data)) != 0)
+			goto done;
+	}
+
+done:
+	if (error == GIT_ITEROVER)
+		error = 0;
+
+	return error;
+}
+
 int git_iterator_walk(
 	git_iterator **iterators,
 	size_t cnt,
diff --git a/src/iterator.h b/src/iterator.h
index a6497d87b..bbe357fbd 100644
--- a/src/iterator.h
+++ b/src/iterator.h
@@ -41,6 +41,8 @@ typedef enum {
 	GIT_ITERATOR_INCLUDE_CONFLICTS = (1u << 6),
 	/** descend into symlinked directories */
 	GIT_ITERATOR_DESCEND_SYMLINKS = (1u << 7),
+	/** hash files in workdir or filesystem iterators */
+	GIT_ITERATOR_INCLUDE_HASH = (1u << 8),
 } git_iterator_flag_t;
 
 typedef enum {
@@ -289,6 +291,19 @@ extern int git_iterator_current_workdir_path(
  */
 extern git_index *git_iterator_index(git_iterator *iter);
 
+typedef int (*git_iterator_foreach_cb)(
+	const git_index_entry *entry,
+	void *data);
+
+/**
+ * Walk the given iterator and invoke the callback for each path
+ * contained in the iterator.
+ */
+extern int git_iterator_foreach(
+	git_iterator *iterator,
+	git_iterator_foreach_cb cb,
+	void *data);
+
 typedef int (*git_iterator_walk_cb)(
 	const git_index_entry **entries,
 	void *data);
diff --git a/src/reader.c b/src/reader.c
new file mode 100644
index 000000000..9375ff3f5
--- /dev/null
+++ b/src/reader.c
@@ -0,0 +1,259 @@
+/*
+ * Copyright (C) the libgit2 contributors. All rights reserved.
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+
+#include "reader.h"
+
+#include "fileops.h"
+#include "blob.h"
+
+#include "git2/tree.h"
+#include "git2/blob.h"
+#include "git2/index.h"
+#include "git2/repository.h"
+
+/* tree reader */
+
+typedef struct {
+	git_reader reader;
+	git_tree *tree;
+} tree_reader;
+
+static int tree_reader_read(
+	git_buf *out,
+	git_oid *out_id,
+	git_filemode_t *out_filemode,
+	git_reader *_reader,
+	const char *filename)
+{
+	tree_reader *reader = (tree_reader *)_reader;
+	git_tree_entry *tree_entry = NULL;
+	git_blob *blob = NULL;
+	int error;
+
+	if ((error = git_tree_entry_bypath(&tree_entry, reader->tree, filename)) < 0 ||
+	    (error = git_blob_lookup(&blob, git_tree_owner(reader->tree), git_tree_entry_id(tree_entry))) < 0 ||
+	    (error = git_buf_set(out, git_blob_rawcontent(blob), git_blob_rawsize(blob))) < 0)
+		goto done;
+
+	if (out_id)
+		git_oid_cpy(out_id, git_tree_entry_id(tree_entry));
+
+	if (out_filemode)
+		*out_filemode = git_tree_entry_filemode(tree_entry);
+
+done:
+	git_blob_free(blob);
+	git_tree_entry_free(tree_entry);
+	return error;
+}
+
+int git_reader_for_tree(git_reader **out, git_tree *tree)
+{
+	tree_reader *reader;
+
+	assert(out && tree);
+
+	reader = git__calloc(1, sizeof(tree_reader));
+	GITERR_CHECK_ALLOC(reader);
+
+	reader->reader.read = tree_reader_read;
+	reader->tree = tree;
+
+	*out = (git_reader *)reader;
+	return 0;
+}
+
+/* workdir reader */
+
+typedef struct {
+	git_reader reader;
+	git_repository *repo;
+	git_index *index;
+} workdir_reader;
+
+static int workdir_reader_read(
+	git_buf *out,
+	git_oid *out_id,
+	git_filemode_t *out_filemode,
+	git_reader *_reader,
+	const char *filename)
+{
+	workdir_reader *reader = (workdir_reader *)_reader;
+	git_buf path = GIT_BUF_INIT;
+	struct stat st;
+	git_filemode_t filemode;
+	git_filter_list *filters = NULL;
+	const git_index_entry *idx_entry;
+	git_oid id;
+	int error;
+
+	if ((error = git_buf_joinpath(&path,
+		git_repository_workdir(reader->repo), filename)) < 0)
+		goto done;
+
+	if ((error = p_lstat(path.ptr, &st)) < 0) {
+		if (error == -1 && errno == ENOENT)
+			error = GIT_ENOTFOUND;
+
+		giterr_set(GITERR_OS, "could not stat '%s'", path.ptr);
+		goto done;
+	}
+
+	filemode = git_futils_canonical_mode(st.st_mode);
+
+	/*
+	 * Patch application - for example - uses the filtered version of
+	 * the working directory data to match git.  So we will run the
+	 * workdir -> ODB filter on the contents in this workdir reader.
+	 */
+	if ((error = git_filter_list_load(&filters, reader->repo, NULL, filename,
+		GIT_FILTER_TO_ODB, GIT_FILTER_DEFAULT)) < 0)
+		goto done;
+
+	if ((error = git_filter_list_apply_to_file(out,
+	    filters, reader->repo, path.ptr)) < 0)
+		goto done;
+
+	if (out_id || reader->index) {
+		if ((error = git_odb_hash(&id, out->ptr, out->size, GIT_OBJ_BLOB)) < 0)
+			goto done;
+	}
+
+	if (reader->index) {
+		if (!(idx_entry = git_index_get_bypath(reader->index, filename, 0)) ||
+		    filemode != idx_entry->mode ||
+		    !git_oid_equal(&id, &idx_entry->id)) {
+			error = GIT_READER_MISMATCH;
+			goto done;
+		}
+	}
+
+	if (out_id)
+		git_oid_cpy(out_id, &id);
+
+	if (out_filemode)
+		*out_filemode = filemode;
+
+done:
+	git_filter_list_free(filters);
+	git_buf_dispose(&path);
+	return error;
+}
+
+int git_reader_for_workdir(
+	git_reader **out,
+	git_repository *repo,
+	bool validate_index)
+{
+	workdir_reader *reader;
+	int error;
+
+	assert(out && repo);
+
+	reader = git__calloc(1, sizeof(workdir_reader));
+	GITERR_CHECK_ALLOC(reader);
+
+	reader->reader.read = workdir_reader_read;
+	reader->repo = repo;
+
+	if (validate_index &&
+	    (error = git_repository_index__weakptr(&reader->index, repo)) < 0) {
+		git__free(reader);
+		return error;
+	}
+
+	*out = (git_reader *)reader;
+	return 0;
+}
+
+/* index reader */
+
+typedef struct {
+	git_reader reader;
+	git_repository *repo;
+	git_index *index;
+} index_reader;
+
+static int index_reader_read(
+	git_buf *out,
+	git_oid *out_id,
+	git_filemode_t *out_filemode,
+	git_reader *_reader,
+	const char *filename)
+{
+	index_reader *reader = (index_reader *)_reader;
+	const git_index_entry *entry;
+	git_blob *blob;
+	int error;
+
+	if ((entry = git_index_get_bypath(reader->index, filename, 0)) == NULL)
+		return GIT_ENOTFOUND;
+
+	if ((error = git_blob_lookup(&blob, reader->repo, &entry->id)) < 0)
+		goto done;
+
+	if (out_id)
+		git_oid_cpy(out_id, &entry->id);
+
+	if (out_filemode)
+		*out_filemode = entry->mode;
+
+	error = git_blob__getbuf(out, blob);
+
+done:
+	git_blob_free(blob);
+	return error;
+}
+
+int git_reader_for_index(
+	git_reader **out,
+	git_repository *repo,
+	git_index *index)
+{
+	index_reader *reader;
+	int error;
+
+	assert(out && repo);
+
+	reader = git__calloc(1, sizeof(index_reader));
+	GITERR_CHECK_ALLOC(reader);
+
+	reader->reader.read = index_reader_read;
+	reader->repo = repo;
+
+	if (index) {
+		reader->index = index;
+	} else if ((error = git_repository_index__weakptr(&reader->index, repo)) < 0) {
+		git__free(reader);
+		return error;
+	}
+
+	*out = (git_reader *)reader;
+	return 0;
+}
+
+/* generic */
+
+int git_reader_read(
+	git_buf *out,
+	git_oid *out_id,
+	git_filemode_t *out_filemode,
+	git_reader *reader,
+	const char *filename)
+{
+	assert(out && reader && filename);
+
+	return reader->read(out, out_id, out_filemode, reader, filename);
+}
+
+void git_reader_free(git_reader *reader)
+{
+	if (!reader)
+		return;
+
+	git__free(reader);
+}
diff --git a/src/reader.h b/src/reader.h
new file mode 100644
index 000000000..18a6a1103
--- /dev/null
+++ b/src/reader.h
@@ -0,0 +1,107 @@
+/*
+ * Copyright (C) the libgit2 contributors. All rights reserved.
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+#ifndef INCLUDE_reader_h__
+#define INCLUDE_reader_h__
+
+#include "common.h"
+
+/* Returned when the workdir does not match the index */
+#define GIT_READER_MISMATCH	1
+
+typedef struct git_reader git_reader;
+
+/*
+ * The `git_reader` structure is a generic interface for reading the
+ * contents of a file by its name, and implementations are provided
+ * for reading out of a tree, the index, and the working directory.
+ *
+ * Note that the reader implementation is meant to have a short
+ * lifecycle and does not increase the refcount of the object that
+ * it's reading.  Callers should ensure that they do not use a
+ * reader after disposing the underlying object that it reads.
+ */
+struct git_reader {
+	int (*read)(git_buf *out, git_oid *out_oid, git_filemode_t *mode, git_reader *reader, const char *filename);
+};
+
+/**
+ * Create a `git_reader` that will allow random access to the given
+ * tree.  Paths requested via `git_reader_read` will be rooted at this
+ * tree, callers are not expected to recurse through tree lookups.  Thus,
+ * you can request to read `/src/foo.c` and the tree provided to this
+ * function will be searched to find another tree named `src`, which
+ * will then be opened to find `foo.c`.
+ *
+ * @param out The reader for the given tree
+ * @param tree The tree object to read
+ * @return 0 on success, or an error code < 0
+ */
+extern int git_reader_for_tree(
+	git_reader **out,
+	git_tree *tree);
+
+/**
+ * Create a `git_reader` that will allow random access to the given
+ * index, or the repository's index.
+ *
+ * @param out The reader for the given index
+ * @param repo The repository containing the index
+ * @param index The index to read, or NULL to use the repository's index
+ * @return 0 on success, or an error code < 0
+ */
+extern int git_reader_for_index(
+	git_reader **out,
+	git_repository *repo,
+	git_index *index);
+
+/**
+ * Create a `git_reader` that will allow random access to the given
+ * repository's working directory.  Note that the contents are read
+ * in repository format, meaning any workdir -> odb filters are
+ * applied.
+ *
+ * If `validate_index` is set to true, reads of files will hash the
+ * on-disk contents and ensure that the resulting object ID matches
+ * the repository's index.  This ensures that the working directory
+ * is unmodified from the index contents.
+ *
+ * @param out The reader for the given working directory
+ * @param repo The repository containing the working directory
+ * @param validate_index If true, the working directory contents will
+ *        be compared to the index contents during read to ensure that
+ *        the working directory is unmodified.
+ * @return 0 on success, or an error code < 0
+ */
+extern int git_reader_for_workdir(
+	git_reader **out,
+	git_repository *repo,
+	bool validate_index);
+
+/**
+ * Read the given filename from the reader and populate the given buffer
+ * with the contents and the given oid with the object ID.
+ *
+ * @param out The buffer to populate with the file contents
+ * @param out_id The oid to populate with the object ID
+ * @param reader The reader to read
+ * @param filename The filename to read from the reader
+ */
+extern int git_reader_read(
+	git_buf *out,
+	git_oid *out_id,
+	git_filemode_t *out_filemode,
+	git_reader *reader,
+	const char *filename);
+
+/**
+ * Free the given reader and any associated objects.
+ *
+ * @param reader The reader to free
+ */
+extern void git_reader_free(git_reader *reader);
+
+#endif