diff options
| author | Edward Thomson <ethomson@edwardthomson.com> | 2023-02-14 17:01:23 +0000 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2023-02-14 17:01:23 +0000 |
| commit | 80742e159c3221fd853d9a0fe735b382a7e3419d (patch) | |
| tree | 60514c97fde6a8630a174d3b61781347913b4f70 /src/libgit2 | |
| parent | f7963f28cad7b2416a3cb0e55bc5adf85c82607b (diff) | |
| parent | e3cd859150f720914dfbbdd2d80d12b1482a397a (diff) | |
| download | libgit2-80742e159c3221fd853d9a0fe735b382a7e3419d.tar.gz | |
Merge pull request #6456 from libgit2/ethomson/sha256_experimental
SHA256: more SHA256 support
Diffstat (limited to 'src/libgit2')
38 files changed, 1122 insertions, 405 deletions
diff --git a/src/libgit2/blob.c b/src/libgit2/blob.c index b1680d3a8..5cfd7474b 100644 --- a/src/libgit2/blob.c +++ b/src/libgit2/blob.c @@ -52,11 +52,12 @@ void git_blob__free(void *_blob) git__free(blob); } -int git_blob__parse_raw(void *_blob, const char *data, size_t size) +int git_blob__parse_raw(void *_blob, const char *data, size_t size, git_oid_t oid_type) { git_blob *blob = (git_blob *) _blob; GIT_ASSERT_ARG(blob); + GIT_UNUSED(oid_type); blob->raw = 1; blob->data.raw.data = data; @@ -64,11 +65,12 @@ int git_blob__parse_raw(void *_blob, const char *data, size_t size) return 0; } -int git_blob__parse(void *_blob, git_odb_object *odb_obj) +int git_blob__parse(void *_blob, git_odb_object *odb_obj, git_oid_t oid_type) { git_blob *blob = (git_blob *) _blob; GIT_ASSERT_ARG(blob); + GIT_UNUSED(oid_type); git_cached_obj_incref((git_cached_obj *)odb_obj); blob->raw = 0; diff --git a/src/libgit2/blob.h b/src/libgit2/blob.h index 9a5dda225..d6c9dd99b 100644 --- a/src/libgit2/blob.h +++ b/src/libgit2/blob.h @@ -36,8 +36,8 @@ struct git_blob { } while(0) void git_blob__free(void *blob); -int git_blob__parse(void *blob, git_odb_object *obj); -int git_blob__parse_raw(void *blob, const char *data, size_t size); +int git_blob__parse(void *blob, git_odb_object *obj, git_oid_t oid_type); +int git_blob__parse_raw(void *blob, const char *data, size_t size, git_oid_t oid_type); int git_blob__getbuf(git_str *buffer, git_blob *blob); extern int git_blob__create_from_paths( diff --git a/src/libgit2/clone.c b/src/libgit2/clone.c index 0d393eb85..e8885f22a 100644 --- a/src/libgit2/clone.c +++ b/src/libgit2/clone.c @@ -393,12 +393,19 @@ static int checkout_branch(git_repository *repo, git_remote *remote, const git_c return error; } -static int clone_into(git_repository *repo, git_remote *_remote, const git_fetch_options *opts, const git_checkout_options *co_opts, const char *branch) +static int clone_into( + git_repository *repo, + git_remote *_remote, + const git_fetch_options *opts, + const git_checkout_options *co_opts, + const char *branch) { int error; git_str reflog_message = GIT_STR_INIT; + git_remote_connect_options connect_opts = GIT_REMOTE_CONNECT_OPTIONS_INIT; git_fetch_options fetch_opts; git_remote *remote; + git_oid_t oid_type; GIT_ASSERT_ARG(repo); GIT_ASSERT_ARG(_remote); @@ -414,8 +421,25 @@ static int clone_into(git_repository *repo, git_remote *_remote, const git_fetch memcpy(&fetch_opts, opts, sizeof(git_fetch_options)); fetch_opts.update_fetchhead = 0; fetch_opts.download_tags = GIT_REMOTE_DOWNLOAD_TAGS_ALL; + + if ((error = git_remote_connect_options__from_fetch_opts(&connect_opts, remote, &fetch_opts)) < 0) + return error; + git_str_printf(&reflog_message, "clone: from %s", git_remote_url(remote)); + /* + * Connect to the server so that we can identify the remote + * object format. + */ + + if ((error = git_remote_connect_ext(remote, GIT_DIRECTION_FETCH, + &connect_opts)) < 0) + goto cleanup; + + if ((error = git_remote_oid_type(&oid_type, remote)) < 0 || + (error = git_repository__set_objectformat(repo, oid_type)) < 0) + goto cleanup; + if ((error = git_remote_fetch(remote, NULL, &fetch_opts, git_str_cstr(&reflog_message))) != 0) goto cleanup; @@ -423,6 +447,7 @@ static int clone_into(git_repository *repo, git_remote *_remote, const git_fetch cleanup: git_remote_free(remote); + git_remote_connect_options_dispose(&connect_opts); git_str_dispose(&reflog_message); return error; diff --git a/src/libgit2/commit.c b/src/libgit2/commit.c index 114ae6fc9..d85fefb3d 100644 --- a/src/libgit2/commit.c +++ b/src/libgit2/commit.c @@ -390,7 +390,11 @@ int git_commit_amend( return error; } -static int commit_parse(git_commit *commit, const char *data, size_t size, unsigned int flags) +static int commit_parse( + git_commit *commit, + const char *data, + size_t size, + git_commit__parse_options *opts) { const char *buffer_start = data, *buffer; const char *buffer_end = buffer_start + size; @@ -401,6 +405,7 @@ static int commit_parse(git_commit *commit, const char *data, size_t size, unsig GIT_ASSERT_ARG(commit); GIT_ASSERT_ARG(data); + GIT_ASSERT_ARG(opts); buffer = buffer_start; @@ -409,13 +414,14 @@ static int commit_parse(git_commit *commit, const char *data, size_t size, unsig GIT_ERROR_CHECK_ARRAY(commit->parent_ids); /* The tree is always the first field */ - if (!(flags & GIT_COMMIT_PARSE_QUICK)) { + if (!(opts->flags & GIT_COMMIT_PARSE_QUICK)) { if (git_object__parse_oid_header(&commit->tree_id, &buffer, buffer_end, "tree ", - GIT_OID_SHA1) < 0) + opts->oid_type) < 0) goto bad_buffer; } else { - size_t tree_len = strlen("tree ") + GIT_OID_SHA1_HEXSIZE + 1; + size_t tree_len = strlen("tree ") + git_oid_hexsize(opts->oid_type) + 1; + if (buffer + tree_len > buffer_end) goto bad_buffer; buffer += tree_len; @@ -427,14 +433,14 @@ static int commit_parse(git_commit *commit, const char *data, size_t size, unsig while (git_object__parse_oid_header(&parent_id, &buffer, buffer_end, "parent ", - GIT_OID_SHA1) == 0) { + opts->oid_type) == 0) { git_oid *new_id = git_array_alloc(commit->parent_ids); GIT_ERROR_CHECK_ALLOC(new_id); git_oid_cpy(new_id, &parent_id); } - if (!(flags & GIT_COMMIT_PARSE_QUICK)) { + if (!opts || !(opts->flags & GIT_COMMIT_PARSE_QUICK)) { commit->author = git__malloc(sizeof(git_signature)); GIT_ERROR_CHECK_ALLOC(commit->author); @@ -458,7 +464,7 @@ static int commit_parse(git_commit *commit, const char *data, size_t size, unsig if ((error = git_signature__parse(commit->committer, &buffer, buffer_end, "committer ", '\n')) < 0) return error; - if (flags & GIT_COMMIT_PARSE_QUICK) + if (opts && opts->flags & GIT_COMMIT_PARSE_QUICK) return 0; /* Parse add'l header entries */ @@ -503,19 +509,39 @@ bad_buffer: return GIT_EINVALID; } -int git_commit__parse_raw(void *commit, const char *data, size_t size) +int git_commit__parse( + void *commit, + git_odb_object *odb_obj, + git_oid_t oid_type) { - return commit_parse(commit, data, size, 0); + git_commit__parse_options parse_options = {0}; + parse_options.oid_type = oid_type; + + return git_commit__parse_ext(commit, odb_obj, &parse_options); } -int git_commit__parse_ext(git_commit *commit, git_odb_object *odb_obj, unsigned int flags) +int git_commit__parse_raw( + void *commit, + const char *data, + size_t size, + git_oid_t oid_type) { - return commit_parse(commit, git_odb_object_data(odb_obj), git_odb_object_size(odb_obj), flags); + git_commit__parse_options parse_options = {0}; + parse_options.oid_type = oid_type; + + return commit_parse(commit, data, size, &parse_options); } -int git_commit__parse(void *_commit, git_odb_object *odb_obj) +int git_commit__parse_ext( + git_commit *commit, + git_odb_object *odb_obj, + git_commit__parse_options *parse_opts) { - return git_commit__parse_ext(_commit, odb_obj, 0); + return commit_parse( + commit, + git_odb_object_data(odb_obj), + git_odb_object_size(odb_obj), + parse_opts); } #define GIT_COMMIT_GETTER(_rvalue, _name, _return, _invalid) \ @@ -985,11 +1011,14 @@ int git_commit_create_with_signature( git_str commit = GIT_STR_INIT; git_commit *parsed; git_array_oid_t parents = GIT_ARRAY_INIT; + git_commit__parse_options parse_opts = {0}; + + parse_opts.oid_type = repo->oid_type; /* The first step is to verify that all the tree and parents exist */ parsed = git__calloc(1, sizeof(git_commit)); GIT_ERROR_CHECK_ALLOC(parsed); - if (commit_parse(parsed, commit_content, strlen(commit_content), 0) < 0) { + if (commit_parse(parsed, commit_content, strlen(commit_content), &parse_opts) < 0) { error = -1; goto cleanup; } diff --git a/src/libgit2/commit.h b/src/libgit2/commit.h index 7a2454e61..c25fee327 100644 --- a/src/libgit2/commit.h +++ b/src/libgit2/commit.h @@ -33,6 +33,16 @@ struct git_commit { char *body; }; +typedef struct { + git_oid_t oid_type; + unsigned int flags; +} git_commit__parse_options; + +typedef enum { + /** Only parse parents and committer info */ + GIT_COMMIT_PARSE_QUICK = (1 << 0) +} git_commit__parse_flags; + int git_commit__header_field( git_str *out, const git_commit *commit, @@ -56,14 +66,22 @@ int git_commit__create_buffer( size_t parent_count, const git_commit *parents[]); -void git_commit__free(void *commit); -int git_commit__parse(void *commit, git_odb_object *obj); -int git_commit__parse_raw(void *commit, const char *data, size_t size); +int git_commit__parse( + void *commit, + git_odb_object *obj, + git_oid_t oid_type); -typedef enum { - GIT_COMMIT_PARSE_QUICK = (1 << 0) /**< Only parse parents and committer info */ -} git_commit__parse_flags; +int git_commit__parse_raw( + void *commit, + const char *data, + size_t size, + git_oid_t oid_type); -int git_commit__parse_ext(git_commit *commit, git_odb_object *odb_obj, unsigned int flags); +int git_commit__parse_ext( + git_commit *commit, + git_odb_object *odb_obj, + git_commit__parse_options *parse_opts); + +void git_commit__free(void *commit); #endif diff --git a/src/libgit2/commit_graph.c b/src/libgit2/commit_graph.c index 9554fe855..bf557f7ad 100644 --- a/src/libgit2/commit_graph.c +++ b/src/libgit2/commit_graph.c @@ -538,7 +538,7 @@ int git_commit_graph_entry_find( hi = ntohl(file->oid_fanout[(int)short_oid->id[0]]); lo = ((short_oid->id[0] == 0x0) ? 0 : ntohl(file->oid_fanout[(int)short_oid->id[0] - 1])); - pos = git_pack__lookup_sha1(file->oid_lookup, GIT_OID_SHA1_SIZE, lo, hi, short_oid->id); + pos = git_pack__lookup_id(file->oid_lookup, GIT_OID_SHA1_SIZE, lo, hi, short_oid->id, GIT_OID_SHA1); if (pos >= 0) { /* An object matching exactly the oid was found */ @@ -726,7 +726,8 @@ int git_commit_graph_writer_add_index_file( if (error < 0) goto cleanup; - error = git_mwindow_get_pack(&p, idx_path); + /* TODO: SHA256 */ + error = git_mwindow_get_pack(&p, idx_path, 0); if (error < 0) goto cleanup; diff --git a/src/libgit2/commit_list.c b/src/libgit2/commit_list.c index 511d7291f..12b329b25 100644 --- a/src/libgit2/commit_list.c +++ b/src/libgit2/commit_list.c @@ -124,13 +124,17 @@ static int commit_quick_parse( { git_oid *parent_oid; git_commit *commit; + git_commit__parse_options parse_opts = { + GIT_OID_SHA1, + GIT_COMMIT_PARSE_QUICK + }; size_t i; commit = git__calloc(1, sizeof(*commit)); GIT_ERROR_CHECK_ALLOC(commit); commit->object.repo = walk->repo; - if (git_commit__parse_ext(commit, obj, GIT_COMMIT_PARSE_QUICK) < 0) { + if (git_commit__parse_ext(commit, obj, &parse_opts) < 0) { git__free(commit); return -1; } diff --git a/src/libgit2/fetch.c b/src/libgit2/fetch.c index 5c2fee617..003b5198a 100644 --- a/src/libgit2/fetch.c +++ b/src/libgit2/fetch.c @@ -95,7 +95,6 @@ static int filter_wants(git_remote *remote, const git_fetch_options *opts) git_remote_head **heads; git_refspec tagspec, head, *spec; int error = 0; - git_odb *odb; size_t i, heads_len; unsigned int remote_caps; unsigned int oid_mask = GIT_REMOTE_CAPABILITY_TIP_OID | @@ -126,9 +125,6 @@ static int filter_wants(git_remote *remote, const git_fetch_options *opts) goto cleanup; } - if ((error = git_repository_odb__weakptr(&odb, remote->repo)) < 0) - goto cleanup; - if ((error = git_remote_ls((const git_remote_head ***)&heads, &heads_len, remote)) < 0 || (error = git_remote_capabilities(&remote_caps, remote)) < 0) goto cleanup; diff --git a/src/libgit2/indexer.c b/src/libgit2/indexer.c index 62bb70393..fa55fb5ea 100644 --- a/src/libgit2/indexer.c +++ b/src/libgit2/indexer.c @@ -42,6 +42,7 @@ struct git_indexer { have_delta :1, do_fsync :1, do_verify :1; + git_oid_t oid_type; struct git_pack_header hdr; struct git_pack_file *pack; unsigned int mode; @@ -55,8 +56,8 @@ struct git_indexer { git_vector deltas; unsigned int fanout[256]; git_hash_ctx hash_ctx; - unsigned char checksum[GIT_HASH_SHA1_SIZE]; - char name[(GIT_HASH_SHA1_SIZE * 2) + 1]; + unsigned char checksum[GIT_HASH_MAX_SIZE]; + char name[(GIT_HASH_MAX_SIZE * 2) + 1]; git_indexer_progress_cb progress_cb; void *progress_payload; char objbuf[8*1024]; @@ -68,7 +69,7 @@ struct git_indexer { git_odb *odb; /* Fields for calculating the packfile trailer (hash of everything before it) */ - char inbuf[GIT_OID_SHA1_SIZE]; + char inbuf[GIT_HASH_MAX_SIZE]; size_t inbuf_len; git_hash_ctx trailer; }; @@ -136,17 +137,33 @@ int git_indexer_init_options(git_indexer_options *opts, unsigned int version) } #endif -int git_indexer_new( - git_indexer **out, - const char *prefix, - unsigned int mode, - git_odb *odb, - git_indexer_options *in_opts) +GIT_INLINE(git_hash_algorithm_t) indexer_hash_algorithm(git_indexer *idx) +{ + switch (idx->oid_type) { + case GIT_OID_SHA1: + return GIT_HASH_ALGORITHM_SHA1; +#ifdef GIT_EXPERIMENTAL_SHA256 + case GIT_OID_SHA256: + return GIT_HASH_ALGORITHM_SHA256; +#endif + } + + return GIT_HASH_ALGORITHM_NONE; +} + +static int indexer_new( + git_indexer **out, + const char *prefix, + git_oid_t oid_type, + unsigned int mode, + git_odb *odb, + git_indexer_options *in_opts) { git_indexer_options opts = GIT_INDEXER_OPTIONS_INIT; git_indexer *idx; git_str path = GIT_STR_INIT, tmp_path = GIT_STR_INIT; static const char suff[] = "/pack"; + git_hash_algorithm_t checksum_type; int error, fd = -1; if (in_opts) @@ -154,14 +171,17 @@ int git_indexer_new( idx = git__calloc(1, sizeof(git_indexer)); GIT_ERROR_CHECK_ALLOC(idx); + idx->oid_type = oid_type; idx->odb = odb; idx->progress_cb = opts.progress_cb; idx->progress_payload = opts.progress_cb_payload; idx->mode = mode ? mode : GIT_PACK_FILE_MODE; git_str_init(&idx->entry_data, 0); - if ((error = git_hash_ctx_init(&idx->hash_ctx, GIT_HASH_ALGORITHM_SHA1)) < 0 || - (error = git_hash_ctx_init(&idx->trailer, GIT_HASH_ALGORITHM_SHA1)) < 0 || + checksum_type = indexer_hash_algorithm(idx); + + if ((error = git_hash_ctx_init(&idx->hash_ctx, checksum_type)) < 0 || + (error = git_hash_ctx_init(&idx->trailer, checksum_type)) < 0 || (error = git_oidmap_new(&idx->expected_oids)) < 0) goto cleanup; @@ -179,7 +199,7 @@ int git_indexer_new( if (fd < 0) goto cleanup; - error = git_packfile_alloc(&idx->pack, git_str_cstr(&tmp_path)); + error = git_packfile_alloc(&idx->pack, git_str_cstr(&tmp_path), oid_type); git_str_dispose(&tmp_path); if (error < 0) @@ -208,6 +228,33 @@ cleanup: return -1; } +#ifdef GIT_EXPERIMENTAL_SHA256 +int git_indexer_new( + git_indexer **out, + const char *prefix, + git_oid_t oid_type, + git_indexer_options *opts) +{ + return indexer_new( + out, + prefix, + oid_type, + opts ? opts->mode : 0, + opts ? opts->odb : NULL, + opts); +} +#else +int git_indexer_new( + git_indexer **out, + const char *prefix, + unsigned int mode, + git_odb *odb, + git_indexer_options *opts) +{ + return indexer_new(out, prefix, GIT_OID_SHA1, mode, odb, opts); +} +#endif + void git_indexer__set_fsync(git_indexer *idx, int do_fsync) { idx->do_fsync = !!do_fsync; @@ -272,7 +319,7 @@ static int advance_delta_offset(git_indexer *idx, git_object_t type) GIT_ASSERT_ARG(type == GIT_OBJECT_REF_DELTA || type == GIT_OBJECT_OFS_DELTA); if (type == GIT_OBJECT_REF_DELTA) { - idx->off += GIT_OID_SHA1_SIZE; + idx->off += git_oid_size(idx->oid_type); } else { off64_t base_off; int error = get_delta_base(&base_off, idx->pack, &w, &idx->off, type, idx->entry_start); @@ -356,7 +403,7 @@ static int check_object_connectivity(git_indexer *idx, const git_rawobj *obj) obj->type != GIT_OBJECT_TAG) return 0; - if (git_object__from_raw(&object, obj->data, obj->len, obj->type) < 0) { + if (git_object__from_raw(&object, obj->data, obj->len, obj->type, idx->oid_type) < 0) { /* * parse_raw returns EINVALID on invalid data; downgrade * that to a normal -1 error code. @@ -446,7 +493,7 @@ static int store_object(git_indexer *idx) } #ifdef GIT_EXPERIMENTAL_SHA256 - oid.type = GIT_OID_SHA1; + oid.type = idx->oid_type; #endif entry_size = idx->off - entry_start; @@ -468,16 +515,16 @@ static int store_object(git_indexer *idx) goto on_error; } - git_oid_cpy(&pentry->sha1, &oid); + git_oid_cpy(&pentry->id, &oid); pentry->offset = entry_start; - if (git_oidmap_exists(idx->pack->idx_cache, &pentry->sha1)) { - git_error_set(GIT_ERROR_INDEXER, "duplicate object %s found in pack", git_oid_tostr_s(&pentry->sha1)); + if (git_oidmap_exists(idx->pack->idx_cache, &pentry->id)) { + git_error_set(GIT_ERROR_INDEXER, "duplicate object %s found in pack", git_oid_tostr_s(&pentry->id)); git__free(pentry); goto on_error; } - if ((error = git_oidmap_set(idx->pack->idx_cache, &pentry->sha1, pentry)) < 0) { + if ((error = git_oidmap_set(idx->pack->idx_cache, &pentry->id, pentry)) < 0) { git__free(pentry); git_error_set_oom(); goto on_error; @@ -522,8 +569,8 @@ static int save_entry(git_indexer *idx, struct entry *entry, struct git_pack_ent pentry->offset = entry_start; - if (git_oidmap_exists(idx->pack->idx_cache, &pentry->sha1) || - git_oidmap_set(idx->pack->idx_cache, &pentry->sha1, pentry) < 0) { + if (git_oidmap_exists(idx->pack->idx_cache, &pentry->id) || + git_oidmap_set(idx->pack->idx_cache, &pentry->id, pentry) < 0) { git_error_set(GIT_ERROR_INDEXER, "cannot insert object into pack"); return -1; } @@ -549,7 +596,7 @@ static int hash_and_save(git_indexer *idx, git_rawobj *obj, off64_t entry_start) entry = git__calloc(1, sizeof(*entry)); GIT_ERROR_CHECK_ALLOC(entry); - if (git_odb__hashobj(&oid, obj, GIT_OID_SHA1) < 0) { + if (git_odb__hashobj(&oid, obj, idx->oid_type) < 0) { git_error_set(GIT_ERROR_INDEXER, "failed to hash object"); goto on_error; } @@ -557,7 +604,7 @@ static int hash_and_save(git_indexer *idx, git_rawobj *obj, off64_t entry_start) pentry = git__calloc(1, sizeof(struct git_pack_entry)); GIT_ERROR_CHECK_ALLOC(pentry); - git_oid_cpy(&pentry->sha1, &oid); + git_oid_cpy(&pentry->id, &oid); git_oid_cpy(&entry->oid, &oid); entry->crc = crc32(0L, Z_NULL, 0); @@ -583,34 +630,38 @@ static int do_progress_callback(git_indexer *idx, git_indexer_progress *stats) return 0; } -/* Hash everything but the last 20B of input */ +/* Hash everything but the checksum trailer */ static void hash_partially(git_indexer *idx, const uint8_t *data, size_t size) { size_t to_expell, to_keep; + size_t oid_size = git_oid_size(idx->oid_type); if (size == 0) return; - /* Easy case, dump the buffer and the data minus the last 20 bytes */ - if (size >= GIT_OID_SHA1_SIZE) { + /* + * Easy case, dump the buffer and the data minus the trailing + * checksum (SHA1 or SHA256). + */ + if (size >= oid_size) { git_hash_update(&idx->trailer, idx->inbuf, idx->inbuf_len); - git_hash_update(&idx->trailer, data, size - GIT_OID_SHA1_SIZE); + git_hash_update(&idx->trailer, data, size - oid_size); - data += size - GIT_OID_SHA1_SIZE; - memcpy(idx->inbuf, data, GIT_OID_SHA1_SIZE); - idx->inbuf_len = GIT_OID_SHA1_SIZE; + data += size - oid_size; + memcpy(idx->inbuf, data, oid_size); + idx->inbuf_len = oid_size; return; } /* We can just append */ - if (idx->inbuf_len + size <= GIT_OID_SHA1_SIZE) { + if (idx->inbuf_len + size <= oid_size) { memcpy(idx->inbuf + idx->inbuf_len, data, size); idx->inbuf_len += size; return; } /* We need to partially drain the buffer and then append */ - to_keep = GIT_OID_SHA1_SIZE - size; + to_keep = oid_size - size; to_expell = idx->inbuf_len - to_keep; git_hash_update(&idx->trailer, idx->inbuf, to_expell); @@ -729,12 +780,14 @@ static int read_stream_object(git_indexer *idx, git_indexer_progress *stats) { git_packfile_stream *stream = &idx->stream; off64_t entry_start = idx->off; - size_t entry_size; + size_t oid_size, entry_size; git_object_t type; git_mwindow *w = NULL; int error; - if (idx->pack->mwf.size <= idx->off + 20) + oid_size = git_oid_size(idx->oid_type); + + if (idx->pack->mwf.size <= idx->off + (long long)oid_size) return GIT_EBUFS; if (!idx->have_stream) { @@ -905,7 +958,7 @@ static int index_path(git_str *path, git_indexer *idx, const char *suffix) slash--; if (git_str_grow(path, slash + 1 + strlen(prefix) + - GIT_OID_SHA1_HEXSIZE + strlen(suffix) + 1) < 0) + git_oid_hexsize(idx->oid_type) + strlen(suffix) + 1) < 0) return -1; git_str_truncate(path, slash); @@ -922,7 +975,7 @@ static int index_path(git_str *path, git_indexer *idx, const char *suffix) */ static int seek_back_trailer(git_indexer *idx) { - idx->pack->mwf.size -= GIT_OID_SHA1_SIZE; + idx->pack->mwf.size -= git_oid_size(idx->oid_type); return git_mwindow_free_all(&idx->pack->mwf); } @@ -931,15 +984,17 @@ static int inject_object(git_indexer *idx, git_oid *id) git_odb_object *obj = NULL; struct entry *entry = NULL; struct git_pack_entry *pentry = NULL; - unsigned char empty_checksum[GIT_HASH_SHA1_SIZE] = {0}; + unsigned char empty_checksum[GIT_HASH_MAX_SIZE] = {0}; unsigned char hdr[64]; git_str buf = GIT_STR_INIT; off64_t entry_start; const void *data; size_t len, hdr_len; - size_t checksum_size = GIT_HASH_SHA1_SIZE; + size_t checksum_size; int error; + checksum_size = git_hash_size(indexer_hash_algorithm(idx)); + if ((error = seek_back_trailer(idx)) < 0) goto cleanup; @@ -982,12 +1037,12 @@ static int inject_object(git_indexer *idx, git_oid *id) if ((error = append_to_pack(idx, empty_checksum, checksum_size)) < 0) goto cleanup; - idx->pack->mwf.size += GIT_OID_SHA1_SIZE; + idx->pack->mwf.size += git_oid_size(idx->oid_type); pentry = git__calloc(1, sizeof(struct git_pack_entry)); GIT_ERROR_CHECK_ALLOC(pentry); - git_oid_cpy(&pentry->sha1, id); + git_oid_cpy(&pentry->id, id); git_oid_cpy(&entry->oid, id); idx->off = entry_start + hdr_len + len; @@ -1045,13 +1100,13 @@ static int fix_thin_pack(git_indexer *idx, git_indexer_progress *stats) } /* curpos now points to the base information, which is an OID */ - base_info = git_mwindow_open(&idx->pack->mwf, &w, curpos, GIT_OID_SHA1_SIZE, &left); + base_info = git_mwindow_open(&idx->pack->mwf, &w, curpos, git_oid_size(idx->oid_type), &left); if (base_info == NULL) { git_error_set(GIT_ERROR_INDEXER, "failed to map delta information"); return -1; } - git_oid__fromraw(&base, base_info, GIT_OID_SHA1); + git_oid__fromraw(&base, base_info, idx->oid_type); git_mwindow_close(&w); if (has_entry(idx, &base)) @@ -1173,10 +1228,10 @@ int git_indexer_commit(git_indexer *idx, git_indexer_progress *stats) struct git_pack_idx_header hdr; git_str filename = GIT_STR_INIT; struct entry *entry; - unsigned char checksum[GIT_HASH_SHA1_SIZE]; + unsigned char checksum[GIT_HASH_MAX_SIZE]; git_filebuf index_file = {0}; void *packfile_trailer; - size_t checksum_size = GIT_HASH_SHA1_SIZE; + size_t checksum_size; bool mismatch; if (!idx->parsed_header) { @@ -1184,6 +1239,9 @@ int git_indexer_commit(git_indexer *idx, git_indexer_progress *stats) return -1; } + checksum_size = git_hash_size(indexer_hash_algorithm(idx)); + GIT_ASSERT(checksum_size); + /* Test for this before resolve_deltas(), as it plays with idx->off */ if (idx->off + (ssize_t)checksum_size < idx->pack->mwf.size) { git_error_set(GIT_ERROR_INDEXER, "unexpected data at the end of the pack"); @@ -1274,7 +1332,7 @@ int git_indexer_commit(git_indexer *idx, git_indexer_progress *stats) /* Write out the object names (SHA-1 hashes) */ git_vector_foreach(&idx->objects, i, entry) { - git_filebuf_write(&index_file, &entry->oid.id, GIT_OID_SHA1_SIZE); + git_filebuf_write(&index_file, &entry->oid.id, git_oid_size(idx->oid_type)); } /* Write out the CRC32 values */ diff --git a/src/libgit2/midx.c b/src/libgit2/midx.c index 51b2d6cc7..b09055237 100644 --- a/src/libgit2/midx.c +++ b/src/libgit2/midx.c @@ -392,7 +392,7 @@ int git_midx_entry_find( hi = ntohl(idx->oid_fanout[(int)short_oid->id[0]]); lo = ((short_oid->id[0] == 0x0) ? 0 : ntohl(idx->oid_fanout[(int)short_oid->id[0] - 1])); - pos = git_pack__lookup_sha1(idx->oid_lookup, GIT_OID_SHA1_SIZE, lo, hi, short_oid->id); + pos = git_pack__lookup_id(idx->oid_lookup, GIT_OID_SHA1_SIZE, lo, hi, short_oid->id, GIT_OID_SHA1); if (pos >= 0) { /* An object matching exactly the oid was found */ @@ -549,7 +549,8 @@ int git_midx_writer_add( if (error < 0) return error; - error = git_mwindow_get_pack(&p, git_str_cstr(&idx_path_buf)); + /* TODO: SHA256 */ + error = git_mwindow_get_pack(&p, git_str_cstr(&idx_path_buf), 0); git_str_dispose(&idx_path_buf); if (error < 0) return error; diff --git a/src/libgit2/mwindow.c b/src/libgit2/mwindow.c index ad649490a..b8295d9d1 100644 --- a/src/libgit2/mwindow.c +++ b/src/libgit2/mwindow.c @@ -61,7 +61,10 @@ int git_mwindow_global_init(void) return git_runtime_shutdown_register(git_mwindow_global_shutdown); } -int git_mwindow_get_pack(struct git_pack_file **out, const char *path) +int git_mwindow_get_pack( + struct git_pack_file **out, + const char *path, + git_oid_t oid_type) { struct git_pack_file *pack; char *packname; @@ -86,7 +89,7 @@ int git_mwindow_get_pack(struct git_pack_file **out, const char *path) } /* If we didn't find it, we need to create it */ - if ((error = git_packfile_alloc(&pack, path)) < 0) { + if ((error = git_packfile_alloc(&pack, path, oid_type)) < 0) { git_mutex_unlock(&git__mwindow_mutex); return error; } diff --git a/src/libgit2/mwindow.h b/src/libgit2/mwindow.h index e32ab99d4..8e6df2613 100644 --- a/src/libgit2/mwindow.h +++ b/src/libgit2/mwindow.h @@ -48,7 +48,10 @@ void git_mwindow_close(git_mwindow **w_cursor); extern int git_mwindow_global_init(void); struct git_pack_file; /* just declaration to avoid cyclical includes */ -int git_mwindow_get_pack(struct git_pack_file **out, const char *path); +int git_mwindow_get_pack( + struct git_pack_file **out, + const char *path, + git_oid_t oid_type); int git_mwindow_put_pack(struct git_pack_file *pack); #endif diff --git a/src/libgit2/object.c b/src/libgit2/object.c index d45465678..d87d40cf1 100644 --- a/src/libgit2/object.c +++ b/src/libgit2/object.c @@ -27,8 +27,8 @@ typedef struct { const char *str; /* type name string */ size_t size; /* size in bytes of the object structure */ - int (*parse)(void *self, git_odb_object *obj); - int (*parse_raw)(void *self, const char *data, size_t size); + int (*parse)(void *self, git_odb_object *obj, git_oid_t oid_type); + int (*parse_raw)(void *self, const char *data, size_t size, git_oid_t oid_type); void (*free)(void *self); } git_object_def; @@ -60,7 +60,8 @@ int git_object__from_raw( git_object **object_out, const char *data, size_t size, - git_object_t type) + git_object_t object_type, + git_oid_t oid_type) { git_object_def *def; git_object *object; @@ -71,12 +72,15 @@ int git_object__from_raw( *object_out = NULL; /* Validate type match */ - if (type != GIT_OBJECT_BLOB && type != GIT_OBJECT_TREE && type != GIT_OBJECT_COMMIT && type != GIT_OBJECT_TAG) { + if (object_type != GIT_OBJECT_BLOB && + object_type != GIT_OBJECT_TREE && + object_type != GIT_OBJECT_COMMIT && + object_type != GIT_OBJECT_TAG) { git_error_set(GIT_ERROR_INVALID, "the requested type is invalid"); return GIT_ENOTFOUND; } - if ((object_size = git_object__size(type)) == 0) { + if ((object_size = git_object__size(object_type)) == 0) { git_error_set(GIT_ERROR_INVALID, "the requested type is invalid"); return GIT_ENOTFOUND; } @@ -85,15 +89,15 @@ int git_object__from_raw( object = git__calloc(1, object_size); GIT_ERROR_CHECK_ALLOC(object); object->cached.flags = GIT_CACHE_STORE_PARSED; - object->cached.type = type; - if ((error = git_odb__hash(&object->cached.oid, data, size, type, GIT_OID_SHA1)) < 0) + object->cached.type = object_type; + if ((error = git_odb__hash(&object->cached.oid, data, size, object_type, oid_type)) < 0) return error; /* Parse raw object data */ - def = &git_objects_table[type]; + def = &git_objects_table[object_type]; GIT_ASSERT(def->free && def->parse_raw); - if ((error = def->parse_raw(object, data, size)) < 0) { + if ((error = def->parse_raw(object, data, size, oid_type)) < 0) { def->free(object); return error; } @@ -143,7 +147,7 @@ int git_object__from_odb_object( def = &git_objects_table[odb_obj->cached.type]; GIT_ASSERT(def->free && def->parse); - if ((error = def->parse(object, odb_obj)) < 0) { + if ((error = def->parse(object, odb_obj, repo->oid_type)) < 0) { /* * parse returns EINVALID on invalid data; downgrade * that to a normal -1 error code. @@ -357,12 +361,11 @@ static int dereference_object(git_object **dereferenced, git_object *obj) static int peel_error(int error, const git_oid *oid, git_object_t type) { const char *type_name; - char hex_oid[GIT_OID_SHA1_HEXSIZE + 1]; + char hex_oid[GIT_OID_MAX_HEXSIZE + 1]; type_name = git_object_type2string(type); - git_oid_fmt(hex_oid, oid); - hex_oid[GIT_OID_SHA1_HEXSIZE] = '\0'; + git_oid_nfmt(hex_oid, GIT_OID_MAX_HEXSIZE + 1, oid); git_error_set(GIT_ERROR_OBJECT, "the git_object of id '%s' can not be " "successfully peeled into a %s (git_object_t=%i).", hex_oid, type_name, type); @@ -576,21 +579,29 @@ int git_object_rawcontent_is_valid( int *valid, const char *buf, size_t len, - git_object_t type) + git_object_t object_type +#ifdef GIT_EXPERIMENTAL_SHA256 + , git_oid_t oid_type +#endif + ) { git_object *obj = NULL; int error; +#ifndef GIT_EXPERIMENTAL_SHA256 + git_oid_t oid_type = GIT_OID_SHA1; +#endif + GIT_ASSERT_ARG(valid); GIT_ASSERT_ARG(buf); /* Blobs are always valid; don't bother parsing. */ - if (type == GIT_OBJECT_BLOB) { + if (object_type == GIT_OBJECT_BLOB) { *valid = 1; return 0; } - error = git_object__from_raw(&obj, buf, len, type); + error = git_object__from_raw(&obj, buf, len, object_type, oid_type); git_object_free(obj); if (error == 0) { @@ -611,7 +622,7 @@ int git_object__parse_oid_header( const char *header, git_oid_t oid_type) { - const size_t sha_len = GIT_OID_SHA1_HEXSIZE; + const size_t sha_len = git_oid_hexsize(oid_type); const size_t header_len = strlen(header); const char *buffer = *buffer_out; diff --git a/src/libgit2/object.h b/src/libgit2/object.h index 980e8627e..a29fdfbf3 100644 --- a/src/libgit2/object.h +++ b/src/libgit2/object.h @@ -33,7 +33,8 @@ int git_object__from_raw( git_object **object_out, const char *data, size_t size, - git_object_t type); + git_object_t object_type, + git_oid_t oid_type); int git_object__from_odb_object( git_object **object_out, diff --git a/src/libgit2/odb.c b/src/libgit2/odb.c index aa2dd3cb2..edf4f001f 100644 --- a/src/libgit2/odb.c +++ b/src/libgit2/odb.c @@ -684,6 +684,7 @@ int git_odb__add_default_backends( ino_t inode; git_odb_backend *loose, *packed; git_odb_backend_loose_options loose_opts = GIT_ODB_BACKEND_LOOSE_OPTIONS_INIT; + git_odb_backend_pack_options pack_opts = GIT_ODB_BACKEND_PACK_OPTIONS_INIT; /* TODO: inodes are not really relevant on Win32, so we need to find * a cross-platform workaround for this */ @@ -722,6 +723,7 @@ int git_odb__add_default_backends( loose_opts.flags |= GIT_ODB_BACKEND_LOOSE_FSYNC; loose_opts.oid_type = db->options.oid_type; + pack_opts.oid_type = db->options.oid_type; /* add the loose object backend */ if (git_odb__backend_loose(&loose, objects_dir, &loose_opts) < 0 || @@ -729,8 +731,17 @@ int git_odb__add_default_backends( return -1; /* add the packed file backend */ - if (git_odb_backend_pack(&packed, objects_dir) < 0 || - add_backend_internal(db, packed, git_odb__packed_priority, as_alternates, inode) < 0) +#ifdef GIT_EXPERIMENTAL_SHA256 + if (git_odb_backend_pack(&packed, objects_dir, &pack_opts) < 0) + return -1; +#else + GIT_UNUSED(pack_opts); + + if (git_odb_backend_pack(&packed, objects_dir) < 0) + return -1; +#endif + + if (add_backend_internal(db, packed, git_odb__packed_priority, as_alternates, inode) < 0) return -1; if (git_mutex_lock(&db->lock) < 0) { diff --git a/src/libgit2/odb_pack.c b/src/libgit2/odb_pack.c index 49a655b44..1b1d122b0 100644 --- a/src/libgit2/odb_pack.c +++ b/src/libgit2/odb_pack.c @@ -26,6 +26,7 @@ struct pack_backend { git_odb_backend parent; + git_odb_backend_pack_options opts; git_midx_file *midx; git_vector midx_packs; git_vector packs; @@ -95,24 +96,24 @@ struct pack_writepack { * -------------------------------------------------- * * # pack_backend__exists / pack_backend__exists_prefix - * | Check if the given SHA1 oid (or a SHA1 oid prefix) exists in any of the + * | Check if the given oid (or an oid prefix) exists in any of the * | packs that have been loaded for our ODB. * | * |-# pack_entry_find / pack_entry_find_prefix - * | If there is a multi-pack-index present, search the SHA1 oid in that + * | If there is a multi-pack-index present, search the oid in that * | index first. If it is not found there, iterate through all the unindexed * | packs that have been preloaded (starting by the pack where the latest * | object was found) to try to find the OID in one of them. * | * |-# git_midx_entry_find - * | Search for the SHA1 oid in the multi-pack-index. See + * | Search for the oid in the multi-pack-index. See * | <https://github.com/git/git/blob/master/Documentation/technical/pack-format.txt> * | for specifics on the multi-pack-index format and how do we find * | entries in it. * | * |-# git_pack_entry_find - * | Check the index of an individual unindexed pack to see if the SHA1 - * | OID can be found. If we can find the offset to that SHA1 inside of the + * | Check the index of an individual unindexed pack to see if the + * | OID can be found. If we can find the offset to that inside of the * | index, that means the object is contained inside of the packfile and * | we can stop searching. Before returning, we verify that the * | packfile behind the index we are searching still exists on disk. @@ -141,13 +142,13 @@ struct pack_writepack { * -------------------------------------------------- * * # pack_backend__read / pack_backend__read_prefix - * | Check if the given SHA1 oid (or a SHA1 oid prefix) exists in any of the + * | Check if the given oid (or an oid prefix) exists in any of the * | packs that have been loaded for our ODB. If it does, open the packfile and * | read from it. * | * |-# git_packfile_unpack * Armed with a packfile and the offset within it, we can finally unpack - * the object pointed at by the SHA1 oid. This involves mmapping part of + * the object pointed at by the oid. This involves mmapping part of * the `.pack` file, and uncompressing the object within it (if it is * stored in the undelfitied representation), or finding a base object and * applying some deltas to its uncompressed representation (if it is stored @@ -177,7 +178,7 @@ static int pack_entry_find(struct git_pack_entry *e, * a prefix of an identifier. * Sets GIT_EAMBIGUOUS if short oid is ambiguous. * This method assumes that len is between - * GIT_OID_MINPREFIXLEN and GIT_OID_SHA1_HEXSIZE. + * GIT_OID_MINPREFIXLEN and the hexsize for the hash type. */ static int pack_entry_find_prefix( struct git_pack_entry *e, @@ -251,7 +252,7 @@ static int packfile_load__cb(void *data, git_str *path) if (git_vector_search2(NULL, &backend->packs, packfile_byname_search_cmp, &index_prefix) == 0) return 0; - error = git_mwindow_get_pack(&pack, path->ptr); + error = git_mwindow_get_pack(&pack, path->ptr, backend->opts.oid_type); /* ignore missing .pack file as git does */ if (error == GIT_ENOTFOUND) { @@ -270,33 +271,34 @@ static int pack_entry_find(struct git_pack_entry *e, struct pack_backend *backen { struct git_pack_file *last_found = backend->last_found, *p; git_midx_entry midx_entry; + size_t oid_hexsize = git_oid_hexsize(backend->opts.oid_type); size_t i; if (backend->midx && - git_midx_entry_find(&midx_entry, backend->midx, oid, GIT_OID_SHA1_HEXSIZE) == 0 && + git_midx_entry_find(&midx_entry, backend->midx, oid, oid_hexsize) == 0 && midx_entry.pack_index < git_vector_length(&backend->midx_packs)) { e->offset = midx_entry.offset; - git_oid_cpy(&e->sha1, &midx_entry.sha1); + git_oid_cpy(&e->id, &midx_entry.sha1); e->p = git_vector_get(&backend->midx_packs, midx_entry.pack_index); return 0; } if (last_found && - git_pack_entry_find(e, last_found, oid, GIT_OID_SHA1_HEXSIZE) == 0) + git_pack_entry_find(e, last_found, oid, oid_hexsize) == 0) return 0; git_vector_foreach(&backend->packs, i, p) { if (p == last_found) continue; - if (git_pack_entry_find(e, p, oid, GIT_OID_SHA1_HEXSIZE) == 0) { + if (git_pack_entry_find(e, p, oid, oid_hexsize) == 0) { backend->last_found = p; return 0; } } return git_odb__error_notfound( - "failed to find pack entry", oid, GIT_OID_SHA1_HEXSIZE); + "failed to find pack entry", oid, oid_hexsize); } static int pack_entry_find_prefix( @@ -318,9 +320,9 @@ static int pack_entry_find_prefix( return error; if (!error && midx_entry.pack_index < git_vector_length(&backend->midx_packs)) { e->offset = midx_entry.offset; - git_oid_cpy(&e->sha1, &midx_entry.sha1); + git_oid_cpy(&e->id, &midx_entry.sha1); e->p = git_vector_get(&backend->midx_packs, midx_entry.pack_index); - git_oid_cpy(&found_full_oid, &e->sha1); + git_oid_cpy(&found_full_oid, &e->id); found = true; } } @@ -330,9 +332,9 @@ static int pack_entry_find_prefix( if (error == GIT_EAMBIGUOUS) return error; if (!error) { - if (found && git_oid_cmp(&e->sha1, &found_full_oid)) + if (found && git_oid_cmp(&e->id, &found_full_oid)) return git_odb__error_ambiguous("found multiple pack entries"); - git_oid_cpy(&found_full_oid, &e->sha1); + git_oid_cpy(&found_full_oid, &e->id); found = true; } } @@ -345,9 +347,9 @@ static int pack_entry_find_prefix( if (error == GIT_EAMBIGUOUS) return error; if (!error) { - if (found && git_oid_cmp(&e->sha1, &found_full_oid)) + if (found && git_oid_cmp(&e->id, &found_full_oid)) return git_odb__error_ambiguous("found multiple pack entries"); - git_oid_cpy(&found_full_oid, &e->sha1); + git_oid_cpy(&found_full_oid, &e->id); found = true; backend->last_found = p; } @@ -425,7 +427,10 @@ static int process_multi_pack_index_pack( } /* Pack was not found. Allocate a new one. */ - error = git_mwindow_get_pack(&pack, git_str_cstr(&pack_path)); + error = git_mwindow_get_pack( + &pack, + git_str_cstr(&pack_path), + backend->opts.oid_type); git_str_dispose(&pack_path); if (error < 0) return error; @@ -596,32 +601,33 @@ static int pack_backend__read_prefix( void **buffer_p, size_t *len_p, git_object_t *type_p, - git_odb_backend *backend, + git_odb_backend *_backend, const git_oid *short_oid, size_t len) { + struct pack_backend *backend = (struct pack_backend *)_backend; int error = 0; if (len < GIT_OID_MINPREFIXLEN) error = git_odb__error_ambiguous("prefix length too short"); - else if (len >= GIT_OID_SHA1_HEXSIZE) { + else if (len >= git_oid_hexsize(backend->opts.oid_type)) { /* We can fall back to regular read method */ - error = pack_backend__read(buffer_p, len_p, type_p, backend, short_oid); + error = pack_backend__read(buffer_p, len_p, type_p, _backend, short_oid); if (!error) git_oid_cpy(out_oid, short_oid); } else { struct git_pack_entry e; git_rawobj raw = {NULL}; - if ((error = pack_entry_find_prefix( - &e, (struct pack_backend *)backend, short_oid, len)) == 0 && - (error = git_packfile_unpack(&raw, e.p, &e.offset)) == 0) + if ((error = pack_entry_find_prefix(&e, + backend, short_oid, len)) == 0 && + (error = git_packfile_unpack(&raw, e.p, &e.offset)) == 0) { *buffer_p = raw.data; *len_p = raw.len; *type_p = raw.type; - git_oid_cpy(out_oid, &e.sha1); + git_oid_cpy(out_oid, &e.id); } } @@ -642,7 +648,7 @@ static int pack_backend__exists_prefix( struct git_pack_entry e = {0}; error = pack_entry_find_prefix(&e, pb, short_id, len); - git_oid_cpy(out, &e.sha1); + git_oid_cpy(out, &e.id); return error; } @@ -712,6 +718,7 @@ static int pack_backend__writepack(struct git_odb_writepack **out, git_indexer_options opts = GIT_INDEXER_OPTIONS_INIT; struct pack_backend *backend; struct pack_writepack *writepack; + int error; GIT_ASSERT_ARG(out); GIT_ASSERT_ARG(_backend); @@ -726,11 +733,20 @@ static int pack_backend__writepack(struct git_odb_writepack **out, writepack = git__calloc(1, sizeof(struct pack_writepack)); GIT_ERROR_CHECK_ALLOC(writepack); - if (git_indexer_new(&writepack->indexer, - backend->pack_folder, 0, odb, &opts) < 0) { - git__free(writepack); +#ifdef GIT_EXPERIMENTAL_SHA256 + opts.odb = odb; + + error = git_indexer_new(&writepack->indexer, + backend->pack_folder, + backend->opts.oid_type, + &opts); +#else + error = git_indexer_new(&writepack->indexer, + backend->pack_folder, 0, odb, &opts); +#endif + + if (error < 0) return -1; - } writepack->parent.backend = _backend; writepack->parent.append = pack_backend__writepack_append; @@ -840,7 +856,10 @@ static void pack_backend__free(git_odb_backend *_backend) git__free(backend); } -static int pack_backend__alloc(struct pack_backend **out, size_t initial_size) +static int pack_backend__alloc( + struct pack_backend **out, + size_t initial_size, + const git_odb_backend_pack_options *opts) { struct pack_backend *backend = git__calloc(1, sizeof(struct pack_backend)); GIT_ERROR_CHECK_ALLOC(backend); @@ -849,12 +868,19 @@ static int pack_backend__alloc(struct pack_backend **out, size_t initial_size) git__free(backend); return -1; } + if (git_vector_init(&backend->packs, initial_size, packfile_sort__cb) < 0) { git_vector_free(&backend->midx_packs); git__free(backend); return -1; } + if (opts) + memcpy(&backend->opts, opts, sizeof(git_odb_backend_pack_options)); + + if (!backend->opts.oid_type) + backend->opts.oid_type = GIT_OID_DEFAULT; + backend->parent.version = GIT_ODB_BACKEND_VERSION; backend->parent.read = &pack_backend__read; @@ -873,17 +899,31 @@ static int pack_backend__alloc(struct pack_backend **out, size_t initial_size) return 0; } -int git_odb_backend_one_pack(git_odb_backend **backend_out, const char *idx) +#ifdef GIT_EXPERIMENTAL_SHA256 +int git_odb_backend_one_pack( + git_odb_backend **backend_out, + const char *idx, + const git_odb_backend_pack_options *opts) +#else +int git_odb_backend_one_pack( + git_odb_backend **backend_out, + const char *idx) +#endif { struct pack_backend *backend = NULL; struct git_pack_file *packfile = NULL; - if (pack_backend__alloc(&backend, 1) < 0) +#ifndef GIT_EXPERIMENTAL_SHA256 + git_odb_backend_pack_options *opts = NULL; +#endif + + git_oid_t oid_type = opts ? opts->oid_type : 0; + + if (pack_backend__alloc(&backend, 1, opts) < 0) return -1; - if (git_mwindow_get_pack(&packfile, idx) < 0 || - git_vector_insert(&backend->packs, packfile) < 0) - { + if (git_mwindow_get_pack(&packfile, idx, oid_type) < 0 || + git_vector_insert(&backend->packs, packfile) < 0) { pack_backend__free((git_odb_backend *)backend); return -1; } @@ -892,18 +932,30 @@ int git_odb_backend_one_pack(git_odb_backend **backend_out, const char *idx) return 0; } -int git_odb_backend_pack(git_odb_backend **backend_out, const char *objects_dir) +#ifdef GIT_EXPERIMENTAL_SHA256 +int git_odb_backend_pack( + git_odb_backend **backend_out, + const char *objects_dir, + const git_odb_backend_pack_options *opts) +#else +int git_odb_backend_pack( + git_odb_backend **backend_out, + const char *objects_dir) +#endif { int error = 0; struct pack_backend *backend = NULL; git_str path = GIT_STR_INIT; - if (pack_backend__alloc(&backend, 8) < 0) +#ifndef GIT_EXPERIMENTAL_SHA256 + git_odb_backend_pack_options *opts = NULL; +#endif + + if (pack_backend__alloc(&backend, 8, opts) < 0) return -1; if (!(error = git_str_joinpath(&path, objects_dir, "pack")) && - git_fs_path_isdir(git_str_cstr(&path))) - { + git_fs_path_isdir(git_str_cstr(&path))) { backend->pack_folder = git_str_detach(&path); error = pack_backend__refresh((git_odb_backend *)backend); } diff --git a/src/libgit2/oid.h b/src/libgit2/oid.h index d775e180b..7b6b09d8b 100644 --- a/src/libgit2/oid.h +++ b/src/libgit2/oid.h @@ -66,6 +66,47 @@ GIT_INLINE(size_t) git_oid_hexsize(git_oid_t type) return 0; } +GIT_INLINE(const char *) git_oid_type_name(git_oid_t type) +{ + switch (type) { + case GIT_OID_SHA1: + return "sha1"; + +#ifdef GIT_EXPERIMENTAL_SHA256 + case GIT_OID_SHA256: + return "sha256"; +#endif + } + + return "unknown"; +} + +GIT_INLINE(git_oid_t) git_oid_type_fromstr(const char *name) +{ + if (strcmp(name, "sha1") == 0) + return GIT_OID_SHA1; + +#ifdef GIT_EXPERIMENTAL_SHA256 + if (strcmp(name, "sha256") == 0) + return GIT_OID_SHA256; +#endif + + return 0; +} + +GIT_INLINE(git_oid_t) git_oid_type_fromstrn(const char *name, size_t len) +{ + if (len == CONST_STRLEN("sha1") && strncmp(name, "sha1", len) == 0) + return GIT_OID_SHA1; + +#ifdef GIT_EXPERIMENTAL_SHA256 + if (len == CONST_STRLEN("sha256") && strncmp(name, "sha256", len) == 0) + return GIT_OID_SHA256; +#endif + + return 0; +} + GIT_INLINE(git_hash_algorithm_t) git_oid_algorithm(git_oid_t type) { switch (type) { diff --git a/src/libgit2/pack-objects.c b/src/libgit2/pack-objects.c index 068231649..20a5dfcbd 100644 --- a/src/libgit2/pack-objects.c +++ b/src/libgit2/pack-objects.c @@ -1407,7 +1407,18 @@ int git_packbuilder_write( opts.progress_cb = progress_cb; opts.progress_cb_payload = progress_cb_payload; - if ((error = git_indexer_new(&indexer, path, mode, pb->odb, &opts)) < 0) + /* TODO: SHA256 */ + +#ifdef GIT_EXPERIMENTAL_SHA256 + opts.mode = mode; + opts.odb = pb->odb; + + error = git_indexer_new(&indexer, path, GIT_OID_SHA1, &opts); +#else + error = git_indexer_new(&indexer, path, mode, pb->odb, &opts); +#endif + + if (error < 0) goto cleanup; if (!git_repository__configmap_lookup(&t, pb->repo, GIT_CONFIGMAP_FSYNCOBJECTFILES) && t) diff --git a/src/libgit2/pack.c b/src/libgit2/pack.c index d428729ea..c30801844 100644 --- a/src/libgit2/pack.c +++ b/src/libgit2/pack.c @@ -32,7 +32,7 @@ static int packfile_unpack_compressed( * Throws GIT_EAMBIGUOUSOIDPREFIX if short oid * is ambiguous within the pack. * This method assumes that len is between - * GIT_OID_MINPREFIXLEN and GIT_OID_SHA1_HEXSIZE. + * GIT_OID_MINPREFIXLEN and the oid type's hexsize. */ static int pack_entry_find_offset( off64_t *offset_out, @@ -186,9 +186,9 @@ static int cache_add( static void pack_index_free(struct git_pack_file *p) { - if (p->oids) { - git__free(p->oids); - p->oids = NULL; + if (p->ids) { + git__free(p->ids); + p->ids = NULL; } if (p->index_map.data) { git_futils_mmap_free(&p->index_map); @@ -205,6 +205,7 @@ static int pack_index_check_locked(const char *path, struct git_pack_file *p) size_t idx_size; struct stat st; int error; + /* TODO: properly open the file without access time using O_NOATIME */ git_file fd = git_futils_open_ro(path); if (fd < 0) @@ -218,8 +219,7 @@ static int pack_index_check_locked(const char *path, struct git_pack_file *p) if (!S_ISREG(st.st_mode) || !git__is_sizet(st.st_size) || - (idx_size = (size_t)st.st_size) < 4 * 256 + 20 + 20) - { + (idx_size = (size_t)st.st_size) < (size_t)((4 * 256) + (p->oid_size * 2))) { p_close(fd); git_error_set(GIT_ERROR_ODB, "invalid pack index '%s'", path); return -1; @@ -242,8 +242,9 @@ static int pack_index_check_locked(const char *path, struct git_pack_file *p) return packfile_error("unsupported index version"); } - } else + } else { version = 1; + } nr = 0; index = idx_map; @@ -264,11 +265,11 @@ static int pack_index_check_locked(const char *path, struct git_pack_file *p) /* * Total size: * - 256 index entries 4 bytes each - * - 24-byte entries * nr (20-byte sha1 + 4-byte offset) - * - 20-byte SHA1 of the packfile - * - 20-byte SHA1 file checksum + * - 24/36-byte entries * nr (20/32 byte SHA + 4-byte offset) + * - 20/32-byte SHA of the packfile + * - 20/32-byte SHA file checksum */ - if (idx_size != 4*256 + nr * 24 + 20 + 20) { + if (idx_size != (4 * 256 + (nr * (p->oid_size + 4)) + (p->oid_size * 2))) { git_futils_mmap_free(&p->index_map); return packfile_error("index is corrupted"); } @@ -277,16 +278,16 @@ static int pack_index_check_locked(const char *path, struct git_pack_file *p) * Minimum size: * - 8 bytes of header * - 256 index entries 4 bytes each - * - 20-byte sha1 entry * nr + * - 20/32-byte SHA entry * nr * - 4-byte crc entry * nr * - 4-byte offset entry * nr - * - 20-byte SHA1 of the packfile - * - 20-byte SHA1 file checksum + * - 20/32-byte SHA of the packfile + * - 20/32-byte SHA file checksum * And after the 4-byte offset table might be a * variable sized table containing 8-byte entries * for offsets larger than 2^31. */ - unsigned long min_size = 8 + 4*256 + nr*(20 + 4 + 4) + 20 + 20; + unsigned long min_size = 8 + (4 * 256) + (nr * (p->oid_size + 4 + 4)) + (p->oid_size * 2); unsigned long max_size = min_size; if (nr) @@ -365,12 +366,12 @@ static unsigned char *pack_window_open( * Don't allow a negative offset, as that means we've wrapped * around. */ - if (offset > (p->mwf.size - 20)) + if (offset > (p->mwf.size - p->oid_size)) goto cleanup; if (offset < 0) goto cleanup; - pack_data = git_mwindow_open(&p->mwf, w_cursor, offset, 20, left); + pack_data = git_mwindow_open(&p->mwf, w_cursor, offset, p->oid_size, left); cleanup: git_mutex_unlock(&p->mwf.lock); @@ -473,13 +474,13 @@ int git_packfile_unpack_header( return error; } - /* pack_window_open() assures us we have [base, base + 20) available - * as a range that we can look at at. (Its actually the hash - * size that is assured.) With our object header encoding - * the maximum deflated object size is 2^137, which is just - * insane, so we know won't exceed what we have been given. + /* pack_window_open() assures us we have [base, base + oid_size) + * available as a range that we can look at at. (It's actually + * the hash size that is assured.) With our object header + * encoding the maximum deflated object size is 2^137, which is + * just insane, so we know won't exceed what we have been given. */ - base = git_mwindow_open(&p->mwf, w_curs, *curpos, 20, &left); + base = git_mwindow_open(&p->mwf, w_curs, *curpos, p->oid_size, &left); git_mutex_unlock(&p->lock); git_mutex_unlock(&p->mwf.lock); if (base == NULL) @@ -977,11 +978,12 @@ int get_delta_base( /* Assumption: the only reason this would fail is because the file is too small */ if (base_info == NULL) return GIT_EBUFS; - /* pack_window_open() assured us we have [base_info, base_info + 20) - * as a range that we can look at without walking off the - * end of the mapped window. Its actually the hash size - * that is assured. An OFS_DELTA longer than the hash size - * is stupid, as then a REF_DELTA would be smaller to store. + /* pack_window_open() assured us we have + * [base_info, base_info + oid_size) as a range that we can look + * at without walking off the end of the mapped window. Its + * actually the hash size that is assured. An OFS_DELTA longer + * than the hash size is stupid, as then a REF_DELTA would be + * smaller to store. */ if (type == GIT_OBJECT_OFS_DELTA) { unsigned used = 0; @@ -1002,7 +1004,7 @@ int get_delta_base( *curpos += used; } else if (type == GIT_OBJECT_REF_DELTA) { git_oid base_oid; - git_oid__fromraw(&base_oid, base_info, GIT_OID_SHA1); + git_oid__fromraw(&base_oid, base_info, p->oid_type); /* If we have the cooperative cache, search in it first */ if (p->has_cache) { @@ -1012,7 +1014,7 @@ int get_delta_base( if (entry->offset == 0) return packfile_error("delta offset is zero"); - *curpos += 20; + *curpos += p->oid_size; *delta_base_out = entry->offset; return 0; } else { @@ -1025,9 +1027,9 @@ int get_delta_base( } /* The base entry _must_ be in the same pack */ - if (pack_entry_find_offset(&base_offset, &unused, p, &base_oid, GIT_OID_SHA1_HEXSIZE) < 0) + if (pack_entry_find_offset(&base_offset, &unused, p, &base_oid, p->oid_hexsize) < 0) return packfile_error("base entry delta is not in the same pack"); - *curpos += 20; + *curpos += p->oid_size; } else return packfile_error("unknown object type"); @@ -1070,7 +1072,7 @@ void git_packfile_free(struct git_pack_file *p, bool unlink_packfile) pack_index_free(p); - git__free(p->bad_object_sha1); + git__free(p->bad_object_ids); git_mutex_free(&p->bases.lock); git_mutex_free(&p->mwf.lock); @@ -1083,8 +1085,8 @@ static int packfile_open_locked(struct git_pack_file *p) { struct stat st; struct git_pack_header hdr; - unsigned char sha1[GIT_OID_SHA1_SIZE]; - unsigned char *idx_sha1; + unsigned char checksum[GIT_OID_MAX_SIZE]; + unsigned char *idx_checksum; if (pack_index_open_locked(p) < 0) return git_odb__error_notfound("failed to open packfile", NULL, 0); @@ -1131,12 +1133,13 @@ static int packfile_open_locked(struct git_pack_file *p) /* Verify the pack matches its index. */ if (p->num_objects != ntohl(hdr.hdr_entries) || - p_pread(p->mwf.fd, sha1, GIT_OID_SHA1_SIZE, p->mwf.size - GIT_OID_SHA1_SIZE) < 0) + p_pread(p->mwf.fd, checksum, p->oid_size, p->mwf.size - p->oid_size) < 0) goto cleanup; - idx_sha1 = ((unsigned char *)p->index_map.data) + p->index_map.len - 40; + idx_checksum = ((unsigned char *)p->index_map.data) + + p->index_map.len - (p->oid_size * 2); - if (git_oid_raw_cmp(sha1, idx_sha1, GIT_OID_SHA1_SIZE) != 0) + if (git_oid_raw_cmp(checksum, idx_checksum, p->oid_size) != 0) goto cleanup; if (git_mwindow_file_register(&p->mwf) < 0) @@ -1171,7 +1174,10 @@ int git_packfile__name(char **out, const char *path) return 0; } -int git_packfile_alloc(struct git_pack_file **pack_out, const char *path) +int git_packfile_alloc( + struct git_pack_file **pack_out, + const char *path, + git_oid_t oid_type) { struct stat st; struct git_pack_file *p; @@ -1219,6 +1225,9 @@ int git_packfile_alloc(struct git_pack_file **pack_out, const char *path) p->pack_local = 1; p->mtime = (git_time_t)st.st_mtime; p->index_version = -1; + p->oid_type = oid_type ? oid_type : GIT_OID_DEFAULT; + p->oid_size = (unsigned int)git_oid_size(p->oid_type); + p->oid_hexsize = (unsigned int)git_oid_hexsize(p->oid_type); if (git_mutex_init(&p->lock) < 0) { git_error_set(GIT_ERROR_OS, "failed to initialize packfile mutex"); @@ -1260,9 +1269,9 @@ static off64_t nth_packed_object_offset_locked(struct git_pack_file *p, uint32_t end = index + p->index_map.len; index += 4 * 256; if (p->index_version == 1) - return ntohl(*((uint32_t *)(index + 24 * n))); + return ntohl(*((uint32_t *)(index + (p->oid_size + 4) * n))); - index += 8 + p->num_objects * (20 + 4); + index += 8 + p->num_objects * (p->oid_size + 4); off32 = ntohl(*((uint32_t *)(index + 4 * n))); if (!(off32 & 0x80000000)) return off32; @@ -1273,7 +1282,7 @@ static off64_t nth_packed_object_offset_locked(struct git_pack_file *p, uint32_t return -1; return (((uint64_t)ntohl(*((uint32_t *)(index + 0)))) << 32) | - ntohl(*((uint32_t *)(index + 4))); + ntohl(*((uint32_t *)(index + 4))); } static int git__memcmp4(const void *a, const void *b) { @@ -1312,7 +1321,7 @@ int git_pack_foreach_entry( index += 4 * 256; - if (p->oids == NULL) { + if (p->ids == NULL) { git_vector offsets, oids; if ((error = git_vector_init(&oids, p->num_objects, NULL))) { @@ -1326,22 +1335,25 @@ int git_pack_foreach_entry( } if (p->index_version > 1) { - const unsigned char *off = index + 24 * p->num_objects; + const unsigned char *off = index + + (p->oid_size + 4) * p->num_objects; + for (i = 0; i < p->num_objects; i++) git_vector_insert(&offsets, (void*)&off[4 * i]); + git_vector_sort(&offsets); git_vector_foreach(&offsets, i, current) git_vector_insert(&oids, (void*)&index[5 * (current - off)]); } else { for (i = 0; i < p->num_objects; i++) - git_vector_insert(&offsets, (void*)&index[24 * i]); + git_vector_insert(&offsets, (void*)&index[(p->oid_size + 4) * i]); git_vector_sort(&offsets); git_vector_foreach(&offsets, i, current) git_vector_insert(&oids, (void*)¤t[4]); } git_vector_free(&offsets); - p->oids = (unsigned char **)git_vector_detach(NULL, NULL, &oids); + p->ids = (unsigned char **)git_vector_detach(NULL, NULL, &oids); } /* @@ -1362,7 +1374,7 @@ int git_pack_foreach_entry( git_array_clear(oids); GIT_ERROR_CHECK_ALLOC(oid); } - git_oid__fromraw(oid, p->oids[i], GIT_OID_SHA1); + git_oid__fromraw(oid, p->ids[i], p->oid_type); } git_mutex_unlock(&p->lock); @@ -1412,10 +1424,13 @@ int git_pack_foreach_entry_offset( /* all offsets should have been validated by pack_index_check_locked */ if (p->index_version > 1) { - const unsigned char *offsets = index + 24 * p->num_objects; + const unsigned char *offsets = index + + (p->oid_size + 4) * p->num_objects; const unsigned char *large_offset_ptr; - const unsigned char *large_offsets = index + 28 * p->num_objects; - const unsigned char *large_offsets_end = ((const unsigned char *)p->index_map.data) + p->index_map.len - 20; + const unsigned char *large_offsets = index + + (p->oid_size + 8) * p->num_objects; + const unsigned char *large_offsets_end = ((const unsigned char *)p->index_map.data) + p->index_map.len - p->oid_size; + for (i = 0; i < p->num_objects; i++) { current_offset = ntohl(*(const uint32_t *)(offsets + 4 * i)); if (current_offset & 0x80000000) { @@ -1428,7 +1443,7 @@ int git_pack_foreach_entry_offset( ntohl(*((uint32_t *)(large_offset_ptr + 4))); } - git_oid__fromraw(¤t_oid, (index + 20 * i), GIT_OID_SHA1); + git_oid__fromraw(¤t_oid, (index + p->oid_size * i), p->oid_type); if ((error = cb(¤t_oid, current_offset, data)) != 0) { error = git_error_set_after_callback(error); goto cleanup; @@ -1436,8 +1451,8 @@ int git_pack_foreach_entry_offset( } } else { for (i = 0; i < p->num_objects; i++) { - current_offset = ntohl(*(const uint32_t *)(index + 24 * i)); - git_oid__fromraw(¤t_oid, (index + 24 * i + 4), GIT_OID_SHA1); + current_offset = ntohl(*(const uint32_t *)(index + (p->oid_size + 4) * i)); + git_oid__fromraw(¤t_oid, (index + (p->oid_size + 4) * i + 4), p->oid_type); if ((error = cb(¤t_oid, current_offset, data)) != 0) { error = git_error_set_after_callback(error); goto cleanup; @@ -1450,14 +1465,20 @@ cleanup: return error; } -int git_pack__lookup_sha1(const void *oid_lookup_table, size_t stride, unsigned lo, - unsigned hi, const unsigned char *oid_prefix) +int git_pack__lookup_id( + const void *oid_lookup_table, + size_t stride, + unsigned lo, + unsigned hi, + const unsigned char *oid_prefix, + const git_oid_t oid_type) { const unsigned char *base = oid_lookup_table; + size_t oid_size = git_oid_size(oid_type); while (lo < hi) { unsigned mi = (lo + hi) / 2; - int cmp = git_oid_raw_cmp(base + mi * stride, oid_prefix, GIT_OID_SHA1_SIZE); + int cmp = git_oid_raw_cmp(base + mi * stride, oid_prefix, oid_size); if (!cmp) return mi; @@ -1512,9 +1533,9 @@ static int pack_entry_find_offset( lo = ((short_oid->id[0] == 0x0) ? 0 : ntohl(level1_ofs[(int)short_oid->id[0] - 1])); if (p->index_version > 1) { - stride = 20; + stride = p->oid_size; } else { - stride = 24; + stride = p->oid_size + 4; index += 4; } @@ -1523,7 +1544,8 @@ static int pack_entry_find_offset( short_oid->id[0], short_oid->id[1], short_oid->id[2], lo, hi, p->num_objects); #endif - pos = git_pack__lookup_sha1(index, stride, lo, hi, short_oid->id); + pos = git_pack__lookup_id(index, stride, lo, hi, + short_oid->id, p->oid_type); if (pos >= 0) { /* An object matching exactly the oid was found */ @@ -1541,7 +1563,9 @@ static int pack_entry_find_offset( } } - if (found && len != GIT_OID_SHA1_HEXSIZE && pos + 1 < (int)p->num_objects) { + if (found && + len != p->oid_hexsize && + pos + 1 < (int)p->num_objects) { /* Check for ambiguousity */ const unsigned char *next = current + stride; @@ -1566,13 +1590,13 @@ static int pack_entry_find_offset( } *offset_out = offset; - git_oid__fromraw(found_oid, current, GIT_OID_SHA1); + git_oid__fromraw(found_oid, current, p->oid_type); #ifdef INDEX_DEBUG_LOOKUP { - unsigned char hex_sha1[GIT_OID_SHA1_HEXSIZE + 1]; + char hex_sha1[p->oid_hexsize + 1]; git_oid_fmt(hex_sha1, found_oid); - hex_sha1[GIT_OID_SHA1_HEXSIZE] = '\0'; + hex_sha1[p->oid_hexsize] = '\0'; printf("found lo=%d %s\n", lo, hex_sha1); } #endif @@ -1594,10 +1618,10 @@ int git_pack_entry_find( GIT_ASSERT_ARG(p); - if (len == GIT_OID_SHA1_HEXSIZE && p->num_bad_objects) { + if (len == p->oid_hexsize && p->num_bad_objects) { unsigned i; for (i = 0; i < p->num_bad_objects; i++) - if (git_oid__cmp(short_oid, &p->bad_object_sha1[i]) == 0) + if (git_oid__cmp(short_oid, &p->bad_object_ids[i]) == 0) return packfile_error("bad object found in packfile"); } @@ -1630,6 +1654,6 @@ int git_pack_entry_find( e->offset = offset; e->p = p; - git_oid_cpy(&e->sha1, &found_oid); + git_oid_cpy(&e->id, &found_oid); return 0; } diff --git a/src/libgit2/pack.h b/src/libgit2/pack.h index d90588f79..1a9eb14b2 100644 --- a/src/libgit2/pack.h +++ b/src/libgit2/pack.h @@ -99,13 +99,19 @@ struct git_pack_file { uint32_t num_objects; uint32_t num_bad_objects; - git_oid *bad_object_sha1; /* array of git_oid */ + git_oid *bad_object_ids; /* array of git_oid */ + + git_oid_t oid_type; + unsigned oid_hexsize:7, + oid_size:6, + pack_local:1, + pack_keep:1, + has_cache:1; int index_version; git_time_t mtime; - unsigned pack_local:1, pack_keep:1, has_cache:1; git_oidmap *idx_cache; - unsigned char **oids; + unsigned char **ids; git_pack_cache bases; /* delta base cache */ @@ -116,21 +122,26 @@ struct git_pack_file { }; /** - * Return the position where an OID (or a prefix) would be inserted within the - * OID Lookup Table of an .idx file. This performs binary search between the lo - * and hi indices. + * Return the position where an OID (or a prefix) would be inserted within + * the OID Lookup Table of an .idx file. This performs binary search + * between the lo and hi indices. * - * The stride parameter is provided because .idx files version 1 store the OIDs - * interleaved with the 4-byte file offsets of the objects within the .pack - * file (stride = 24), whereas files with version 2 store them in a contiguous - * flat array (stride = 20). + * The stride parameter is provided because .idx files version 1 store the + * OIDs interleaved with the 4-byte file offsets of the objects within the + * .pack file (stride = oid_size + 4), whereas files with version 2 store + * them in a contiguous flat array (stride = oid_size). */ -int git_pack__lookup_sha1(const void *oid_lookup_table, size_t stride, unsigned lo, - unsigned hi, const unsigned char *oid_prefix); +int git_pack__lookup_id( + const void *id_lookup_table, + size_t stride, + unsigned lo, + unsigned hi, + const unsigned char *id_prefix, + const git_oid_t oid_type); struct git_pack_entry { off64_t offset; - git_oid sha1; + git_oid id; struct git_pack_file *p; }; @@ -174,12 +185,15 @@ int get_delta_base( off64_t delta_obj_offset); void git_packfile_free(struct git_pack_file *p, bool unlink_packfile); -int git_packfile_alloc(struct git_pack_file **pack_out, const char *path); +int git_packfile_alloc( + struct git_pack_file **pack_out, + const char *path, + git_oid_t oid_type); int git_pack_entry_find( struct git_pack_entry *e, struct git_pack_file *p, - const git_oid *short_oid, + const git_oid *short_id, size_t len); int git_pack_foreach_entry( struct git_pack_file *p, diff --git a/src/libgit2/refdb_fs.c b/src/libgit2/refdb_fs.c index 43283b3e4..9ce1a9608 100644 --- a/src/libgit2/refdb_fs.c +++ b/src/libgit2/refdb_fs.c @@ -60,15 +60,17 @@ typedef struct refdb_fs_backend { /* path to common objects' directory */ char *commonpath; - git_sortedcache *refcache; + git_oid_t oid_type; + + int fsync : 1, + sorted : 1; int peeling_mode; git_iterator_flag_t iterator_flags; uint32_t direach_flags; - int fsync; + git_sortedcache *refcache; git_map packed_refs_map; git_mutex prlock; /* protect packed_refs_map */ git_futils_filestamp packed_refs_stamp; - bool sorted; } refdb_fs_backend; static int refdb_reflog_fs__delete(git_refdb_backend *_backend, const char *name); @@ -113,6 +115,7 @@ static int packed_reload(refdb_fs_backend *backend) { int error; git_str packedrefs = GIT_STR_INIT; + size_t oid_hexsize = git_oid_hexsize(backend->oid_type); char *scan, *eof, *eol; if (!backend->gitpath) @@ -158,9 +161,9 @@ static int packed_reload(refdb_fs_backend *backend) /* parse "<OID> <refname>\n" */ - if (git_oid__fromstr(&oid, scan, GIT_OID_SHA1) < 0) + if (git_oid__fromstr(&oid, scan, backend->oid_type) < 0) goto parse_failed; - scan += GIT_OID_SHA1_HEXSIZE; + scan += oid_hexsize; if (*scan++ != ' ') goto parse_failed; @@ -179,9 +182,9 @@ static int packed_reload(refdb_fs_backend *backend) /* look for optional "^<OID>\n" */ if (*scan == '^') { - if (git_oid__fromstr(&oid, scan + 1, GIT_OID_SHA1) < 0) + if (git_oid__fromstr(&oid, scan + 1, backend->oid_type) < 0) goto parse_failed; - scan += GIT_OID_SHA1_HEXSIZE + 1; + scan += oid_hexsize + 1; if (scan < eof) { if (!(eol = strchr(scan, '\n'))) @@ -214,19 +217,23 @@ parse_failed: } static int loose_parse_oid( - git_oid *oid, const char *filename, git_str *file_content) + git_oid *oid, + const char *filename, + git_str *file_content, + git_oid_t oid_type) { const char *str = git_str_cstr(file_content); + size_t oid_hexsize = git_oid_hexsize(oid_type); - if (git_str_len(file_content) < GIT_OID_SHA1_HEXSIZE) + if (git_str_len(file_content) < oid_hexsize) goto corrupted; /* we need to get 40 OID characters from the file */ - if (git_oid__fromstr(oid, str, GIT_OID_SHA1) < 0) + if (git_oid__fromstr(oid, str, oid_type) < 0) goto corrupted; /* If the file is longer than 40 chars, the 41st must be a space */ - str += GIT_OID_SHA1_HEXSIZE; + str += oid_hexsize; if (*str == '\0' || git__isspace(*str)) return 0; @@ -266,7 +273,7 @@ static int loose_lookup_to_packfile(refdb_fs_backend *backend, const char *name) goto done; /* parse OID from file */ - if ((error = loose_parse_oid(&oid, name, &ref_file)) < 0) + if ((error = loose_parse_oid(&oid, name, &ref_file, backend->oid_type)) < 0) goto done; if ((error = git_sortedcache_wlock(backend->refcache)) < 0) @@ -437,7 +444,7 @@ static int loose_lookup( } else { git_oid oid; - if (!(error = loose_parse_oid(&oid, ref_name, &ref_file)) && + if (!(error = loose_parse_oid(&oid, ref_name, &ref_file, backend->oid_type)) && out != NULL) *out = git_reference__alloc(ref_name, &oid, NULL); } @@ -615,19 +622,24 @@ static const char *end_of_record(const char *p, const char *end) return p; } -static int -cmp_record_to_refname(const char *rec, size_t data_end, const char *ref_name) +static int cmp_record_to_refname( + const char *rec, + size_t data_end, + const char *ref_name, + git_oid_t oid_type) { const size_t ref_len = strlen(ref_name); int cmp_val; const char *end; + size_t oid_hexsize = git_oid_hexsize(oid_type); - rec += GIT_OID_SHA1_HEXSIZE + 1; /* <oid> + space */ - if (data_end < GIT_OID_SHA1_HEXSIZE + 3) { - /* an incomplete (corrupt) record is treated as less than ref_name */ + rec += oid_hexsize + 1; /* <oid> + space */ + + /* an incomplete (corrupt) record is treated as less than ref_name */ + if (data_end < oid_hexsize + 3) return -1; - } - data_end -= GIT_OID_SHA1_HEXSIZE + 1; + + data_end -= oid_hexsize + 1; end = memchr(rec, '\n', data_end); if (end) @@ -675,6 +687,7 @@ static int packed_lookup( { int error = 0; const char *left, *right, *data_end; + size_t oid_hexsize = git_oid_hexsize(backend->oid_type); if ((error = packed_map_check(backend)) < 0) return error; @@ -698,7 +711,7 @@ static int packed_lookup( mid = left + (right - left) / 2; rec = start_of_record(left, mid); - compare = cmp_record_to_refname(rec, data_end - rec, ref_name); + compare = cmp_record_to_refname(rec, data_end - rec, ref_name, backend->oid_type); if (compare < 0) { left = end_of_record(mid, right); @@ -708,11 +721,11 @@ static int packed_lookup( const char *eol; git_oid oid, peel, *peel_ptr = NULL; - if (data_end - rec < GIT_OID_SHA1_HEXSIZE || - git_oid__fromstr(&oid, rec, GIT_OID_SHA1) < 0) { + if (data_end - rec < (long)oid_hexsize || + git_oid__fromstr(&oid, rec, backend->oid_type) < 0) { goto parse_failed; } - rec += GIT_OID_SHA1_HEXSIZE + 1; + rec += oid_hexsize + 1; if (!(eol = memchr(rec, '\n', data_end - rec))) { goto parse_failed; } @@ -724,8 +737,8 @@ static int packed_lookup( if (*rec == '^') { rec++; - if (data_end - rec < GIT_OID_SHA1_HEXSIZE || - git_oid__fromstr(&peel, rec, GIT_OID_SHA1) < 0) { + if (data_end - rec < (long)oid_hexsize || + git_oid__fromstr(&peel, rec, backend->oid_type) < 0) { goto parse_failed; } peel_ptr = &peel; @@ -1108,7 +1121,7 @@ static int loose_commit(git_filebuf *file, const git_reference *ref) GIT_ASSERT_ARG(ref); if (ref->type == GIT_REFERENCE_DIRECT) { - char oid[GIT_OID_SHA1_HEXSIZE + 1]; + char oid[GIT_OID_MAX_HEXSIZE + 1]; git_oid_nfmt(oid, sizeof(oid), &ref->target.oid); git_filebuf_printf(file, "%s\n", oid); @@ -1224,7 +1237,7 @@ static int packed_find_peel(refdb_fs_backend *backend, struct packref *ref) */ static int packed_write_ref(struct packref *ref, git_filebuf *file) { - char oid[GIT_OID_SHA1_HEXSIZE + 1]; + char oid[GIT_OID_MAX_HEXSIZE + 1]; git_oid_nfmt(oid, sizeof(oid), &ref->oid); /* @@ -1238,7 +1251,7 @@ static int packed_write_ref(struct packref *ref, git_filebuf *file) * The required peels have already been loaded into `ref->peel_target`. */ if (ref->flags & PACKREF_HAS_PEEL) { - char peel[GIT_OID_SHA1_HEXSIZE + 1]; + char peel[GIT_OID_MAX_HEXSIZE + 1]; git_oid_nfmt(peel, sizeof(peel), &ref->peel); if (git_filebuf_printf(file, "%s %s\n^%s\n", oid, ref->name, peel) < 0) @@ -1302,7 +1315,7 @@ static int packed_remove_loose(refdb_fs_backend *backend) continue; /* Figure out the current id; if we find a bad ref file, skip it so we can do the rest */ - if (loose_parse_oid(¤t_id, lock.path_original, &ref_content) < 0) + if (loose_parse_oid(¤t_id, lock.path_original, &ref_content, backend->oid_type) < 0) continue; /* If the ref moved since we packed it, we must not delete it */ @@ -1891,7 +1904,10 @@ done: return out; } -static int reflog_alloc(git_reflog **reflog, const char *name) +static int reflog_alloc( + git_reflog **reflog, + const char *name, + git_oid_t oid_type) { git_reflog *log; @@ -1903,6 +1919,8 @@ static int reflog_alloc(git_reflog **reflog, const char *name) log->ref_name = git__strdup(name); GIT_ERROR_CHECK_ALLOC(log->ref_name); + log->oid_type = oid_type; + if (git_vector_init(&log->entries, 0, NULL) < 0) { git__free(log->ref_name); git__free(log); @@ -2032,7 +2050,10 @@ static int refdb_reflog_fs__has_log(git_refdb_backend *_backend, const char *nam return has_reflog(backend->repo, name); } -static int refdb_reflog_fs__read(git_reflog **out, git_refdb_backend *_backend, const char *name) +static int refdb_reflog_fs__read( + git_reflog **out, + git_refdb_backend *_backend, + const char *name) { int error = -1; git_str log_path = GIT_STR_INIT; @@ -2048,7 +2069,7 @@ static int refdb_reflog_fs__read(git_reflog **out, git_refdb_backend *_backend, backend = GIT_CONTAINER_OF(_backend, refdb_fs_backend, parent); repo = backend->repo; - if (reflog_alloc(&log, name) < 0) + if (reflog_alloc(&log, name, backend->oid_type) < 0) return -1; if (reflog_path(&log_path, repo, name) < 0) @@ -2086,11 +2107,11 @@ static int serialize_reflog_entry( const git_signature *committer, const char *msg) { - char raw_old[GIT_OID_SHA1_HEXSIZE+1]; - char raw_new[GIT_OID_SHA1_HEXSIZE+1]; + char raw_old[GIT_OID_MAX_HEXSIZE + 1]; + char raw_new[GIT_OID_MAX_HEXSIZE + 1]; - git_oid_tostr(raw_old, GIT_OID_SHA1_HEXSIZE+1, oid_old); - git_oid_tostr(raw_new, GIT_OID_SHA1_HEXSIZE+1, oid_new); + git_oid_tostr(raw_old, GIT_OID_MAX_HEXSIZE + 1, oid_old); + git_oid_tostr(raw_new, GIT_OID_MAX_HEXSIZE + 1, oid_new); git_str_clear(buf); @@ -2189,10 +2210,16 @@ success: } /* Append to the reflog, must be called under reference lock */ -static int reflog_append(refdb_fs_backend *backend, const git_reference *ref, const git_oid *old, const git_oid *new, const git_signature *who, const char *message) +static int reflog_append( + refdb_fs_backend *backend, + const git_reference *ref, + const git_oid *old, + const git_oid *new, + const git_signature *who, + const char *message) { int error, is_symbolic, open_flags; - git_oid old_id = GIT_OID_SHA1_ZERO, new_id = GIT_OID_SHA1_ZERO; + git_oid old_id, new_id; git_str buf = GIT_STR_INIT, path = GIT_STR_INIT; git_repository *repo = backend->repo; @@ -2206,6 +2233,9 @@ static int reflog_append(refdb_fs_backend *backend, const git_reference *ref, co /* From here on is_symbolic also means that it's HEAD */ + git_oid_clear(&old_id, backend->oid_type); + git_oid_clear(&new_id, backend->oid_type); + if (old) { git_oid_cpy(&old_id, old); } else { @@ -2402,6 +2432,7 @@ int git_refdb_backend_fs( goto fail; backend->repo = repository; + backend->oid_type = repository->oid_type; if (repository->gitdir) { backend->gitpath = setup_namespace(repository, repository->gitdir); diff --git a/src/libgit2/reflog.c b/src/libgit2/reflog.c index eb0b7e48c..86d4355e3 100644 --- a/src/libgit2/reflog.c +++ b/src/libgit2/reflog.c @@ -71,7 +71,11 @@ int git_reflog_write(git_reflog *reflog) return db->backend->reflog_write(db->backend, reflog); } -int git_reflog_append(git_reflog *reflog, const git_oid *new_oid, const git_signature *committer, const char *msg) +int git_reflog_append( + git_reflog *reflog, + const git_oid *new_oid, + const git_signature *committer, + const char *msg) { const git_reflog_entry *previous; git_reflog_entry *entry; @@ -104,7 +108,7 @@ int git_reflog_append(git_reflog *reflog, const git_oid *new_oid, const git_sign previous = git_reflog_entry_byindex(reflog, 0); if (previous == NULL) - git_oid__fromstr(&entry->oid_old, GIT_OID_SHA1_HEXZERO, GIT_OID_SHA1); + git_oid_clear(&entry->oid_old, reflog->oid_type); else git_oid_cpy(&entry->oid_old, &previous->oid_cur); @@ -218,12 +222,8 @@ int git_reflog_drop(git_reflog *reflog, size_t idx, int rewrite_previous_entry) /* If the oldest entry has just been removed... */ if (idx == entrycount - 1) { - git_oid zero = GIT_OID_SHA1_ZERO; - /* ...clear the oid_old member of the "new" oldest entry */ - if (git_oid_cpy(&entry->oid_old, &zero) < 0) - return -1; - + git_oid_clear(&entry->oid_old, reflog->oid_type); return 0; } diff --git a/src/libgit2/reflog.h b/src/libgit2/reflog.h index 50d1056ed..bc9878598 100644 --- a/src/libgit2/reflog.h +++ b/src/libgit2/reflog.h @@ -16,8 +16,6 @@ #define GIT_REFLOG_DIR_MODE 0777 #define GIT_REFLOG_FILE_MODE 0666 -#define GIT_REFLOG_SIZE_MIN (2*GIT_OID_SHA1_HEXSIZE+2+17) - struct git_reflog_entry { git_oid oid_old; git_oid oid_cur; @@ -30,6 +28,7 @@ struct git_reflog_entry { struct git_reflog { git_refdb *db; char *ref_name; + git_oid_t oid_type; git_vector entries; }; diff --git a/src/libgit2/remote.c b/src/libgit2/remote.c index 02d271d7d..3583ec031 100644 --- a/src/libgit2/remote.c +++ b/src/libgit2/remote.c @@ -1026,6 +1026,24 @@ int git_remote_capabilities(unsigned int *out, git_remote *remote) return remote->transport->capabilities(out, remote->transport); } +int git_remote_oid_type(git_oid_t *out, git_remote *remote) +{ + GIT_ASSERT_ARG(remote); + + if (!remote->transport) { + git_error_set(GIT_ERROR_NET, "this remote has never connected"); + *out = 0; + return -1; + } + +#ifdef GIT_EXPERIMENTAL_SHA256 + return remote->transport->oid_type(out, remote->transport); +#else + *out = GIT_OID_SHA1; + return 0; +#endif +} + static int lookup_config(char **out, git_config *cfg, const char *name) { git_config_entry *ce = NULL; @@ -1225,24 +1243,6 @@ static int ls_to_vector(git_vector *out, git_remote *remote) return 0; } -#define copy_opts(out, in) \ - if (in) { \ - (out)->callbacks = (in)->callbacks; \ - (out)->proxy_opts = (in)->proxy_opts; \ - (out)->custom_headers = (in)->custom_headers; \ - (out)->follow_redirects = (in)->follow_redirects; \ - } - -GIT_INLINE(int) connect_opts_from_fetch_opts( - git_remote_connect_options *out, - git_remote *remote, - const git_fetch_options *fetch_opts) -{ - git_remote_connect_options tmp = GIT_REMOTE_CONNECT_OPTIONS_INIT; - copy_opts(&tmp, fetch_opts); - return git_remote_connect_options_normalize(out, remote->repo, &tmp); -} - static int connect_or_reset_options( git_remote *remote, int direction, @@ -1330,7 +1330,8 @@ int git_remote_download( return -1; } - if (connect_opts_from_fetch_opts(&connect_opts, remote, opts) < 0) + if (git_remote_connect_options__from_fetch_opts(&connect_opts, + remote, opts) < 0) return -1; if ((error = connect_or_reset_options(remote, GIT_DIRECTION_FETCH, &connect_opts)) < 0) @@ -1350,6 +1351,8 @@ int git_remote_fetch( bool prune = false; git_str reflog_msg_buf = GIT_STR_INIT; git_remote_connect_options connect_opts = GIT_REMOTE_CONNECT_OPTIONS_INIT; + unsigned int capabilities; + git_oid_t oid_type; GIT_ASSERT_ARG(remote); @@ -1358,7 +1361,8 @@ int git_remote_fetch( return -1; } - if (connect_opts_from_fetch_opts(&connect_opts, remote, opts) < 0) + if (git_remote_connect_options__from_fetch_opts(&connect_opts, + remote, opts) < 0) return -1; if ((error = connect_or_reset_options(remote, GIT_DIRECTION_FETCH, &connect_opts)) < 0) @@ -1369,6 +1373,10 @@ int git_remote_fetch( tagopt = opts->download_tags; } + if ((error = git_remote_capabilities(&capabilities, remote)) < 0 || + (error = git_remote_oid_type(&oid_type, remote)) < 0) + return error; + /* Connect and download everything */ error = git_remote__download(remote, refspecs, opts); @@ -2896,16 +2904,6 @@ done: return error; } -GIT_INLINE(int) connect_opts_from_push_opts( - git_remote_connect_options *out, - git_remote *remote, - const git_push_options *push_opts) -{ - git_remote_connect_options tmp = GIT_REMOTE_CONNECT_OPTIONS_INIT; - copy_opts(&tmp, push_opts); - return git_remote_connect_options_normalize(out, remote->repo, &tmp); -} - int git_remote_upload( git_remote *remote, const git_strarray *refspecs, @@ -2924,7 +2922,8 @@ int git_remote_upload( return -1; } - if ((error = connect_opts_from_push_opts(&connect_opts, remote, opts)) < 0) + if ((error = git_remote_connect_options__from_push_opts( + &connect_opts, remote, opts)) < 0) goto cleanup; if ((error = connect_or_reset_options(remote, GIT_DIRECTION_PUSH, &connect_opts)) < 0) @@ -2985,7 +2984,8 @@ int git_remote_push( return -1; } - if (connect_opts_from_push_opts(&connect_opts, remote, opts) < 0) + if (git_remote_connect_options__from_push_opts(&connect_opts, + remote, opts) < 0) return -1; if ((error = git_remote_upload(remote, refspecs, opts)) < 0) diff --git a/src/libgit2/remote.h b/src/libgit2/remote.h index 41ee58e0f..676b3c2ab 100644 --- a/src/libgit2/remote.h +++ b/src/libgit2/remote.h @@ -17,6 +17,7 @@ #include "refspec.h" #include "vector.h" #include "net.h" +#include "proxy.h" #define GIT_REMOTE_ORIGIN "origin" @@ -56,5 +57,44 @@ int git_remote_connect_options_normalize( const git_remote_connect_options *src); int git_remote_capabilities(unsigned int *out, git_remote *remote); +int git_remote_oid_type(git_oid_t *out, git_remote *remote); + + +#define git_remote_connect_options__copy_opts(out, in) \ + if (in) { \ + (out)->callbacks = (in)->callbacks; \ + (out)->proxy_opts = (in)->proxy_opts; \ + (out)->custom_headers = (in)->custom_headers; \ + (out)->follow_redirects = (in)->follow_redirects; \ + } + +GIT_INLINE(int) git_remote_connect_options__from_fetch_opts( + git_remote_connect_options *out, + git_remote *remote, + const git_fetch_options *fetch_opts) +{ + git_remote_connect_options tmp = GIT_REMOTE_CONNECT_OPTIONS_INIT; + git_remote_connect_options__copy_opts(&tmp, fetch_opts); + return git_remote_connect_options_normalize(out, remote->repo, &tmp); +} + +GIT_INLINE(int) git_remote_connect_options__from_push_opts( + git_remote_connect_options *out, + git_remote *remote, + const git_push_options *push_opts) +{ + git_remote_connect_options tmp = GIT_REMOTE_CONNECT_OPTIONS_INIT; + git_remote_connect_options__copy_opts(&tmp, push_opts); + return git_remote_connect_options_normalize(out, remote->repo, &tmp); +} + +#undef git_remote_connect_options__copy_opts + +GIT_INLINE(void) git_remote_connect_options__dispose( + git_remote_connect_options *opts) +{ + git_proxy_options_dispose(&opts->proxy_opts); + git_strarray_dispose(&opts->custom_headers); +} #endif diff --git a/src/libgit2/repository.c b/src/libgit2/repository.c index df41f581c..489d627a0 100644 --- a/src/libgit2/repository.c +++ b/src/libgit2/repository.c @@ -67,6 +67,7 @@ static const struct { static int check_repositoryformatversion(int *version, git_config *config); static int check_extensions(git_config *config, int version); static int load_global_config(git_config **config); +static int load_objectformat(git_repository *repo, git_config *config); #define GIT_COMMONDIR_FILE "commondir" #define GIT_GITDIR_FILE "gitdir" @@ -75,8 +76,8 @@ static int load_global_config(git_config **config); #define GIT_BRANCH_DEFAULT "master" -#define GIT_REPO_VERSION 0 -#define GIT_REPO_MAX_VERSION 1 +#define GIT_REPO_VERSION_DEFAULT 0 +#define GIT_REPO_VERSION_MAX 1 git_str git_repository__reserved_names_win32[] = { { DOT_GIT, 0, CONST_STRLEN(DOT_GIT) }, @@ -240,7 +241,7 @@ GIT_INLINE(int) validate_repo_path(git_str *path) */ static size_t suffix_len = CONST_STRLEN("objects/pack/pack-.pack.lock") + - GIT_OID_SHA1_HEXSIZE; + GIT_OID_MAX_HEXSIZE; return git_fs_path_validate_str_length_with_suffix( path, suffix_len); @@ -1015,7 +1016,8 @@ int git_repository_open_ext( if (error < 0 && error != GIT_ENOTFOUND) goto cleanup; - if (config && (error = check_repositoryformatversion(&version, config)) < 0) + if (config && + (error = check_repositoryformatversion(&version, config)) < 0) goto cleanup; if ((error = check_extensions(config, version)) < 0) @@ -1030,6 +1032,13 @@ int git_repository_open_ext( goto cleanup; } + if (version > 0) { + if ((error = load_objectformat(repo, config)) < 0) + goto cleanup; + } else { + repo->oid_type = GIT_OID_SHA1; + } + /* * Ensure that the git directory and worktree are * owned by the current user. @@ -1269,11 +1278,14 @@ int git_repository_odb__weakptr(git_odb **out, git_repository *repo) *out = git_atomic_load(repo->_odb); if (*out == NULL) { git_str odb_path = GIT_STR_INIT; + git_odb_options odb_opts = GIT_ODB_OPTIONS_INIT; git_odb *odb; + odb_opts.oid_type = repo->oid_type; + if ((error = git_repository__item_path(&odb_path, repo, GIT_REPOSITORY_ITEM_OBJECTS)) < 0 || - (error = git_odb__new(&odb, NULL)) < 0) + (error = git_odb__new(&odb, &odb_opts)) < 0) return error; GIT_REFCOUNT_OWN(odb, repo); @@ -1531,6 +1543,7 @@ static int check_repositoryformatversion(int *version, git_config *config) int error; error = git_config_get_int32(version, config, "core.repositoryformatversion"); + /* git ignores this if the config variable isn't there */ if (error == GIT_ENOTFOUND) return 0; @@ -1538,10 +1551,15 @@ static int check_repositoryformatversion(int *version, git_config *config) if (error < 0) return -1; - if (GIT_REPO_MAX_VERSION < *version) { + if (*version < 0) { + git_error_set(GIT_ERROR_REPOSITORY, + "invalid repository version %d", *version); + } + + if (GIT_REPO_VERSION_MAX < *version) { git_error_set(GIT_ERROR_REPOSITORY, "unsupported repository version %d; only versions up to %d are supported", - *version, GIT_REPO_MAX_VERSION); + *version, GIT_REPO_VERSION_MAX); return -1; } @@ -1549,7 +1567,8 @@ static int check_repositoryformatversion(int *version, git_config *config) } static const char *builtin_extensions[] = { - "noop" + "noop", + "objectformat" }; static git_vector user_extensions = GIT_VECTOR_INIT; @@ -1613,6 +1632,79 @@ static int check_extensions(git_config *config, int version) return git_config_foreach_match(config, "^extensions\\.", check_valid_extension, NULL); } +static int load_objectformat(git_repository *repo, git_config *config) +{ + git_config_entry *entry = NULL; + int error; + + if ((error = git_config_get_entry(&entry, config, "extensions.objectformat")) < 0) { + if (error == GIT_ENOTFOUND) { + repo->oid_type = GIT_OID_SHA1; + git_error_clear(); + error = 0; + } + + goto done; + } + + if ((repo->oid_type = git_oid_type_fromstr(entry->value)) == 0) { + git_error_set(GIT_ERROR_REPOSITORY, + "unknown object format '%s'", entry->value); + error = GIT_EINVALID; + } + +done: + git_config_entry_free(entry); + return error; +} + +int git_repository__set_objectformat( + git_repository *repo, + git_oid_t oid_type) +{ + git_config *cfg; + + /* + * Older clients do not necessarily understand the + * `objectformat` extension, even when it's set to an + * object format that they understand (SHA1). Do not set + * the objectformat extension unless we're not using the + * default object format. + */ + if (oid_type == GIT_OID_DEFAULT) + return 0; + + if (!git_repository_is_empty(repo) && repo->oid_type != oid_type) { + git_error_set(GIT_ERROR_REPOSITORY, + "cannot change object id type of existing repository"); + return -1; + } + + if (git_repository_config__weakptr(&cfg, repo) < 0) + return -1; + + if (git_config_set_int32(cfg, + "core.repositoryformatversion", 1) < 0 || + git_config_set_string(cfg, "extensions.objectformat", + git_oid_type_name(oid_type)) < 0) + return -1; + + /* + * During repo init, we may create some backends with the + * default oid type. Clear them so that we create them with + * the proper oid type. + */ + if (repo->oid_type != oid_type) { + set_index(repo, NULL); + set_odb(repo, NULL); + set_refdb(repo, NULL); + + repo->oid_type = oid_type; + } + + return 0; +} + int git_repository__extensions(char ***out, size_t *out_len) { git_vector extensions; @@ -1891,19 +1983,21 @@ static int repo_init_config( const char *repo_dir, const char *work_dir, uint32_t flags, - uint32_t mode) + uint32_t mode, + git_oid_t oid_type) { int error = 0; git_str cfg_path = GIT_STR_INIT, worktree_path = GIT_STR_INIT; git_config *config = NULL; bool is_bare = ((flags & GIT_REPOSITORY_INIT_BARE) != 0); bool is_reinit = ((flags & GIT_REPOSITORY_INIT__IS_REINIT) != 0); - int version = 0; + int version = GIT_REPO_VERSION_DEFAULT; if ((error = repo_local_config(&config, &cfg_path, NULL, repo_dir)) < 0) goto cleanup; - if (is_reinit && (error = check_repositoryformatversion(&version, config)) < 0) + if (is_reinit && + (error = check_repositoryformatversion(&version, config)) < 0) goto cleanup; if ((error = check_extensions(config, version)) < 0) @@ -1914,7 +2008,7 @@ static int repo_init_config( goto cleanup; } while (0) SET_REPO_CONFIG(bool, "core.bare", is_bare); - SET_REPO_CONFIG(int32, "core.repositoryformatversion", GIT_REPO_VERSION); + SET_REPO_CONFIG(int32, "core.repositoryformatversion", version); if ((error = repo_init_fs_configs( config, cfg_path.ptr, repo_dir, work_dir, !is_reinit)) < 0) @@ -1947,6 +2041,11 @@ static int repo_init_config( SET_REPO_CONFIG(bool, "receive.denyNonFastforwards", true); } + if (oid_type != GIT_OID_SHA1) { + SET_REPO_CONFIG(int32, "core.repositoryformatversion", 1); + SET_REPO_CONFIG(string, "extensions.objectformat", git_oid_type_name(oid_type)); + } + cleanup: git_str_dispose(&cfg_path); git_str_dispose(&worktree_path); @@ -2427,6 +2526,7 @@ int git_repository_init_ext( common_path = GIT_STR_INIT; const char *wd; bool is_valid; + git_oid_t oid_type = GIT_OID_DEFAULT; int error; GIT_ASSERT_ARG(out); @@ -2435,6 +2535,11 @@ int git_repository_init_ext( GIT_ERROR_CHECK_VERSION(opts, GIT_REPOSITORY_INIT_OPTIONS_VERSION, "git_repository_init_options"); +#ifdef GIT_EXPERIMENTAL_SHA256 + if (opts->oid_type) + oid_type = opts->oid_type; +#endif + if ((error = repo_init_directories(&repo_path, &wd_path, given_repo, opts)) < 0) goto out; @@ -2453,13 +2558,13 @@ int git_repository_init_ext( opts->flags |= GIT_REPOSITORY_INIT__IS_REINIT; - if ((error = repo_init_config(repo_path.ptr, wd, opts->flags, opts->mode)) < 0) + if ((error = repo_init_config(repo_path.ptr, wd, opts->flags, opts->mode, oid_type)) < 0) goto out; /* TODO: reinitialize the templates */ } else { if ((error = repo_init_structure(repo_path.ptr, wd, opts)) < 0 || - (error = repo_init_config(repo_path.ptr, wd, opts->flags, opts->mode)) < 0 || + (error = repo_init_config(repo_path.ptr, wd, opts->flags, opts->mode, oid_type)) < 0 || (error = repo_init_head(repo_path.ptr, opts->initial_head)) < 0) goto out; } @@ -2922,14 +3027,14 @@ int git_repository__set_orig_head(git_repository *repo, const git_oid *orig_head { git_filebuf file = GIT_FILEBUF_INIT; git_str file_path = GIT_STR_INIT; - char orig_head_str[GIT_OID_SHA1_HEXSIZE]; + char orig_head_str[GIT_OID_MAX_HEXSIZE]; int error = 0; git_oid_fmt(orig_head_str, orig_head); if ((error = git_str_joinpath(&file_path, repo->gitdir, GIT_ORIG_HEAD_FILE)) == 0 && (error = git_filebuf_open(&file, file_path.ptr, GIT_FILEBUF_CREATE_LEADING_DIRS, GIT_MERGE_FILE_MODE)) == 0 && - (error = git_filebuf_printf(&file, "%.*s\n", GIT_OID_SHA1_HEXSIZE, orig_head_str)) == 0) + (error = git_filebuf_printf(&file, "%.*s\n", (int)git_oid_hexsize(repo->oid_type), orig_head_str)) == 0) error = git_filebuf_commit(&file); if (error < 0) @@ -3042,7 +3147,7 @@ int git_repository_hashfile( goto cleanup; } - error = git_odb__hashfd_filtered(out, fd, (size_t)len, type, GIT_OID_SHA1, fl); + error = git_odb__hashfd_filtered(out, fd, (size_t)len, type, repo->oid_type, fl); cleanup: if (fd >= 0) @@ -3389,3 +3494,8 @@ int git_repository_submodule_cache_clear(git_repository *repo) repo->submodule_cache = NULL; return error; } + +git_oid_t git_repository_oid_type(git_repository *repo) +{ + return repo ? repo->oid_type : 0; +} diff --git a/src/libgit2/repository.h b/src/libgit2/repository.h index a488f2bf2..75380ae53 100644 --- a/src/libgit2/repository.h +++ b/src/libgit2/repository.h @@ -153,6 +153,7 @@ struct git_repository { unsigned is_bare:1; unsigned is_worktree:1; + git_oid_t oid_type; unsigned int lru_counter; @@ -256,4 +257,12 @@ int git_repository__extensions(char ***out, size_t *out_len); int git_repository__set_extensions(const char **extensions, size_t len); void git_repository__free_extensions(void); +/* + * Set the object format (OID type) for a repository; this will set + * both the configuration and the internal value for the oid type. + */ +int git_repository__set_objectformat( + git_repository *repo, + git_oid_t oid_type); + #endif diff --git a/src/libgit2/revparse.c b/src/libgit2/revparse.c index 780f1a0cb..964afe378 100644 --- a/src/libgit2/revparse.c +++ b/src/libgit2/revparse.c @@ -15,21 +15,28 @@ #include "git2.h" -static int maybe_sha_or_abbrev(git_object **out, git_repository *repo, const char *spec, size_t speclen) +static int maybe_sha_or_abbrev( + git_object **out, + git_repository *repo, + const char *spec, + size_t speclen) { git_oid oid; - if (git_oid__fromstrn(&oid, spec, speclen, GIT_OID_SHA1) < 0) + if (git_oid__fromstrn(&oid, spec, speclen, repo->oid_type) < 0) return GIT_ENOTFOUND; return git_object_lookup_prefix(out, repo, &oid, speclen, GIT_OBJECT_ANY); } -static int maybe_sha(git_object **out, git_repository *repo, const char *spec) +static int maybe_sha( + git_object **out, + git_repository *repo, + const char *spec) { size_t speclen = strlen(spec); - if (speclen != GIT_OID_SHA1_HEXSIZE) + if (speclen != git_oid_hexsize(repo->oid_type)) return GIT_ENOTFOUND; return maybe_sha_or_abbrev(out, repo, spec, speclen); @@ -110,8 +117,8 @@ static int revparse_lookup_object( if (error != GIT_ENOTFOUND) return error; - if ((strlen(spec) < GIT_OID_SHA1_HEXSIZE) && - ((error = maybe_abbrev(object_out, repo, spec)) != GIT_ENOTFOUND)) + if ((strlen(spec) < git_oid_hexsize(repo->oid_type)) && + ((error = maybe_abbrev(object_out, repo, spec)) != GIT_ENOTFOUND)) return error; if ((error = maybe_describe(object_out, repo, spec)) != GIT_ENOTFOUND) diff --git a/src/libgit2/streams/socket.c b/src/libgit2/streams/socket.c index 9415fe892..cbf805879 100644 --- a/src/libgit2/streams/socket.c +++ b/src/libgit2/streams/socket.c @@ -135,9 +135,11 @@ static ssize_t socket_write(git_stream *stream, const char *data, size_t len, in git_socket_stream *st = (git_socket_stream *) stream; ssize_t written; + assert(flags == 0); + errno = 0; - if ((written = p_send(st->s, data, len, flags)) < 0) { + if ((written = p_send(st->s, data, len, 0)) < 0) { net_set_error("error sending data"); return -1; } diff --git a/src/libgit2/tag.c b/src/libgit2/tag.c index 0a90e393c..562ec13ea 100644 --- a/src/libgit2/tag.c +++ b/src/libgit2/tag.c @@ -65,7 +65,11 @@ static int tag_error(const char *str) return GIT_EINVALID; } -static int tag_parse(git_tag *tag, const char *buffer, const char *buffer_end) +static int tag_parse( + git_tag *tag, + const char *buffer, + const char *buffer_end, + git_oid_t oid_type) { static const char *tag_types[] = { NULL, "commit\n", "tree\n", "blob\n", "tag\n" @@ -76,7 +80,7 @@ static int tag_parse(git_tag *tag, const char *buffer, const char *buffer_end) int error; if (git_object__parse_oid_header(&tag->target, - &buffer, buffer_end, "object ", GIT_OID_SHA1) < 0) + &buffer, buffer_end, "object ", oid_type) < 0) return tag_error("object field invalid"); if (buffer + 5 >= buffer_end) @@ -161,18 +165,25 @@ static int tag_parse(git_tag *tag, const char *buffer, const char *buffer_end) return 0; } -int git_tag__parse_raw(void *_tag, const char *data, size_t size) +int git_tag__parse_raw( + void *_tag, + const char *data, + size_t size, + git_oid_t oid_type) { - return tag_parse(_tag, data, data + size); + return tag_parse(_tag, data, data + size, oid_type); } -int git_tag__parse(void *_tag, git_odb_object *odb_obj) +int git_tag__parse( + void *_tag, + git_odb_object *odb_obj, + git_oid_t oid_type) { git_tag *tag = _tag; const char *buffer = git_odb_object_data(odb_obj); const char *buffer_end = buffer + git_odb_object_size(odb_obj); - return tag_parse(tag, buffer, buffer_end); + return tag_parse(tag, buffer, buffer_end, oid_type); } static int retrieve_tag_reference( @@ -374,7 +385,7 @@ int git_tag_create_from_buffer(git_oid *oid, git_repository *repo, const char *b return -1; /* validate the buffer */ - if (tag_parse(&tag, buffer, buffer + strlen(buffer)) < 0) + if (tag_parse(&tag, buffer, buffer + strlen(buffer), repo->oid_type) < 0) return -1; /* validate the target */ diff --git a/src/libgit2/tag.h b/src/libgit2/tag.h index 76ae1508e..fdaaa463c 100644 --- a/src/libgit2/tag.h +++ b/src/libgit2/tag.h @@ -25,7 +25,7 @@ struct git_tag { }; void git_tag__free(void *tag); -int git_tag__parse(void *tag, git_odb_object *obj); -int git_tag__parse_raw(void *tag, const char *data, size_t size); +int git_tag__parse(void *tag, git_odb_object *obj, git_oid_t oid_type); +int git_tag__parse_raw(void *tag, const char *data, size_t size, git_oid_t oid_type); #endif diff --git a/src/libgit2/transports/local.c b/src/libgit2/transports/local.c index 6c754a034..4d86f1713 100644 --- a/src/libgit2/transports/local.c +++ b/src/libgit2/transports/local.c @@ -266,6 +266,17 @@ static int local_capabilities(unsigned int *capabilities, git_transport *transpo return 0; } +#ifdef GIT_EXPERIMENTAL_SHA256 +static int local_oid_type(git_oid_t *out, git_transport *transport) +{ + transport_local *t = (transport_local *)transport; + + *out = t->repo->oid_type; + + return 0; +} +#endif + static int local_ls(const git_remote_head ***out, size_t *size, git_transport *transport) { transport_local *t = (transport_local *)transport; @@ -732,6 +743,9 @@ int git_transport_local(git_transport **out, git_remote *owner, void *param) t->parent.connect = local_connect; t->parent.set_connect_opts = local_set_connect_opts; t->parent.capabilities = local_capabilities; +#ifdef GIT_EXPERIMENTAL_SHA256 + t->parent.oid_type = local_oid_type; +#endif t->parent.negotiate_fetch = local_negotiate_fetch; t->parent.download_pack = local_download_pack; t->parent.push = local_push; diff --git a/src/libgit2/transports/smart.c b/src/libgit2/transports/smart.c index 7f57dba2a..c3a764bd3 100644 --- a/src/libgit2/transports/smart.c +++ b/src/libgit2/transports/smart.c @@ -54,6 +54,12 @@ GIT_INLINE(int) git_smart__reset_stream(transport_smart *t, bool close_subtransp return -1; } + git__free(t->caps.object_format); + t->caps.object_format = NULL; + + git__free(t->caps.agent); + t->caps.agent = NULL; + return 0; } @@ -242,6 +248,30 @@ static int git_smart__capabilities(unsigned int *capabilities, git_transport *tr return 0; } +#ifdef GIT_EXPERIMENTAL_SHA256 +static int git_smart__oid_type(git_oid_t *out, git_transport *transport) +{ + transport_smart *t = GIT_CONTAINER_OF(transport, transport_smart, parent); + + *out = 0; + + if (t->caps.object_format == NULL) { + *out = GIT_OID_DEFAULT; + } else { + *out = git_oid_type_fromstr(t->caps.object_format); + + if (!*out) { + git_error_set(GIT_ERROR_INVALID, + "unknown object format '%s'", + t->caps.object_format); + return -1; + } + } + + return 0; +} +#endif + static int git_smart__ls(const git_remote_head ***out, size_t *size, git_transport *transport) { transport_smart *t = GIT_CONTAINER_OF(transport, transport_smart, parent); @@ -386,6 +416,8 @@ static void git_smart__free(git_transport *transport) git_remote_connect_options_dispose(&t->connect_opts); + git__free(t->caps.object_format); + git__free(t->caps.agent); git__free(t); } @@ -452,6 +484,9 @@ int git_transport_smart(git_transport **out, git_remote *owner, void *param) t->parent.connect = git_smart__connect; t->parent.set_connect_opts = git_smart__set_connect_opts; t->parent.capabilities = git_smart__capabilities; +#ifdef GIT_EXPERIMENTAL_SHA256 + t->parent.oid_type = git_smart__oid_type; +#endif t->parent.close = git_smart__close; t->parent.free = git_smart__free; t->parent.negotiate_fetch = git_smart__negotiate_fetch; diff --git a/src/libgit2/transports/smart.h b/src/libgit2/transports/smart.h index 9323d6c44..d71160d8e 100644 --- a/src/libgit2/transports/smart.h +++ b/src/libgit2/transports/smart.h @@ -32,6 +32,8 @@ #define GIT_CAP_SYMREF "symref" #define GIT_CAP_WANT_TIP_SHA1 "allow-tip-sha1-in-want" #define GIT_CAP_WANT_REACHABLE_SHA1 "allow-reachable-sha1-in-want" +#define GIT_CAP_OBJECT_FORMAT "object-format=" +#define GIT_CAP_AGENT "agent=" extern bool git_smart__ofs_delta_enabled; @@ -133,6 +135,8 @@ typedef struct transport_smart_caps { thin_pack:1, want_tip_sha1:1, want_reachable_sha1:1; + char *object_format; + char *agent; } transport_smart_caps; typedef int (*packetsize_cb)(size_t received, void *payload); @@ -182,7 +186,12 @@ int git_smart__get_push_stream(transport_smart *t, git_smart_subtransport_stream int git_smart__update_heads(transport_smart *t, git_vector *symrefs); /* smart_pkt.c */ -int git_pkt_parse_line(git_pkt **head, const char **endptr, const char *line, size_t linelen); +typedef struct { + git_oid_t oid_type; + int seen_capabilities: 1; +} git_pkt_parse_data; + +int git_pkt_parse_line(git_pkt **head, const char **endptr, const char *line, size_t linelen, git_pkt_parse_data *data); int git_pkt_buffer_flush(git_str *buf); int git_pkt_send_flush(GIT_SOCKET s); int git_pkt_buffer_done(git_str *buf); diff --git a/src/libgit2/transports/smart_pkt.c b/src/libgit2/transports/smart_pkt.c index e679819fa..5fce42175 100644 --- a/src/libgit2/transports/smart_pkt.c +++ b/src/libgit2/transports/smart_pkt.c @@ -21,11 +21,14 @@ #include <ctype.h> -#define PKT_LEN_SIZE 4 -static const char pkt_done_str[] = "0009done\n"; -static const char pkt_flush_str[] = "0000"; -static const char pkt_have_prefix[] = "0032have "; -static const char pkt_want_prefix[] = "0032want "; +#define PKT_DONE_STR "0009done\n" +#define PKT_FLUSH_STR "0000" +#define PKT_HAVE_PREFIX "have " +#define PKT_WANT_PREFIX "want " + +#define PKT_LEN_SIZE 4 +#define PKT_MAX_SIZE 0xffff +#define PKT_MAX_WANTLEN (PKT_LEN_SIZE + CONST_STRLEN(PKT_WANT_PREFIX) + GIT_OID_MAX_HEXSIZE + 1) static int flush_pkt(git_pkt **out) { @@ -212,26 +215,87 @@ static int sideband_error_pkt(git_pkt **out, const char *line, size_t len) return 0; } +static int set_data( + git_pkt_parse_data *data, + const char *line, + size_t len) +{ + const char *caps, *format_str = NULL, *eos; + size_t format_len; + git_oid_t remote_oid_type; + + GIT_ASSERT_ARG(data); + + if ((caps = memchr(line, '\0', len)) != NULL) { + caps++; + + if (strncmp(caps, "object-format=", CONST_STRLEN("object-format=")) == 0) + format_str = caps + CONST_STRLEN("object-format="); + else if ((format_str = strstr(caps, " object-format=")) != NULL) + format_str += CONST_STRLEN(" object-format="); + } + + if (format_str) { + if ((eos = strchr(format_str, ' ')) == NULL) + eos = strchr(format_str, '\0'); + + GIT_ASSERT(eos); + + format_len = eos - format_str; + + if ((remote_oid_type = git_oid_type_fromstrn(format_str, format_len)) == 0) { + git_error_set(GIT_ERROR_INVALID, "unknown remote object format '%.*s'", (int)format_len, format_str); + return -1; + } + } else { + remote_oid_type = GIT_OID_SHA1; + } + + if (!data->oid_type) { + data->oid_type = remote_oid_type; + } else if (data->oid_type != remote_oid_type) { + git_error_set(GIT_ERROR_INVALID, + "the local object format '%s' does not match the remote object format '%s'", + git_oid_type_name(data->oid_type), + git_oid_type_name(remote_oid_type)); + return -1; + } + + return 0; +} + /* * Parse an other-ref line. */ -static int ref_pkt(git_pkt **out, const char *line, size_t len) +static int ref_pkt( + git_pkt **out, + const char *line, + size_t len, + git_pkt_parse_data *data) { git_pkt_ref *pkt; - size_t alloclen; + size_t alloclen, oid_hexsize; pkt = git__calloc(1, sizeof(git_pkt_ref)); GIT_ERROR_CHECK_ALLOC(pkt); pkt->type = GIT_PKT_REF; - if (len < GIT_OID_SHA1_HEXSIZE || - git_oid__fromstr(&pkt->head.oid, line, GIT_OID_SHA1) < 0) + /* Determine OID type from capabilities */ + if (!data->seen_capabilities && set_data(data, line, len) < 0) + return -1; + + GIT_ASSERT(data->oid_type); + oid_hexsize = git_oid_hexsize(data->oid_type); + + if (len < oid_hexsize || + git_oid__fromstr(&pkt->head.oid, line, data->oid_type) < 0) goto out_err; - line += GIT_OID_SHA1_HEXSIZE; - len -= GIT_OID_SHA1_HEXSIZE; + line += oid_hexsize; + len -= oid_hexsize; if (git__prefixncmp(line, len, " ")) goto out_err; + line++; len--; @@ -248,8 +312,14 @@ static int ref_pkt(git_pkt **out, const char *line, size_t len) memcpy(pkt->head.name, line, len); pkt->head.name[len] = '\0'; - if (strlen(pkt->head.name) < len) - pkt->capabilities = strchr(pkt->head.name, '\0') + 1; + if (strlen(pkt->head.name) < len) { + if (!data->seen_capabilities) + pkt->capabilities = strchr(pkt->head.name, '\0') + 1; + else + goto out_err; + } + + data->seen_capabilities = 1; *out = (git_pkt *)pkt; return 0; @@ -418,7 +488,11 @@ static int parse_len(size_t *out, const char *line, size_t linelen) */ int git_pkt_parse_line( - git_pkt **pkt, const char **endptr, const char *line, size_t linelen) + git_pkt **pkt, + const char **endptr, + const char *line, + size_t linelen, + git_pkt_parse_data *data) { int error; size_t len; @@ -493,7 +567,7 @@ int git_pkt_parse_line( else if (!git__prefixncmp(line, len, "unpack")) error = unpack_pkt(pkt, line, len); else - error = ref_pkt(pkt, line, len); + error = ref_pkt(pkt, line, len, data); *endptr = line + len; @@ -527,14 +601,21 @@ void git_pkt_free(git_pkt *pkt) int git_pkt_buffer_flush(git_str *buf) { - return git_str_put(buf, pkt_flush_str, strlen(pkt_flush_str)); + return git_str_put(buf, PKT_FLUSH_STR, CONST_STRLEN(PKT_FLUSH_STR)); } -static int buffer_want_with_caps(const git_remote_head *head, transport_smart_caps *caps, git_str *buf) +static int buffer_want_with_caps( + const git_remote_head *head, + transport_smart_caps *caps, + git_oid_t oid_type, + git_str *buf) { git_str str = GIT_STR_INIT; - char oid[GIT_OID_SHA1_HEXSIZE +1] = {0}; - size_t len; + char oid[GIT_OID_MAX_HEXSIZE]; + size_t oid_hexsize, len; + + oid_hexsize = git_oid_hexsize(oid_type); + git_oid_fmt(oid, &head->oid); /* Prefer multi_ack_detailed */ if (caps->multi_ack_detailed) @@ -560,19 +641,20 @@ static int buffer_want_with_caps(const git_remote_head *head, transport_smart_ca if (git_str_oom(&str)) return -1; - len = strlen("XXXXwant ") + GIT_OID_SHA1_HEXSIZE + 1 /* NUL */ + - git_str_len(&str) + 1 /* LF */; - - if (len > 0xffff) { + if (str.size > (PKT_MAX_SIZE - (PKT_MAX_WANTLEN + 1))) { git_error_set(GIT_ERROR_NET, - "tried to produce packet with invalid length %" PRIuZ, len); + "tried to produce packet with invalid caps length %" PRIuZ, str.size); return -1; } + len = PKT_LEN_SIZE + CONST_STRLEN(PKT_WANT_PREFIX) + + oid_hexsize + 1 /* NUL */ + + git_str_len(&str) + 1 /* LF */; + git_str_grow_by(buf, len); - git_oid_fmt(oid, &head->oid); git_str_printf(buf, - "%04xwant %s %s\n", (unsigned int)len, oid, git_str_cstr(&str)); + "%04x%s%.*s %s\n", (unsigned int)len, PKT_WANT_PREFIX, + (int)oid_hexsize, oid, git_str_cstr(&str)); git_str_dispose(&str); GIT_ERROR_CHECK_ALLOC_STR(buf); @@ -591,8 +673,21 @@ int git_pkt_buffer_wants( transport_smart_caps *caps, git_str *buf) { - size_t i = 0; const git_remote_head *head; + char oid[GIT_OID_MAX_HEXSIZE]; + git_oid_t oid_type; + size_t oid_hexsize, want_len, i = 0; + +#ifdef GIT_EXPERIMENTAL_SHA256 + oid_type = count > 0 ? refs[0]->oid.type : GIT_OID_SHA1; +#else + oid_type = GIT_OID_SHA1; +#endif + + oid_hexsize = git_oid_hexsize(oid_type); + + want_len = PKT_LEN_SIZE + CONST_STRLEN(PKT_WANT_PREFIX) + + oid_hexsize + 1 /* LF */; if (caps->common) { for (; i < count; ++i) { @@ -601,23 +696,24 @@ int git_pkt_buffer_wants( break; } - if (buffer_want_with_caps(refs[i], caps, buf) < 0) + if (buffer_want_with_caps(refs[i], caps, oid_type, buf) < 0) return -1; i++; } for (; i < count; ++i) { - char oid[GIT_OID_SHA1_HEXSIZE]; - head = refs[i]; + if (head->local) continue; git_oid_fmt(oid, &head->oid); - git_str_put(buf, pkt_want_prefix, strlen(pkt_want_prefix)); - git_str_put(buf, oid, GIT_OID_SHA1_HEXSIZE); - git_str_putc(buf, '\n'); + + git_str_printf(buf, "%04x%s%.*s\n", + (unsigned int)want_len, PKT_WANT_PREFIX, + (int)oid_hexsize, oid); + if (git_str_oom(buf)) return -1; } @@ -627,14 +723,27 @@ int git_pkt_buffer_wants( int git_pkt_buffer_have(git_oid *oid, git_str *buf) { - char oidhex[GIT_OID_SHA1_HEXSIZE + 1]; - - memset(oidhex, 0x0, sizeof(oidhex)); - git_oid_fmt(oidhex, oid); - return git_str_printf(buf, "%s%s\n", pkt_have_prefix, oidhex); + char oid_str[GIT_OID_MAX_HEXSIZE]; + git_oid_t oid_type; + size_t oid_hexsize, have_len; + +#ifdef GIT_EXPERIMENTAL_SHA256 + oid_type = oid->type; +#else + oid_type = GIT_OID_SHA1; +#endif + + oid_hexsize = git_oid_hexsize(oid_type); + have_len = PKT_LEN_SIZE + CONST_STRLEN(PKT_HAVE_PREFIX) + + oid_hexsize + 1 /* LF */; + + git_oid_fmt(oid_str, oid); + return git_str_printf(buf, "%04x%s%.*s\n", + (unsigned int)have_len, PKT_HAVE_PREFIX, + (int)oid_hexsize, oid_str); } int git_pkt_buffer_done(git_str *buf) { - return git_str_puts(buf, pkt_done_str); + return git_str_put(buf, PKT_DONE_STR, CONST_STRLEN(PKT_DONE_STR)); } diff --git a/src/libgit2/transports/smart_protocol.c b/src/libgit2/transports/smart_protocol.c index 88c208fb5..0d47acafe 100644 --- a/src/libgit2/transports/smart_protocol.c +++ b/src/libgit2/transports/smart_protocol.c @@ -32,6 +32,7 @@ int git_smart__store_refs(transport_smart *t, int flushes) int error, flush = 0, recvd; const char *line_end = NULL; git_pkt *pkt = NULL; + git_pkt_parse_data pkt_parse_data = { 0 }; size_t i; /* Clear existing refs in case git_remote_connect() is called again @@ -45,7 +46,7 @@ int git_smart__store_refs(transport_smart *t, int flushes) do { if (buf->offset > 0) - error = git_pkt_parse_line(&pkt, &line_end, buf->data, buf->offset); + error = git_pkt_parse_line(&pkt, &line_end, buf->data, buf->offset, &pkt_parse_data); else error = GIT_EBUFS; @@ -133,9 +134,12 @@ on_invalid: return -1; } -int git_smart__detect_caps(git_pkt_ref *pkt, transport_smart_caps *caps, git_vector *symrefs) +int git_smart__detect_caps( + git_pkt_ref *pkt, + transport_smart_caps *caps, + git_vector *symrefs) { - const char *ptr; + const char *ptr, *start; /* No refs or capabilities, odd but not a problem */ if (pkt == NULL || pkt->capabilities == NULL) @@ -217,6 +221,28 @@ int git_smart__detect_caps(git_pkt_ref *pkt, transport_smart_caps *caps, git_vec continue; } + if (!git__prefixcmp(ptr, GIT_CAP_OBJECT_FORMAT)) { + ptr += strlen(GIT_CAP_OBJECT_FORMAT); + + start = ptr; + ptr = strchr(ptr, ' '); + + if ((caps->object_format = git__strndup(start, (ptr - start))) == NULL) + return -1; + continue; + } + + if (!git__prefixcmp(ptr, GIT_CAP_AGENT)) { + ptr += strlen(GIT_CAP_AGENT); + + start = ptr; + ptr = strchr(ptr, ' '); + + if ((caps->agent = git__strndup(start, (ptr - start))) == NULL) + return -1; + continue; + } + /* We don't know this capability, so skip it */ ptr = strchr(ptr, ' '); } @@ -228,11 +254,12 @@ static int recv_pkt(git_pkt **out_pkt, git_pkt_type *out_type, gitno_buffer *buf { const char *ptr = buf->data, *line_end = ptr; git_pkt *pkt = NULL; + git_pkt_parse_data pkt_parse_data = { 0 }; int error = 0, ret; do { if (buf->offset > 0) - error = git_pkt_parse_line(&pkt, &line_end, ptr, buf->offset); + error = git_pkt_parse_line(&pkt, &line_end, ptr, buf->offset, &pkt_parse_data); else error = GIT_EBUFS; @@ -723,6 +750,7 @@ static int add_push_report_pkt(git_push *push, git_pkt *pkt) static int add_push_report_sideband_pkt(git_push *push, git_pkt_data *data_pkt, git_str *data_pkt_buf) { git_pkt *pkt; + git_pkt_parse_data pkt_parse_data = { 0 }; const char *line, *line_end = NULL; size_t line_len; int error; @@ -741,7 +769,7 @@ static int add_push_report_sideband_pkt(git_push *push, git_pkt_data *data_pkt, } while (line_len > 0) { - error = git_pkt_parse_line(&pkt, &line_end, line, line_len); + error = git_pkt_parse_line(&pkt, &line_end, line, line_len, &pkt_parse_data); if (error == GIT_EBUFS) { /* Buffer the data when the inner packet is split @@ -777,6 +805,7 @@ done: static int parse_report(transport_smart *transport, git_push *push) { git_pkt *pkt = NULL; + git_pkt_parse_data pkt_parse_data = { 0 }; const char *line_end = NULL; gitno_buffer *buf = &transport->buffer; int error, recvd; @@ -785,7 +814,8 @@ static int parse_report(transport_smart *transport, git_push *push) for (;;) { if (buf->offset > 0) error = git_pkt_parse_line(&pkt, &line_end, - buf->data, buf->offset); + buf->data, buf->offset, + &pkt_parse_data); else error = GIT_EBUFS; diff --git a/src/libgit2/tree.c b/src/libgit2/tree.c index 9a43d585c..9293d9422 100644 --- a/src/libgit2/tree.c +++ b/src/libgit2/tree.c @@ -85,11 +85,17 @@ static git_tree_entry *alloc_entry(const char *filename, size_t filename_len, co char *filename_ptr; size_t tree_len; +#ifdef GIT_EXPERIMENTAL_SHA256 + size_t oid_size = git_oid_size(id->type); +#else + size_t oid_size = GIT_OID_SHA1_SIZE; +#endif + TREE_ENTRY_CHECK_NAMELEN(filename_len); if (GIT_ADD_SIZET_OVERFLOW(&tree_len, sizeof(git_tree_entry), filename_len) || GIT_ADD_SIZET_OVERFLOW(&tree_len, tree_len, 1) || - GIT_ADD_SIZET_OVERFLOW(&tree_len, tree_len, GIT_OID_SHA1_SIZE)) + GIT_ADD_SIZET_OVERFLOW(&tree_len, tree_len, oid_size)) return NULL; entry = git__calloc(1, tree_len); @@ -383,11 +389,12 @@ static int parse_mode(uint16_t *mode_out, const char *buffer, size_t buffer_len, return 0; } -int git_tree__parse_raw(void *_tree, const char *data, size_t size) +int git_tree__parse_raw(void *_tree, const char *data, size_t size, git_oid_t oid_type) { git_tree *tree = _tree; const char *buffer; const char *buffer_end; + const long oid_size = (long)git_oid_size(oid_type); buffer = data; buffer_end = buffer + size; @@ -414,35 +421,33 @@ int git_tree__parse_raw(void *_tree, const char *data, size_t size) if ((filename_len = nul - buffer) == 0 || filename_len > UINT16_MAX) return tree_parse_error("failed to parse tree: can't parse filename", NULL); - if ((buffer_end - (nul + 1)) < GIT_OID_SHA1_SIZE) + if ((buffer_end - (nul + 1)) < (long)oid_size) return tree_parse_error("failed to parse tree: can't parse OID", NULL); /* Allocate the entry */ - { - entry = git_array_alloc(tree->entries); - GIT_ERROR_CHECK_ALLOC(entry); - - entry->attr = attr; - entry->filename_len = (uint16_t)filename_len; - entry->filename = buffer; - git_oid__fromraw(&entry->oid, ((unsigned char *) buffer + filename_len + 1), GIT_OID_SHA1); - } + entry = git_array_alloc(tree->entries); + GIT_ERROR_CHECK_ALLOC(entry); + entry->attr = attr; + entry->filename_len = (uint16_t)filename_len; + entry->filename = buffer; buffer += filename_len + 1; - buffer += GIT_OID_SHA1_SIZE; + + git_oid__fromraw(&entry->oid, (unsigned char *)buffer, oid_type); + buffer += oid_size; } return 0; } -int git_tree__parse(void *_tree, git_odb_object *odb_obj) +int git_tree__parse(void *_tree, git_odb_object *odb_obj, git_oid_t oid_type) { git_tree *tree = _tree; const char *data = git_odb_object_data(odb_obj); size_t size = git_odb_object_size(odb_obj); int error; - if ((error = git_tree__parse_raw(tree, data, size)) < 0 || + if ((error = git_tree__parse_raw(tree, data, size, oid_type)) < 0 || (error = git_odb_object_dup(&tree->odb_obj, odb_obj)) < 0) return error; @@ -506,6 +511,7 @@ static int git_treebuilder__write_with_buffer( git_odb *odb; git_tree_entry *entry; git_vector entries = GIT_VECTOR_INIT; + size_t oid_size = git_oid_size(bld->repo->oid_type); git_str_clear(buf); @@ -529,7 +535,7 @@ static int git_treebuilder__write_with_buffer( git_str_printf(buf, "%o ", entry->attr); git_str_put(buf, entry->filename, entry->filename_len + 1); - git_str_put(buf, (char *)entry->oid.id, GIT_OID_SHA1_SIZE); + git_str_put(buf, (char *)entry->oid.id, oid_size); if (git_str_oom(buf)) { error = -1; diff --git a/src/libgit2/tree.h b/src/libgit2/tree.h index 0dd963ff2..5088450ab 100644 --- a/src/libgit2/tree.h +++ b/src/libgit2/tree.h @@ -41,8 +41,8 @@ GIT_INLINE(bool) git_tree_entry__is_tree(const struct git_tree_entry *e) } void git_tree__free(void *tree); -int git_tree__parse(void *tree, git_odb_object *obj); -int git_tree__parse_raw(void *_tree, const char *data, size_t size); +int git_tree__parse(void *tree, git_odb_object *obj, git_oid_t oid_type); +int git_tree__parse_raw(void *_tree, const char *data, size_t size, git_oid_t oid_type); /** * Write a tree to the given repository |
