From 26022f0719f652ae8311abea6f6de92bd4a75a87 Mon Sep 17 00:00:00 2001 From: Vicent Marti Date: Sat, 5 Mar 2011 23:54:49 +0200 Subject: Add `git_oid_shorten` (unique OID minimzer) Set of methods to find the minimal-length to uniquely identify every OID in a list. Useful for GUI applications, commit logs and so on. Includes stress test. Signed-off-by: Vicent Marti --- src/oid.c | 178 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 178 insertions(+) (limited to 'src') diff --git a/src/oid.c b/src/oid.c index 698d0f927..81b7d6005 100644 --- a/src/oid.c +++ b/src/oid.c @@ -27,6 +27,7 @@ #include "git2/oid.h" #include "repository.h" #include +#include static signed char from_hex[] = { -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, /* 00 */ @@ -166,3 +167,180 @@ int git_oid_cmp(const git_oid *a, const git_oid *b) { return memcmp(a->id, b->id, sizeof(a->id)); } + + +typedef short node_index; + +typedef union { + const char *tail; + node_index children[16]; +} trie_node; + +struct git_oid_shorten { + trie_node *nodes; + size_t node_count, size; + int min_length, full; +}; + +static int resize_trie(git_oid_shorten *self, size_t new_size) +{ + self->nodes = realloc(self->nodes, new_size * sizeof(trie_node)); + if (self->nodes == NULL) + return GIT_ENOMEM; + + if (new_size > self->size) { + memset(&self->nodes[self->size], 0x0, (new_size - self->size) * sizeof(trie_node)); + } + + self->size = new_size; + return GIT_SUCCESS; +} + +static trie_node *push_leaf(git_oid_shorten *os, node_index idx, int push_at, const char *oid) +{ + trie_node *node, *leaf; + node_index idx_leaf; + + if (os->node_count >= os->size) { + if (resize_trie(os, os->size * 2) < GIT_SUCCESS) + return NULL; + } + + idx_leaf = (node_index)os->node_count++; + + if (os->node_count == SHRT_MAX) + os->full = 1; + + node = &os->nodes[idx]; + node->children[push_at] = -idx_leaf; + + leaf = &os->nodes[idx_leaf]; + leaf->tail = oid; + + return node; +} + +git_oid_shorten *git_oid_shorten_new(size_t min_length) +{ + git_oid_shorten *os; + + os = git__malloc(sizeof(git_oid_shorten)); + if (os == NULL) + return NULL; + + memset(os, 0x0, sizeof(git_oid_shorten)); + + if (resize_trie(os, 16) < GIT_SUCCESS) { + free(os); + return NULL; + } + + os->node_count = 1; + os->min_length = min_length; + + return os; +} + +void git_oid_shorten_free(git_oid_shorten *os) +{ + free(os->nodes); + free(os); +} + + +/* + * What wizardry is this? + * + * This is just a memory-optimized trie: basically a very fancy + * 16-ary tree, which is used to store the prefixes of the OID + * strings. + * + * Read more: http://en.wikipedia.org/wiki/Trie + * + * Magic that happens in this method: + * + * - Each node in the trie is an union, so it can work both as + * a normal node, or as a leaf. + * + * - Each normal node points to 16 children (one for each possible + * character in the oid). This is *not* stored in an array of + * pointers, because in a 64-bit arch this would be sucking + * 16*sizeof(void*) = 128 bytes of memory per node, which is fucking + * insane. What we do is store Node Indexes, and use these indexes + * to look up each node in the om->index array. These indexes are + * signed shorts, so this limits the amount of unique OIDs that + * fit in the structure to about 20000 (assuming a more or less uniform + * distribution). + * + * - All the nodes in om->index array are stored contiguously in + * memory, and each of them is 32 bytes, so we fit 2x nodes per + * cache line. Convenient for speed. + * + * - To differentiate the leafs from the normal nodes, we store all + * the indexes towards a leaf as a negative index (indexes to normal + * nodes are positives). When we find that one of the children for + * a node has a negative value, that means it's going to be a leaf. + * This reduces the amount of indexes we have by two, but also reduces + * the size of each node by 1-4 bytes (the amount we would need to + * add a `is_leaf` field): this is good because it allows the nodes + * to fit cleanly in cache lines. + * + * - Once we reach an empty children, instead of continuing to insert + * new nodes for each remaining character of the OID, we store a pointer + * to the tail in the leaf; if the leaf is reached again, we turn it + * into a normal node and use the tail to create a new leaf. + * + * This is a pretty good balance between performance and memory usage. + */ +int git_oid_shorten_add(git_oid_shorten *os, const char *text_oid) +{ + int i, is_leaf; + node_index idx; + + if (os->full) + return GIT_ENOMEM; + + idx = 0; + is_leaf = 0; + + for (i = 0; i < GIT_OID_HEXSZ; ++i) { + int c = from_hex[(int)text_oid[i]]; + trie_node *node; + + if (c == -1) + return GIT_ENOTOID; + + node = &os->nodes[idx]; + + if (is_leaf) { + const char *tail; + + tail = node->tail; + node->tail = NULL; + + node = push_leaf(os, idx, from_hex[(int)tail[0]], &tail[1]); + if (node == NULL) + return GIT_ENOMEM; + } + + if (node->children[c] == 0) { + if (push_leaf(os, idx, c, &text_oid[i + 1]) == NULL) + return GIT_ENOMEM; + break; + } + + idx = node->children[c]; + is_leaf = 0; + + if (idx < 0) { + node->children[c] = idx = -idx; + is_leaf = 1; + } + } + + if (++i > os->min_length) + os->min_length = i; + + return os->min_length; +} + -- cgit v1.2.1 From 71db842fac3ba8582255bc5b61361ddef08ef105 Mon Sep 17 00:00:00 2001 From: Vicent Marti Date: Tue, 8 Mar 2011 14:57:03 +0200 Subject: Rewrite the Revision Walker The new revision walker uses an internal Commit object storage system, custom memory allocator and much improved topological and time sorting algorithms. It's about 20x times faster than the previous implementation when browsing big repositories. The following external API calls have changed: `git_revwalk_next` returns an OID instead of a full commit object. The initial call to `git_revwalk_next` is no longer blocking when iterating through a repo with a time-sorting mode. Iterating with Topological or inverted modes still makes the initial call blocking to preprocess the commit list, but this block should be mostly unnoticeable on most repositories (topological preprocessing times at 0.3s on the git.git repo). `git_revwalk_push` and `git_revwalk_hide` now take an OID instead of a full commit object. --- src/pqueue.c | 153 ++++++++++++++ src/pqueue.h | 92 +++++++++ src/repository.c | 2 +- src/revwalk.c | 618 ++++++++++++++++++++++++++++++++----------------------- src/revwalk.h | 56 ----- src/util.h | 2 + 6 files changed, 611 insertions(+), 312 deletions(-) create mode 100644 src/pqueue.c create mode 100644 src/pqueue.h (limited to 'src') diff --git a/src/pqueue.c b/src/pqueue.c new file mode 100644 index 000000000..98152cb85 --- /dev/null +++ b/src/pqueue.c @@ -0,0 +1,153 @@ +/* + * BORING COPYRIGHT NOTICE: + * + * This file is a heavily modified version of the priority queue found + * in the Apache project and the libpqueue library. + * + * https://github.com/vy/libpqueue + * + * These are the original authors: + * + * Copyright 2010 Volkan Yazıcı + * Copyright 2006-2010 The Apache Software Foundation + * + * This file is licensed under the Apache 2.0 license, which + * supposedly makes it compatible with the GPLv2 that libgit2 uses. + * + * Check the Apache license at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * So much licensing trouble for a binary heap. Oh well. + */ + +#include "common.h" +#include "pqueue.h" + +#define left(i) ((i) << 1) +#define right(i) (((i) << 1) + 1) +#define parent(i) ((i) >> 1) + +int git_pqueue_init(git_pqueue *q, size_t n, git_pqueue_cmp cmppri) +{ + assert(q); + + /* Need to allocate n+1 elements since element 0 isn't used. */ + if ((q->d = malloc((n + 1) * sizeof(void *))) == NULL) + return GIT_ENOMEM; + + q->size = 1; + q->avail = q->step = (n + 1); /* see comment above about n+1 */ + q->cmppri = cmppri; + + return GIT_SUCCESS; +} + + +void git_pqueue_free(git_pqueue *q) +{ + free(q->d); + q->d = NULL; +} + + +size_t git_pqueue_size(git_pqueue *q) +{ + /* queue element 0 exists but doesn't count since it isn't used. */ + return (q->size - 1); +} + + +static void bubble_up(git_pqueue *q, size_t i) +{ + size_t parent_node; + void *moving_node = q->d[i]; + + for (parent_node = parent(i); + ((i > 1) && q->cmppri(q->d[parent_node], moving_node)); + i = parent_node, parent_node = parent(i)) { + q->d[i] = q->d[parent_node]; + } + + q->d[i] = moving_node; +} + + +static size_t maxchild(git_pqueue *q, size_t i) +{ + size_t child_node = left(i); + + if (child_node >= q->size) + return 0; + + if ((child_node + 1) < q->size && + q->cmppri(q->d[child_node], q->d[child_node + 1])) + child_node++; /* use right child instead of left */ + + return child_node; +} + + +static void percolate_down(git_pqueue *q, size_t i) +{ + size_t child_node; + void *moving_node = q->d[i]; + + while ((child_node = maxchild(q, i)) != 0 && + q->cmppri(moving_node, q->d[child_node])) { + q->d[i] = q->d[child_node]; + i = child_node; + } + + q->d[i] = moving_node; +} + + +int git_pqueue_insert(git_pqueue *q, void *d) +{ + void *tmp; + size_t i; + size_t newsize; + + if (!q) return 1; + + /* allocate more memory if necessary */ + if (q->size >= q->avail) { + newsize = q->size + q->step; + if ((tmp = realloc(q->d, sizeof(void *) * newsize)) == NULL) + return GIT_ENOMEM; + + q->d = tmp; + q->avail = newsize; + } + + /* insert item */ + i = q->size++; + q->d[i] = d; + bubble_up(q, i); + + return GIT_SUCCESS; +} + + +void *git_pqueue_pop(git_pqueue *q) +{ + void *head; + + if (!q || q->size == 1) + return NULL; + + head = q->d[1]; + q->d[1] = q->d[--q->size]; + percolate_down(q, 1); + + return head; +} + + +void *git_pqueue_peek(git_pqueue *q) +{ + if (!q || q->size == 1) + return NULL; + return q->d[1]; +} diff --git a/src/pqueue.h b/src/pqueue.h new file mode 100644 index 000000000..6db74661d --- /dev/null +++ b/src/pqueue.h @@ -0,0 +1,92 @@ +/* + * BORING COPYRIGHT NOTICE: + * + * This file is a heavily modified version of the priority queue found + * in the Apache project and the libpqueue library. + * + * https://github.com/vy/libpqueue + * + * These are the original authors: + * + * Copyright 2010 Volkan Yazıcı + * Copyright 2006-2010 The Apache Software Foundation + * + * This file is licensed under the Apache 2.0 license, which + * supposedly makes it compatible with the GPLv2 that libgit2 uses. + * + * Check the Apache license at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * So much licensing trouble for a binary heap. Oh well. + */ + +#ifndef INCLUDE_pqueue_h__ +#define INCLUDE_pqueue_h__ + +/** callback functions to get/set/compare the priority of an element */ +typedef int (*git_pqueue_cmp)(void *a, void *b); + +/** the priority queue handle */ +typedef struct { + size_t size, avail, step; + git_pqueue_cmp cmppri; + void **d; +} git_pqueue; + + +/** + * initialize the queue + * + * @param n the initial estimate of the number of queue items for which memory + * should be preallocated + * @param cmppri the callback function to compare two nodes of the queue + * + * @Return the handle or NULL for insufficent memory + */ +int git_pqueue_init(git_pqueue *q, size_t n, git_pqueue_cmp cmppri); + + +/** + * free all memory used by the queue + * @param q the queue + */ +void git_pqueue_free(git_pqueue *q); + + +/** + * return the size of the queue. + * @param q the queue + */ +size_t git_pqueue_size(git_pqueue *q); + + +/** + * insert an item into the queue. + * @param q the queue + * @param d the item + * @return 0 on success + */ +int git_pqueue_insert(git_pqueue *q, void *d); + + +/** + * pop the highest-ranking item from the queue. + * @param p the queue + * @param d where to copy the entry to + * @return NULL on error, otherwise the entry + */ +void *git_pqueue_pop(git_pqueue *q); + + +/** + * access highest-ranking item without removing it. + * @param q the queue + * @param d the entry + * @return NULL on error, otherwise the entry + */ +void *git_pqueue_peek(git_pqueue *q); + +#endif /* PQUEUE_H */ +/** @} */ + diff --git a/src/repository.c b/src/repository.c index f2cb985af..c24560665 100644 --- a/src/repository.c +++ b/src/repository.c @@ -53,7 +53,7 @@ typedef struct { * Callbacks for the ODB cache, implemented * as a hash table */ -uint32_t object_table_hash(const void *key, int hash_id) +static uint32_t object_table_hash(const void *key, int hash_id) { uint32_t r; git_oid *id; diff --git a/src/revwalk.c b/src/revwalk.c index a1cd0ebb7..edafbe73d 100644 --- a/src/revwalk.c +++ b/src/revwalk.c @@ -27,22 +27,132 @@ #include "commit.h" #include "revwalk.h" #include "hashtable.h" +#include "pqueue.h" -uint32_t git_revwalk__commit_hash(const void *key, int hash_id) +typedef struct commit_object { + git_oid oid; + uint32_t time; + unsigned int seen:1, + uninteresting:1, + topo_delay:1, + parsed:1; + + unsigned short in_degree; + unsigned short out_degree; + + struct commit_object **parents; +} commit_object; + +typedef struct commit_list { + commit_object *item; + struct commit_list *next; +} commit_list; + +struct git_revwalk { + git_repository *repo; + + git_hashtable *commits; + git_vector pending; + + commit_list *iterator_topo; + commit_list *iterator_rand; + commit_list *iterator_reverse; + git_pqueue iterator_time; + + int (*get_next)(commit_object **, git_revwalk *); + int (*enqueue)(git_revwalk *, commit_object *); + + git_vector memory_alloc; + size_t chunk_size; + + unsigned walking:1; + unsigned int sorting; +}; + +commit_list *commit_list_insert(commit_object *item, commit_list **list_p) +{ + commit_list *new_list = git__malloc(sizeof(commit_list)); + new_list->item = item; + new_list->next = *list_p; + *list_p = new_list; + return new_list; +} + +void commit_list_free(commit_list *list) +{ + while (list) { + commit_list *temp = list; + list = temp->next; + free(temp); + } +} + +commit_object *commit_list_pop(commit_list **stack) +{ + commit_list *top = *stack; + commit_object *item = top ? top->item : NULL; + + if (top) { + *stack = top->next; + free(top); + } + return item; +} + +static int commit_time_cmp(void *a, void *b) +{ + commit_object *commit_a = (commit_object *)a; + commit_object *commit_b = (commit_object *)b; + + return (commit_a->time < commit_b->time); +} + +static uint32_t object_table_hash(const void *key, int hash_id) { uint32_t r; - git_commit *commit; + git_oid *id; - commit = (git_commit *)key; - memcpy(&r, commit->object.id.id + (hash_id * sizeof(uint32_t)), sizeof(r)); + id = (git_oid *)key; + memcpy(&r, id->id + (hash_id * sizeof(uint32_t)), sizeof(r)); return r; } -int git_revwalk__commit_keycmp(const void *key_a, const void *key_b) +#define COMMITS_PER_CHUNK 128 +#define CHUNK_STEP 64 +#define PARENTS_PER_COMMIT ((CHUNK_STEP - sizeof(commit_object)) / sizeof(commit_object *)) + +static int alloc_chunk(git_revwalk *walk) { - git_commit *a = (git_commit *)key_a; - git_commit *b = (git_commit *)key_b; - return git_oid_cmp(&a->object.id, &b->object.id); + void *chunk; + + chunk = git__calloc(COMMITS_PER_CHUNK, CHUNK_STEP); + if (chunk == NULL) + return GIT_ENOMEM; + + walk->chunk_size = 0; + return git_vector_insert(&walk->memory_alloc, chunk); +} + +static commit_object *alloc_commit(git_revwalk *walk) +{ + unsigned char *chunk; + + if (walk->chunk_size == COMMITS_PER_CHUNK) + alloc_chunk(walk); + + chunk = git_vector_get(&walk->memory_alloc, walk->memory_alloc.length - 1); + chunk += (walk->chunk_size * CHUNK_STEP); + walk->chunk_size++; + + return (commit_object *)chunk; +} + +static commit_object **alloc_parents(commit_object *commit, size_t n_parents) +{ + if (n_parents <= PARENTS_PER_COMMIT) + return (commit_object **)((unsigned char *)commit + sizeof(commit_object)); + + return git__malloc(n_parents * sizeof(commit_object *)); } int git_revwalk_new(git_revwalk **revwalk_out, git_repository *repo) @@ -56,14 +166,18 @@ int git_revwalk_new(git_revwalk **revwalk_out, git_repository *repo) memset(walk, 0x0, sizeof(git_revwalk)); walk->commits = git_hashtable_alloc(64, - git_revwalk__commit_hash, - git_revwalk__commit_keycmp); + object_table_hash, + (git_hash_keyeq_ptr)git_oid_cmp); if (walk->commits == NULL) { free(walk); return GIT_ENOMEM; } + git_vector_init(&walk->pending, 8, NULL); + git_vector_init(&walk->memory_alloc, 8, NULL); + alloc_chunk(walk); + walk->repo = repo; *revwalk_out = walk; @@ -72,11 +186,20 @@ int git_revwalk_new(git_revwalk **revwalk_out, git_repository *repo) void git_revwalk_free(git_revwalk *walk) { + unsigned int i; + if (walk == NULL) return; git_revwalk_reset(walk); git_hashtable_free(walk->commits); + git_vector_free(&walk->pending); + + for (i = 0; i < walk->memory_alloc.length; ++i) { + free(git_vector_get(&walk->memory_alloc, i)); + } + + git_vector_free(&walk->memory_alloc); free(walk); } @@ -98,364 +221,349 @@ int git_revwalk_sorting(git_revwalk *walk, unsigned int sort_mode) return GIT_SUCCESS; } -static git_revwalk_commit *commit_to_walkcommit(git_revwalk *walk, git_commit *commit_object) +static commit_object *commit_lookup(git_revwalk *walk, const git_oid *oid) { - git_revwalk_commit *commit; - - commit = (git_revwalk_commit *)git_hashtable_lookup(walk->commits, commit_object); + commit_object *commit; - if (commit != NULL) + if ((commit = git_hashtable_lookup(walk->commits, oid)) != NULL) return commit; - commit = git__malloc(sizeof(git_revwalk_commit)); + commit = alloc_commit(walk); if (commit == NULL) return NULL; - memset(commit, 0x0, sizeof(git_revwalk_commit)); - - commit->commit_object = commit_object; - GIT_OBJECT_INCREF(walk->repo, commit_object); + git_oid_cpy(&commit->oid, oid); - git_hashtable_insert(walk->commits, commit_object, commit); + if (git_hashtable_insert(walk->commits, &commit->oid, commit) < GIT_SUCCESS) { + free(commit); + return NULL; + } return commit; } -static git_revwalk_commit *insert_commit(git_revwalk *walk, git_commit *commit_object) +static int commit_quick_parse(git_revwalk *walk, commit_object *commit, git_rawobj *raw) { - git_revwalk_commit *commit; - unsigned int i; + const int parent_len = STRLEN("parent ") + GIT_OID_HEXSZ + 1; - assert(walk && commit_object); + unsigned char *buffer = raw->data; + unsigned char *buffer_end = buffer + raw->len; + unsigned char *parents_start; - if (commit_object->object.repo != walk->repo || walk->walking) - return NULL; + int i, parents = 0; - commit = commit_to_walkcommit(walk, commit_object); - if (commit == NULL) - return NULL; + buffer += STRLEN("tree ") + GIT_OID_HEXSZ + 1; - if (commit->seen) - return commit; + parents_start = buffer; + while (buffer + parent_len < buffer_end && memcmp(buffer, "parent ", STRLEN("parent ")) == 0) { + parents++; + buffer += parent_len; + } - commit->seen = 1; + commit->parents = alloc_parents(commit, parents); + if (commit->parents == NULL) + return GIT_ENOMEM; - for (i = 0; i < commit->commit_object->parents.length; ++i) { - git_commit *parent_object; - git_revwalk_commit *parent; + buffer = parents_start; + for (i = 0; i < parents; ++i) { + git_oid oid; - parent_object = git_vector_get(&commit->commit_object->parents, i); + if (git_oid_mkstr(&oid, (char *)buffer + STRLEN("parent ")) < GIT_SUCCESS) + return GIT_EOBJCORRUPTED; - if ((parent = commit_to_walkcommit(walk, parent_object)) == NULL) - return NULL; + commit->parents[i] = commit_lookup(walk, &oid); + if (commit->parents[i] == NULL) + return GIT_ENOMEM; - parent = insert_commit(walk, parent_object); - if (parent == NULL) - return NULL; + buffer += parent_len; + } - parent->in_degree++; + commit->out_degree = (unsigned short)parents; - git_revwalk_list_push_back(&commit->parents, parent); - } + if ((buffer = memchr(buffer, '\n', buffer_end - buffer)) == NULL) + return GIT_EOBJCORRUPTED; - if (git_revwalk_list_push_back(&walk->iterator, commit)) - return NULL; + buffer = memchr(buffer, '>', buffer_end - buffer); + if (buffer == NULL) + return GIT_EOBJCORRUPTED; - return commit; -} + commit->time = strtol((char *)buffer + 2, NULL, 10); + if (commit->time == 0) + return GIT_EOBJCORRUPTED; -int git_revwalk_push(git_revwalk *walk, git_commit *commit) -{ - assert(walk && commit); - return insert_commit(walk, commit) ? GIT_SUCCESS : GIT_ENOMEM; + commit->parsed = 1; + return GIT_SUCCESS; } -static void mark_uninteresting(git_revwalk_commit *commit) +static int commit_parse(git_revwalk *walk, commit_object *commit) { - git_revwalk_listnode *parent; + git_rawobj data; + int error; - assert(commit); + if (commit->parsed) + return GIT_SUCCESS; - commit->uninteresting = 1; - parent = commit->parents.head; + if ((error = git_odb_read(&data, walk->repo->db, &commit->oid)) < GIT_SUCCESS) + return error; - while (parent) { - mark_uninteresting(parent->walk_commit); - parent = parent->next; + if (data.type != GIT_OBJ_COMMIT) { + git_rawobj_close(&data); + return GIT_EOBJTYPE; } + + error = commit_quick_parse(walk, commit, &data); + git_rawobj_close(&data); + return error; } -int git_revwalk_hide(git_revwalk *walk, git_commit *commit) +static void mark_uninteresting(commit_object *commit) { - git_revwalk_commit *hide; + unsigned short i; + assert(commit); - assert(walk && commit); - - hide = insert_commit(walk, commit); - if (hide == NULL) - return GIT_ENOMEM; + commit->uninteresting = 1; - mark_uninteresting(hide); - return GIT_SUCCESS; + for (i = 0; i < commit->out_degree; ++i) + if (!commit->parents[i]->uninteresting) + mark_uninteresting(commit->parents[i]); } - -static void prepare_walk(git_revwalk *walk) +static int process_commit(git_revwalk *walk, commit_object *commit) { - if (walk->sorting & GIT_SORT_TIME) - git_revwalk_list_timesort(&walk->iterator); - - if (walk->sorting & GIT_SORT_TOPOLOGICAL) - git_revwalk_list_toposort(&walk->iterator); + int error; - if (walk->sorting & GIT_SORT_REVERSE) - walk->next = &git_revwalk_list_pop_back; - else - walk->next = &git_revwalk_list_pop_front; + if (commit->seen) + return GIT_SUCCESS; - walk->walking = 1; -} + commit->seen = 1; -int git_revwalk_next(git_commit **commit, git_revwalk *walk) -{ - git_revwalk_commit *next; + if ((error = commit_parse(walk, commit)) < GIT_SUCCESS) + return error; - assert(walk && commit); + if (commit->uninteresting) + mark_uninteresting(commit); - if (!walk->walking) - prepare_walk(walk); + return walk->enqueue(walk, commit); +} - *commit = NULL; +static int process_commit_parents(git_revwalk *walk, commit_object *commit) +{ + unsigned short i; + int error = GIT_SUCCESS; - while ((next = walk->next(&walk->iterator)) != NULL) { - if (!next->uninteresting) { - *commit = next->commit_object; - GIT_OBJECT_INCREF(walk->repo, *commit); - return GIT_SUCCESS; - } + for (i = 0; i < commit->out_degree && error == GIT_SUCCESS; ++i) { + error = process_commit(walk, commit->parents[i]); } - /* No commits left to iterate */ - git_revwalk_reset(walk); - return GIT_EREVWALKOVER; + return error; } -void git_revwalk_reset(git_revwalk *walk) +static int push_commit(git_revwalk *walk, const git_oid *oid, int uninteresting) { - const void *_unused; - git_revwalk_commit *commit; + commit_object *commit; - assert(walk); + commit = commit_lookup(walk, oid); + if (commit == NULL) + return GIT_ENOTFOUND; - GIT_HASHTABLE_FOREACH(walk->commits, _unused, commit, { - GIT_OBJECT_DECREF(walk->repo, commit->commit_object); - git_revwalk_list_clear(&commit->parents); - free(commit); - }); + if (uninteresting) + mark_uninteresting(commit); - git_hashtable_clear(walk->commits); - git_revwalk_list_clear(&walk->iterator); - walk->walking = 0; + return git_vector_insert(&walk->pending, commit); } +int git_revwalk_push(git_revwalk *walk, const git_oid *oid) +{ + assert(walk && oid); + return push_commit(walk, oid, 0); +} +int git_revwalk_hide(git_revwalk *walk, const git_oid *oid) +{ + assert(walk && oid); + return push_commit(walk, oid, 1); +} - - - -int git_revwalk_list_push_back(git_revwalk_list *list, git_revwalk_commit *commit) +static int revwalk_enqueue_timesort(git_revwalk *walk, commit_object *commit) { - git_revwalk_listnode *node = NULL; + return git_pqueue_insert(&walk->iterator_time, commit); +} - node = git__malloc(sizeof(git_revwalk_listnode)); +static int revwalk_enqueue_unsorted(git_revwalk *walk, commit_object *commit) +{ + return commit_list_insert(commit, &walk->iterator_rand) ? GIT_SUCCESS : GIT_ENOMEM; +} - if (node == NULL) - return GIT_ENOMEM; +static int revwalk_next_timesort(commit_object **object_out, git_revwalk *walk) +{ + int error; + commit_object *next; - node->walk_commit = commit; - node->next = NULL; - node->prev = list->tail; + while ((next = git_pqueue_pop(&walk->iterator_time)) != NULL) { + if ((error = process_commit_parents(walk, next)) < GIT_SUCCESS) + return error; - if (list->tail == NULL) { - list->head = list->tail = node; - } else { - list->tail->next = node; - list->tail = node; + if (!next->uninteresting) { + *object_out = next; + return GIT_SUCCESS; + } } - list->size++; - return 0; + return GIT_EREVWALKOVER; } -int git_revwalk_list_push_front(git_revwalk_list *list, git_revwalk_commit *commit) +static int revwalk_next_unsorted(commit_object **object_out, git_revwalk *walk) { - git_revwalk_listnode *node = NULL; - - node = git__malloc(sizeof(git_revwalk_listnode)); + int error; + commit_object *next; - if (node == NULL) - return GIT_ENOMEM; + while ((next = commit_list_pop(&walk->iterator_rand)) != NULL) { + if ((error = process_commit_parents(walk, next)) < GIT_SUCCESS) + return error; - node->walk_commit = commit; - node->next = list->head; - node->prev = NULL; - - if (list->head == NULL) { - list->head = list->tail = node; - } else { - list->head->prev = node; - list->head = node; + if (!next->uninteresting) { + *object_out = next; + return GIT_SUCCESS; + } } - list->size++; - return 0; + return GIT_EREVWALKOVER; } - -git_revwalk_commit *git_revwalk_list_pop_back(git_revwalk_list *list) +static int revwalk_next_toposort(commit_object **object_out, git_revwalk *walk) { - git_revwalk_listnode *node; - git_revwalk_commit *commit; + commit_object *next; + unsigned short i; - if (list->tail == NULL) - return NULL; + for (;;) { + next = commit_list_pop(&walk->iterator_topo); + if (next == NULL) + return GIT_EREVWALKOVER; - node = list->tail; - list->tail = list->tail->prev; - if (list->tail == NULL) - list->head = NULL; - else - list->tail->next = NULL; + if (next->in_degree > 0) { + next->topo_delay = 1; + continue; + } - commit = node->walk_commit; - free(node); + for (i = 0; i < next->out_degree; ++i) { + commit_object *parent = next->parents[i]; - list->size--; + if (--parent->in_degree == 0 && parent->topo_delay) { + parent->topo_delay = 0; + commit_list_insert(parent, &walk->iterator_topo); + } + } - return commit; + *object_out = next; + return GIT_SUCCESS; + } } -git_revwalk_commit *git_revwalk_list_pop_front(git_revwalk_list *list) +static int revwalk_next_reverse(commit_object **object_out, git_revwalk *walk) { - git_revwalk_listnode *node; - git_revwalk_commit *commit; - - if (list->head == NULL) - return NULL; - - node = list->head; - list->head = list->head->next; - if (list->head == NULL) - list->tail = NULL; - else - list->head->prev = NULL; + *object_out = commit_list_pop(&walk->iterator_reverse); + return *object_out ? GIT_SUCCESS : GIT_EREVWALKOVER; +} - commit = node->walk_commit; - free(node); - list->size--; +static int prepare_walk(git_revwalk *walk) +{ + unsigned int i; + int error; - return commit; -} + if (walk->sorting & GIT_SORT_TIME) { + if ((error = git_pqueue_init(&walk->iterator_time, 32, commit_time_cmp)) < GIT_SUCCESS) + return error; -void git_revwalk_list_clear(git_revwalk_list *list) -{ - git_revwalk_listnode *node, *next_node; + walk->get_next = &revwalk_next_timesort; + walk->enqueue = &revwalk_enqueue_timesort; + } else { + walk->get_next = &revwalk_next_unsorted; + walk->enqueue = &revwalk_enqueue_unsorted; + } - node = list->head; - while (node) { - next_node = node->next; - free(node); - node = next_node; + for (i = 0; i < walk->pending.length; ++i) { + commit_object *commit = walk->pending.contents[i]; + if ((error = process_commit(walk, commit)) < GIT_SUCCESS) { + return error; + } } - list->head = list->tail = NULL; - list->size = 0; -} + if (walk->sorting & GIT_SORT_TOPOLOGICAL) { + commit_object *next; + unsigned short i; + int error; -void git_revwalk_list_timesort(git_revwalk_list *list) -{ - git_revwalk_listnode *p, *q, *e; - int in_size, p_size, q_size, merge_count, i; + while ((error = walk->get_next(&next, walk)) == GIT_SUCCESS) { + for (i = 0; i < next->out_degree; ++i) { + commit_object *parent = next->parents[i]; + parent->in_degree++; + } - if (list->head == NULL) - return; + commit_list_insert(next, &walk->iterator_topo); + } - in_size = 1; + if (error != GIT_EREVWALKOVER) + return error; - do { - p = list->head; - list->tail = NULL; - merge_count = 0; + walk->get_next = &revwalk_next_toposort; + } - while (p != NULL) { - merge_count++; - q = p; - p_size = 0; - q_size = in_size; + if (walk->sorting & GIT_SORT_REVERSE) { + commit_object *next; + int error; - for (i = 0; i < in_size && q; ++i, q = q->next) - p_size++; + while ((error = walk->get_next(&next, walk)) == GIT_SUCCESS) + commit_list_insert(next, &walk->iterator_reverse); - while (p_size > 0 || (q_size > 0 && q)) { + if (error != GIT_EREVWALKOVER) + return error; - if (p_size == 0) - e = q, q = q->next, q_size--; + walk->get_next = &revwalk_next_reverse; + } - else if (q_size == 0 || q == NULL || - p->walk_commit->commit_object->committer->when.time >= - q->walk_commit->commit_object->committer->when.time) - e = p, p = p->next, p_size--; + walk->walking = 1; + return GIT_SUCCESS; +} - else - e = q, q = q->next, q_size--; - if (list->tail != NULL) - list->tail->next = e; - else - list->head = e; +int git_revwalk_next(git_oid *oid, git_revwalk *walk) +{ + int error; + commit_object *next; - e->prev = list->tail; - list->tail = e; - } + assert(walk && oid); - p = q; - } + if (!walk->walking) { + if ((error = prepare_walk(walk)) < GIT_SUCCESS) + return error; + } - list->tail->next = NULL; - in_size *= 2; + error = walk->get_next(&next, walk); + if (error < GIT_SUCCESS) + return error; - } while (merge_count > 1); + git_oid_cpy(oid, &next->oid); + return GIT_SUCCESS; } -void git_revwalk_list_toposort(git_revwalk_list *list) +void git_revwalk_reset(git_revwalk *walk) { - git_revwalk_commit *commit; - git_revwalk_list topo; - memset(&topo, 0x0, sizeof(git_revwalk_list)); - - while ((commit = git_revwalk_list_pop_back(list)) != NULL) { - git_revwalk_listnode *p; - - if (commit->in_degree > 0) { - commit->topo_delay = 1; - continue; - } - - for (p = commit->parents.head; p != NULL; p = p->next) { - p->walk_commit->in_degree--; + const void *_unused; + commit_object *commit; - if (p->walk_commit->in_degree == 0 && p->walk_commit->topo_delay) { - p->walk_commit->topo_delay = 0; - git_revwalk_list_push_back(list, p->walk_commit); - } - } + assert(walk); - git_revwalk_list_push_back(&topo, commit); - } + GIT_HASHTABLE_FOREACH(walk->commits, _unused, commit, + commit->seen = 0; + commit->in_degree = 0; + commit->topo_delay = 0; + ); - list->head = topo.head; - list->tail = topo.tail; - list->size = topo.size; + git_pqueue_free(&walk->iterator_time); + commit_list_free(walk->iterator_topo); + commit_list_free(walk->iterator_rand); + commit_list_free(walk->iterator_reverse); + walk->walking = 0; } diff --git a/src/revwalk.h b/src/revwalk.h index 7b69ccd63..2970d773c 100644 --- a/src/revwalk.h +++ b/src/revwalk.h @@ -8,60 +8,4 @@ #include "repository.h" #include "hashtable.h" -struct git_revwalk_commit; - -typedef struct git_revwalk_listnode { - struct git_revwalk_commit *walk_commit; - struct git_revwalk_listnode *next; - struct git_revwalk_listnode *prev; -} git_revwalk_listnode; - -typedef struct git_revwalk_list { - struct git_revwalk_listnode *head; - struct git_revwalk_listnode *tail; - size_t size; -} git_revwalk_list; - - -struct git_revwalk_commit { - - git_commit *commit_object; - git_revwalk_list parents; - - unsigned short in_degree; - unsigned seen:1, - uninteresting:1, - topo_delay:1, - flags:25; -}; - -typedef struct git_revwalk_commit git_revwalk_commit; - -struct git_revwalk { - git_repository *repo; - - git_hashtable *commits; - git_revwalk_list iterator; - - git_revwalk_commit *(*next)(git_revwalk_list *); - - unsigned walking:1; - unsigned int sorting; -}; - - -void git_revwalk__prepare_walk(git_revwalk *walk); -int git_revwalk__enroot(git_revwalk *walk, git_commit *commit); - -int git_revwalk_list_push_back(git_revwalk_list *list, git_revwalk_commit *commit); -int git_revwalk_list_push_front(git_revwalk_list *list, git_revwalk_commit *obj); - -git_revwalk_commit *git_revwalk_list_pop_back(git_revwalk_list *list); -git_revwalk_commit *git_revwalk_list_pop_front(git_revwalk_list *list); - -void git_revwalk_list_clear(git_revwalk_list *list); - -void git_revwalk_list_timesort(git_revwalk_list *list); -void git_revwalk_list_toposort(git_revwalk_list *list); - #endif /* INCLUDE_revwalk_h__ */ diff --git a/src/util.h b/src/util.h index d5320e15b..fab93c1c2 100644 --- a/src/util.h +++ b/src/util.h @@ -93,6 +93,8 @@ GIT_INLINE(int) git__is_sizet(git_off_t p) extern char *git__strtok(char *output, char *src, char *delimit); extern char *git__strtok_keep(char *output, char *src, char *delimit); +#define STRLEN(str) (sizeof(str) - 1) + /* * Realloc the buffer pointed at by variable 'x' so that it can hold * at least 'nr' entries; the number of entries currently allocated -- cgit v1.2.1 From 58d06cf120eb9bb9247bb807bb105981bb3482a8 Mon Sep 17 00:00:00 2001 From: Vicent Marti Date: Thu, 10 Mar 2011 01:06:24 +0200 Subject: Rewrite the Pack backend The new pack backend is an adaptation of the original git.git code in `sha1_file.c`. It's slightly faster than the previous version and severely less memory-hungry. The call-stack of a normal pack backend query has been properly documented in the top of the header for future reference. And by properly I mean with ASCII diagrams 'n shit. --- src/odb_pack.c | 1914 +++++++++++++++++++++++++++++++------------------------- src/util.h | 2 + 2 files changed, 1058 insertions(+), 858 deletions(-) (limited to 'src') diff --git a/src/odb_pack.c b/src/odb_pack.c index 664b00139..3067179be 100644 --- a/src/odb_pack.c +++ b/src/odb_pack.c @@ -33,503 +33,625 @@ #include "git2/odb_backend.h" -/** First 4 bytes of a pack-*.idx file header. - * - * Note this header exists only in idx v2 and later. The idx v1 - * file format does not have a magic sequence at the front, and - * must be detected by the first four bytes *not* being this value - * and the first 8 bytes matching the following expression: +#define DEFAULT_WINDOW_SIZE \ + (sizeof(void*) >= 8 \ + ? 1 * 1024 * 1024 * 1024 \ + : 32 * 1024 * 1024) + +#define DEFAULT_MAPPED_LIMIT \ + ((1024L * 1024L) * (sizeof(void*) >= 8 ? 8192 : 256)) + +#define PACK_SIGNATURE 0x5041434b /* "PACK" */ +#define PACK_VERSION 2 +#define pack_version_ok(v) ((v) == htonl(2) || (v) == htonl(3)) +struct pack_header { + uint32_t hdr_signature; + uint32_t hdr_version; + uint32_t hdr_entries; +}; + +/* + * The first four bytes of index formats later than version 1 should + * start with this signature, as all older git binaries would find this + * value illegal and abort reading the file. * - * uint32_t *fanout = ... the file data at offset 0 ... - * ntohl(fanout[0]) < ntohl(fanout[1]) + * This is the case because the number of objects in a packfile + * cannot exceed 1,431,660,000 as every object would need at least + * 3 bytes of data and the overall packfile cannot exceed 4 GiB with + * version 1 of the index file due to the offsets limited to 32 bits. + * Clearly the signature exceeds this maximum. * - * The value chosen here for PACK_TOC is such that the above - * cannot be true for an idx v1 file. + * Very old git binaries will also compare the first 4 bytes to the + * next 4 bytes in the index and abort with a "non-monotonic index" + * error if the second 4 byte word is smaller than the first 4 + * byte word. This would be true in the proposed future index + * format as idx_signature would be greater than idx_version. */ -#define PACK_TOC 0xff744f63 /* -1tOc */ +#define PACK_IDX_SIGNATURE 0xff744f63 /* "\377tOc" */ -/** First 4 bytes of a pack-*.pack file header. */ -#define PACK_SIG 0x5041434b /* PACK */ +struct pack_idx_header { + uint32_t idx_signature; + uint32_t idx_version; +}; -#define GIT_PACK_NAME_MAX (5 + 40 + 1) +struct pack_window { + struct pack_window *next; + git_map window_map; + off_t offset; + unsigned int last_used; + unsigned int inuse_cnt; +}; -struct pack_backend; +struct pack_file { + struct pack_window *windows; + off_t pack_size; -typedef struct { - uint32_t n; - unsigned char *oid; - git_off_t offset; - git_off_t size; -} index_entry; + git_map index_map; -typedef struct { /* '.pack' file header */ - uint32_t sig; /* PACK_SIG */ - uint32_t ver; /* pack version */ - uint32_t cnt; /* object count */ -} pack_hdr; + uint32_t num_objects; + uint32_t num_bad_objects; + git_oid *bad_object_sha1; /* array of git_oid */ -typedef struct git_pack { - struct pack_backend *backend; - git_lck lock; - - /** Functions to access idx_map. */ - int (*idx_search)( - uint32_t *, - struct git_pack *, - const git_oid *); - int (*idx_search_offset)( - uint32_t *, - struct git_pack *, - git_off_t); - int (*idx_get)( - index_entry *, - struct git_pack *, - uint32_t n); - - /** The .idx file, mapped into memory. */ - git_file idx_fd; - git_map idx_map; - uint32_t *im_fanout; - unsigned char *im_oid; - uint32_t *im_crc; - uint32_t *im_offset32; - uint32_t *im_offset64; - uint32_t *im_off_idx; - uint32_t *im_off_next; - - /** Number of objects in this pack. */ - uint32_t obj_cnt; - - /** File descriptor for the .pack file. */ - git_file pack_fd; - - /** Memory map of the pack's contents */ - git_map pack_map; - - /** The size of the .pack file. */ - git_off_t pack_size; - - /** The mtime of the .pack file. */ - time_t pack_mtime; - - /** Number of git_packlist we appear in. */ - unsigned int refcnt; - - /** Number of active users of the idx_map data. */ - unsigned int idxcnt; - unsigned - invalid:1 /* the pack is unable to be read by libgit2 */ - ; - - /** Name of the pack file(s), without extension ("pack-abc"). */ - char pack_name[GIT_PACK_NAME_MAX]; -} git_pack; - -typedef struct { - size_t n_packs; - unsigned int refcnt; - git_pack *packs[GIT_FLEX_ARRAY]; -} git_packlist; - -typedef struct pack_backend { - git_odb_backend parent; + int index_version; + time_t mtime; + int pack_fd; + unsigned pack_local:1, pack_keep:1; + git_oid sha1; - git_lck lock; - char *objects_dir; - git_packlist *packlist; -} pack_backend; + /* something like ".git/objects/pack/xxxxx.pack" */ + char pack_name[GIT_FLEX_ARRAY]; /* more */ +}; +struct pack_entry { + off_t offset; + git_oid sha1; + struct pack_file *p; +}; -typedef struct pack_location { - git_pack *ptr; - uint32_t n; -} pack_location; +struct pack__dirent { + struct pack_backend *backend; + int is_pack_local; +}; -static int pack_stat(git_pack *p); -static int pack_openidx(git_pack *p); -static void pack_decidx(git_pack *p); -static int read_pack_hdr(pack_hdr *out, git_file fd); -static int check_pack_hdr(git_pack *p); -static int check_pack_sha1(git_pack *p); -static int open_pack(git_pack *p); +struct pack_backend { + git_odb_backend parent; + git_vector packs; + struct pack_file *last_found; + size_t window_size; /* needs default value */ -static int pack_openidx_map(git_pack *p); -static int pack_openidx_v1(git_pack *p); -static int pack_openidx_v2(git_pack *p); + size_t mapped_limit; /* needs default value */ + size_t peak_mapped; + size_t mapped; + size_t used_ctr; -GIT_INLINE(uint32_t) decode32(void *b) -{ - return ntohl(*((uint32_t *)b)); -} + unsigned int peak_open_windows; + unsigned int open_windows; -GIT_INLINE(uint64_t) decode64(void *b) -{ - uint32_t *p = b; - return (((uint64_t)ntohl(p[0])) << 32) | ntohl(p[1]); -} + unsigned int mmap_calls; +}; + +/** + * The wonderful tale of a Packed Object lookup query + * =================================================== + * A riveting and epic story of epicness and ASCII + * art, presented by yours truly, + * Sir Vicent of Marti + * + * + * Chapter 1: Once upon a time... + * Initialization of the Pack Backend + * -------------------------------------------------- + * + * # git_odb_backend_pack + * | Creates the pack backend structure, initializes the + * | callback pointers to our default read() and exist() methods, + * | and tries to preload all the known packfiles in the ODB. + * | + * |-# packfile_load_all + * | Tries to find the `pack` folder, if it exists. ODBs without + * | a pack folder are ignored altogether. If there's a `pack` folder + * | we run a `dirent` callback through every file in the pack folder + * | to find our packfiles. The packfiles are then sorted according + * | to a sorting callback. + * | + * |-# packfile_load__cb + * | | This callback is called from `dirent` with every single file + * | | inside the pack folder. We find the packs by actually locating + * | | their index (ends in ".idx"). From that index, we verify that + * | | the corresponding packfile exists and is valid, and if so, we + * | | add it to the pack list. + * | | + * | |-# packfile_check + * | Make sure that there's a packfile to back this index, and store + * | some very basic information regarding the packfile itself, + * | such as the full path, the size, and the modification time. + * | We don't actually open the packfile to check for internal consistency. + * | + * |-# packfile_sort__cb + * Sort all the preloaded packs according to some specific criteria: + * we prioritize the "newer" packs because it's more likely they + * contain the objects we are looking for, and we prioritize local + * packs over remote ones. + * + * + * + * Chapter 2: To be, or not to be... + * A standard packed `exist` query for an OID + * -------------------------------------------------- + * + * # pack_backend__exists + * | Check if the given SHA1 oid exists in any of the packs + * | that have been loaded for our ODB. + * | + * |-# pack_entry_find + * | Iterate through all the packs that have been preloaded + * | (starting by the pack where the latest object was found) + * | to try to find the OID in one of them. + * | + * |-# pack_entry_find1 + * | Check the index of an individual pack to see if the SHA1 + * | OID can be found. If we can find the offset to that SHA1 + * | inside of the index, that means the object is contained + * | inside of the packfile and we can stop searching. + * | Before returning, we verify that the packfile behing the + * | index we are searching still exists on disk. + * | + * |-# pack_entry_find_offset + * | | Mmap the actual index file to disk if it hasn't been opened + * | | yet, and run a binary search through it to find the OID. + * | | See for specifics + * | | on the Packfile Index format and how do we find entries in it. + * | | + * | |-# pack_index_open + * | | Guess the name of the index based on the full path to the + * | | packfile, open it and verify its contents. Only if the index + * | | has not been opened already. + * | | + * | |-# pack_index_check + * | Mmap the index file and do a quick run through the header + * | to guess the index version (right now we support v1 and v2), + * | and to verify that the size of the index makes sense. + * | + * |-# packfile_open + * See `packfile_open` in Chapter 3 + * + * + * + * Chapter 3: The neverending story... + * A standard packed `lookup` query for an OID + * -------------------------------------------------- + * TODO + * + */ + /*********************************************************** * - * PACKFILE FUNCTIONS - * - * Locate, open and access the contents of a packfile + * FORWARD DECLARATIONS * ***********************************************************/ -static int pack_stat(git_pack *p) -{ - char pb[GIT_PATH_MAX]; - struct stat sb; +static void pack_window_free_all(struct pack_backend *backend, struct pack_file *p); +static int pack_window_contains(struct pack_window *win, off_t offset); - if (git__fmt(pb, sizeof(pb), "%s/pack/%s.pack", - p->backend->objects_dir, - p->pack_name) < 0) - return GIT_ERROR; +static void pack_window_scan_lru(struct pack_file *p, struct pack_file **lru_p, + struct pack_window **lru_w, struct pack_window **lru_l); - if (gitfo_stat(pb, &sb) || !S_ISREG(sb.st_mode)) - return GIT_ERROR; +static int pack_window_close_lru( struct pack_backend *backend, + struct pack_file *current, git_file keep_fd); - if (sb.st_size < (3 * 4 + GIT_OID_RAWSZ)) - return GIT_ERROR; +static void pack_window_close(struct pack_window **w_cursor); - p->pack_size = sb.st_size; - p->pack_mtime = sb.st_mtime; +static unsigned char *pack_window_open( struct pack_backend *backend, + struct pack_file *p, struct pack_window **w_cursor, off_t offset, + unsigned int *left); - return GIT_SUCCESS; -} +static int packfile_sort__cb(const void *a_, const void *b_); -static int pack_openidx(git_pack *p) -{ - gitlck_lock(&p->lock); +static void pack_index_free(struct pack_file *p); - if (p->invalid) { - gitlck_unlock(&p->lock); - return GIT_ERROR; - } +static int pack_index_check(const char *path, struct pack_file *p); +static int pack_index_open(struct pack_file *p); - if (++p->idxcnt == 1 && !p->idx_search) { - int status, version; - uint32_t *data; +static struct pack_file *packfile_alloc(int extra); +static int packfile_open(struct pack_file *p); +static int packfile_check(struct pack_file **pack_out, const char *path, int local); +static int packfile_load__cb(void *_data, char *path); +static int packfile_load_all(struct pack_backend *backend, const char *odb_path, int local); - if (pack_stat(p) || pack_openidx_map(p)) { - p->invalid = 1; - p->idxcnt--; - gitlck_unlock(&p->lock); - return GIT_ERROR; - } - data = p->idx_map.data; - status = GIT_SUCCESS; - version = 1; +static off_t nth_packed_object_offset(const struct pack_file *p, uint32_t n); - if (decode32(&data[0]) == PACK_TOC) - version = decode32(&data[1]); - - switch (version) { - case 1: - status = pack_openidx_v1(p); - break; - case 2: - status = pack_openidx_v2(p); - break; - default: - status = GIT_ERROR; - } +static int pack_entry_find_offset(off_t *offset_out, + struct pack_file *p, const git_oid *oid); - if (status != GIT_SUCCESS) { - gitfo_free_map(&p->idx_map); - p->invalid = 1; - p->idxcnt--; - gitlck_unlock(&p->lock); - return status; - } - } +static int pack_entry_find1(struct pack_entry *e, + struct pack_file *p, const git_oid *oid); - gitlck_unlock(&p->lock); - return GIT_SUCCESS; -} +static int pack_entry_find(struct pack_entry *e, + struct pack_backend *backend, const git_oid *oid); -static void pack_decidx(git_pack *p) -{ - gitlck_lock(&p->lock); - p->idxcnt--; - gitlck_unlock(&p->lock); -} +static off_t get_delta_base(struct pack_backend *backend, + struct pack_file *p, struct pack_window **w_curs, + off_t *curpos, git_otype type, + off_t delta_obj_offset); -static int read_pack_hdr(pack_hdr *out, git_file fd) -{ - pack_hdr hdr; +static unsigned long packfile_unpack_header1( + size_t *sizep, + git_otype *type, + const unsigned char *buf, + unsigned long len); - if (gitfo_read(fd, &hdr, sizeof(hdr))) - return GIT_ERROR; +static int packfile_unpack_header( + size_t *size_p, + git_otype *type_p, + struct pack_backend *backend, + struct pack_file *p, + struct pack_window **w_curs, + off_t *curpos); - out->sig = decode32(&hdr.sig); - out->ver = decode32(&hdr.ver); - out->cnt = decode32(&hdr.cnt); +static int packfile_unpack_compressed( + git_rawobj *obj, + struct pack_backend *backend, + struct pack_file *p, + struct pack_window **w_curs, + off_t curpos, + size_t size, + git_otype type); - return GIT_SUCCESS; -} +static int packfile_unpack_delta( + git_rawobj *obj, + struct pack_backend *backend, + struct pack_file *p, + struct pack_window **w_curs, + off_t curpos, + size_t delta_size, + git_otype delta_type, + off_t obj_offset); + +static int packfile_unpack(git_rawobj *obj, struct pack_backend *backend, + struct pack_file *p, off_t obj_offset); + + + + + +/*********************************************************** + * + * PACK WINDOW MANAGEMENT + * + ***********************************************************/ -static int check_pack_hdr(git_pack *p) +void pack_window_free_all(struct pack_backend *backend, struct pack_file *p) { - pack_hdr hdr; + while (p->windows) { + struct pack_window *w = p->windows; + assert(w->inuse_cnt == 0); - if (read_pack_hdr(&hdr, p->pack_fd)) - return GIT_ERROR; + backend->mapped -= w->window_map.len; + backend->open_windows--; - if (hdr.sig != PACK_SIG - || (hdr.ver != 2 && hdr.ver != 3) - || hdr.cnt != p->obj_cnt) - return GIT_ERROR; + gitfo_free_map(&w->window_map); - return GIT_SUCCESS; + p->windows = w->next; + free(w); + } } -static int check_pack_sha1(git_pack *p) +GIT_INLINE(int) pack_window_contains(struct pack_window *win, off_t offset) { - unsigned char *data = p->idx_map.data; - git_off_t pack_sha1_off = p->pack_size - GIT_OID_RAWSZ; - size_t idx_pack_sha1_off = p->idx_map.len - 2 * GIT_OID_RAWSZ; - git_oid pack_id, idx_pack_id; + /* We must promise at least 20 bytes (one hash) after the + * offset is available from this window, otherwise the offset + * is not actually in this window and a different window (which + * has that one hash excess) must be used. This is to support + * the object header and delta base parsing routines below. + */ + off_t win_off = win->offset; + return win_off <= offset + && (offset + 20) <= (off_t)(win_off + win->window_map.len); +} - if (gitfo_lseek(p->pack_fd, pack_sha1_off, SEEK_SET) == -1) - return GIT_ERROR; +static void pack_window_scan_lru( + struct pack_file *p, + struct pack_file **lru_p, + struct pack_window **lru_w, + struct pack_window **lru_l) +{ + struct pack_window *w, *w_l; + + for (w_l = NULL, w = p->windows; w; w = w->next) { + if (!w->inuse_cnt) { + if (!*lru_w || w->last_used < (*lru_w)->last_used) { + *lru_p = p; + *lru_w = w; + *lru_l = w_l; + } + } + w_l = w; + } +} - if (gitfo_read(p->pack_fd, pack_id.id, sizeof(pack_id.id))) - return GIT_ERROR; +static int pack_window_close_lru( + struct pack_backend *backend, + struct pack_file *current, + git_file keep_fd) +{ + struct pack_file *lru_p = NULL; + struct pack_window *lru_w = NULL, *lru_l = NULL; + size_t i; + + if (current) + pack_window_scan_lru(current, &lru_p, &lru_w, &lru_l); + + for (i = 0; i < backend->packs.length; ++i) + pack_window_scan_lru(git_vector_get(&backend->packs, i), &lru_p, &lru_w, &lru_l); + + if (lru_p) { + backend->mapped -= lru_w->window_map.len; + gitfo_free_map(&lru_w->window_map); + + if (lru_l) + lru_l->next = lru_w->next; + else { + lru_p->windows = lru_w->next; + if (!lru_p->windows && lru_p->pack_fd != keep_fd) { + gitfo_close(lru_p->pack_fd); + lru_p->pack_fd = -1; + } + } - git_oid_mkraw(&idx_pack_id, data + idx_pack_sha1_off); + free(lru_w); + backend->open_windows--; + return GIT_SUCCESS; + } - if (git_oid_cmp(&pack_id, &idx_pack_id)) - return GIT_ERROR; + return GIT_ERROR; +} - return GIT_SUCCESS; +static void pack_window_close(struct pack_window **w_cursor) +{ + struct pack_window *w = *w_cursor; + if (w) { + w->inuse_cnt--; + *w_cursor = NULL; + } } -static int open_pack(git_pack *p) +static unsigned char *pack_window_open( + struct pack_backend *backend, + struct pack_file *p, + struct pack_window **w_cursor, + off_t offset, + unsigned int *left) { - char pb[GIT_PATH_MAX]; - struct stat sb; + struct pack_window *win = *w_cursor; - if (p->pack_fd != -1) - return GIT_SUCCESS; + if (p->pack_fd == -1 && packfile_open(p) < GIT_SUCCESS) + return NULL; - if (git__fmt(pb, sizeof(pb), "%s/pack/%s.pack", - p->backend->objects_dir, - p->pack_name) < 0) - return GIT_ERROR; + /* Since packfiles end in a hash of their content and it's + * pointless to ask for an offset into the middle of that + * hash, and the pack_window_contains function above wouldn't match + * don't allow an offset too close to the end of the file. + */ + if (offset > (p->pack_size - 20)) + return NULL; - if (pack_openidx(p)) - return GIT_ERROR; + if (!win || !pack_window_contains(win, offset)) { - if ((p->pack_fd = gitfo_open(pb, O_RDONLY)) < 0) - goto error_cleanup; + if (win) + win->inuse_cnt--; - if (gitfo_fstat(p->pack_fd, &sb) - || !S_ISREG(sb.st_mode) || p->pack_size != sb.st_size - || check_pack_hdr(p) || check_pack_sha1(p)) - goto error_cleanup; + for (win = p->windows; win; win = win->next) { + if (pack_window_contains(win, offset)) + break; + } - if (!git__is_sizet(p->pack_size) || - gitfo_map_ro(&p->pack_map, p->pack_fd, 0, (size_t)p->pack_size) < 0) - goto error_cleanup; + if (!win) { + size_t window_align = backend->window_size / 2; + size_t len; - pack_decidx(p); - return GIT_SUCCESS; + win = git__calloc(1, sizeof(*win)); + win->offset = (offset / window_align) * window_align; -error_cleanup: - gitfo_close(p->pack_fd); - p->pack_fd = -1; - pack_decidx(p); - return GIT_ERROR; -} + len = (size_t)(p->pack_size - win->offset); + if (len > backend->window_size) + len = backend->window_size; -static void pack_dec(git_pack *p) -{ - int need_free; - - gitlck_lock(&p->lock); - need_free = !--p->refcnt; - gitlck_unlock(&p->lock); - - if (need_free) { - if (p->idx_search) { - gitfo_free_map(&p->idx_map); - gitfo_close(p->idx_fd); - free(p->im_fanout); - free(p->im_off_idx); - free(p->im_off_next); - if (p->pack_fd != -1) { - gitfo_close(p->pack_fd); - gitfo_free_map(&p->pack_map); - } + backend->mapped += len; + + while (backend->mapped_limit < backend->mapped && + pack_window_close_lru(backend, p, p->pack_fd) == GIT_SUCCESS) {} + + if (gitfo_map_ro(&win->window_map, p->pack_fd, + win->offset, len) < GIT_SUCCESS) + return NULL; + + backend->mmap_calls++; + backend->open_windows++; + + if (backend->mapped > backend->peak_mapped) + backend->peak_mapped = backend->mapped; + + if (backend->open_windows > backend->peak_open_windows) + backend->peak_open_windows = backend->open_windows; + + win->next = p->windows; + p->windows = win; } + } - gitlck_free(&p->lock); - free(p); + if (win != *w_cursor) { + win->last_used = backend->used_ctr++; + win->inuse_cnt++; + *w_cursor = win; } + + offset -= win->offset; + assert(git__is_sizet(offset)); + + if (left) + *left = win->window_map.len - (size_t)offset; + + return (unsigned char *)win->window_map.data + offset; } -static void packlist_dec(pack_backend *backend, git_packlist *pl) -{ - int need_free; - assert(backend && pl); - gitlck_lock(&backend->lock); - need_free = !--pl->refcnt; - gitlck_unlock(&backend->lock); - if (need_free) { - size_t j; - for (j = 0; j < pl->n_packs; j++) - pack_dec(pl->packs[j]); - free(pl); + + + +/*********************************************************** + * + * PACK INDEX METHODS + * + ***********************************************************/ + +static void pack_index_free(struct pack_file *p) +{ + if (p->index_map.data) { + gitfo_free_map(&p->index_map); + p->index_map.data = NULL; } } -static git_pack *alloc_pack(const char *pack_name) +static int pack_index_check(const char *path, struct pack_file *p) { - git_pack *p = git__calloc(1, sizeof(*p)); - if (!p) - return NULL; + struct pack_idx_header *hdr; + uint32_t version, nr, i, *index; - gitlck_init(&p->lock); - strcpy(p->pack_name, pack_name); - p->refcnt = 1; - p->pack_fd = -1; - return p; -} + void *idx_map; + size_t idx_size; -struct scanned_pack { - struct scanned_pack *next; - git_pack *pack; -}; + struct stat st; -static int scan_one_pack(void *state, char *name) -{ - struct scanned_pack **ret = state, *r; - char *s = strrchr(name, '/'), *d; + /* TODO: properly open the file without access time */ + git_file fd = gitfo_open(path, O_RDONLY /*| O_NOATIME */); - if (git__prefixcmp(s + 1, "pack-") - || git__suffixcmp(s, ".pack") - || strlen(s + 1) != GIT_PACK_NAME_MAX + 4) - return 0; - - d = strrchr(s + 1, '.'); - strcpy(d + 1, "idx"); /* "pack-abc.pack" -> "pack-abc.idx" */ - if (gitfo_exists(name)) - return 0; + int error; - if ((r = git__malloc(sizeof(*r))) == NULL) - return GIT_ERROR; + if (fd < 0) + return GIT_EOSERR; - *d = '\0'; /* "pack-abc.pack" -_> "pack-abc" */ - if ((r->pack = alloc_pack(s + 1)) == NULL) { - free(r); - return GIT_ERROR; + if (gitfo_fstat(fd, &st) < GIT_SUCCESS) { + gitfo_close(fd); + return GIT_EOSERR; } - r->next = *ret; - *ret = r; - return 0; -} + if (!git__is_sizet(st.st_size)) + return GIT_ENOMEM; -static git_packlist *scan_packs(pack_backend *backend) -{ - char pb[GIT_PATH_MAX]; - struct scanned_pack *state = NULL, *c; - size_t cnt; - git_packlist *new_list; + idx_size = (size_t)st.st_size; - if (git__fmt(pb, sizeof(pb), "%s/pack", backend->objects_dir) < 0) - return NULL; - gitfo_dirent(pb, sizeof(pb), scan_one_pack, &state); - - /* TODO - merge old entries into the new array */ - for (cnt = 0, c = state; c; c = c->next) - cnt++; - new_list = git__malloc(sizeof(*new_list) - + (sizeof(new_list->packs[0]) * cnt)); - if (!new_list) - goto fail; - - for (cnt = 0, c = state; c; ) { - struct scanned_pack *n = c->next; - c->pack->backend = backend; - new_list->packs[cnt++] = c->pack; - free(c); - c = n; - } - new_list->n_packs = cnt; - new_list->refcnt = 2; - backend->packlist = new_list; - return new_list; - -fail: - while (state) { - struct scanned_pack *n = state->next; - pack_dec(state->pack); - free(state); - state = n; + if (idx_size < 4 * 256 + 20 + 20) { + gitfo_close(fd); + return GIT_EOBJCORRUPTED; } - return NULL; -} -static git_packlist *packlist_get(pack_backend *backend) -{ - git_packlist *pl; - - gitlck_lock(&backend->lock); - if ((pl = backend->packlist) != NULL) - pl->refcnt++; - else - pl = scan_packs(backend); - gitlck_unlock(&backend->lock); - return pl; -} + error = gitfo_map_ro(&p->index_map, fd, 0, idx_size); + gitfo_close(fd); -static int locate_packfile(pack_location *location, pack_backend *backend, const git_oid *id) -{ - git_packlist *pl = packlist_get(backend); - size_t j; + if (error < GIT_SUCCESS) + return error; - if (!pl) - return GIT_ENOTFOUND; + hdr = idx_map = p->index_map.data; - for (j = 0; j < pl->n_packs; j++) { + if (hdr->idx_signature == htonl(PACK_IDX_SIGNATURE)) { + version = ntohl(hdr->idx_version); - git_pack *pack = pl->packs[j]; - uint32_t pos; - int res; + if (version < 2 || version > 2) { + gitfo_free_map(&p->index_map); + return GIT_EOBJCORRUPTED; /* unsupported index version */ + } - if (pack_openidx(pack)) - continue; + } else + version = 1; - res = pack->idx_search(&pos, pack, id); - pack_decidx(pack); + nr = 0; + index = idx_map; - if (!res) { - packlist_dec(backend, pl); + if (version > 1) + index += 2; /* skip index header */ - location->ptr = pack; - location->n = pos; + for (i = 0; i < 256; i++) { + uint32_t n = ntohl(index[i]); + if (n < nr) { + gitfo_free_map(&p->index_map); + return GIT_EOBJCORRUPTED; /* non-monotonic index */ + } + nr = n; + } - return GIT_SUCCESS; + if (version == 1) { + /* + * Total size: + * - 256 index entries 4 bytes each + * - 24-byte entries * nr (20-byte sha1 + 4-byte offset) + * - 20-byte SHA1 of the packfile + * - 20-byte SHA1 file checksum + */ + if (idx_size != 4*256 + nr * 24 + 20 + 20) { + gitfo_free_map(&p->index_map); + return GIT_EOBJCORRUPTED; + } + } else if (version == 2) { + /* + * Minimum size: + * - 8 bytes of header + * - 256 index entries 4 bytes each + * - 20-byte sha1 entry * nr + * - 4-byte crc entry * nr + * - 4-byte offset entry * nr + * - 20-byte SHA1 of the packfile + * - 20-byte SHA1 file checksum + * And after the 4-byte offset table might be a + * variable sized table containing 8-byte entries + * for offsets larger than 2^31. + */ + unsigned long min_size = 8 + 4*256 + nr*(20 + 4 + 4) + 20 + 20; + unsigned long max_size = min_size; + + if (nr) + max_size += (nr - 1)*8; + + if (idx_size < min_size || idx_size > max_size) { + gitfo_free_map(&p->index_map); + return GIT_EOBJCORRUPTED; } + /* Make sure that off_t is big enough to access the whole pack... + * Is this an issue in libgit2? It shouldn't. */ + if (idx_size != min_size && (sizeof(off_t) <= 4)) { + gitfo_free_map(&p->index_map); + return GIT_EOSERR; + } } - packlist_dec(backend, pl); - return GIT_ENOTFOUND; + p->index_version = version; + p->num_objects = nr; + return GIT_SUCCESS; } +static int pack_index_open(struct pack_file *p) +{ + char *idx_name; + int error; + + if (p->index_map.data) + return GIT_SUCCESS; + idx_name = git__strdup(p->pack_name); + strcpy(idx_name + strlen(idx_name) - STRLEN(".pack"), ".idx"); + error = pack_index_check(idx_name, p); + free(idx_name); + return error; +} @@ -541,350 +663,225 @@ static int locate_packfile(pack_location *location, pack_backend *backend, const /*********************************************************** * - * PACKFILE INDEX FUNCTIONS - * - * Get index formation for packfile indexes v1 and v2 + * PACKFILE METHODS * ***********************************************************/ -static int pack_openidx_map(git_pack *p) +static int packfile_sort__cb(const void *a_, const void *b_) { - char pb[GIT_PATH_MAX]; - git_off_t len; - - if (git__fmt(pb, sizeof(pb), "%s/pack/%s.idx", - p->backend->objects_dir, - p->pack_name) < 0) - return GIT_ERROR; - - if ((p->idx_fd = gitfo_open(pb, O_RDONLY)) < 0) - return GIT_ERROR; - - if ((len = gitfo_size(p->idx_fd)) < 0 - || !git__is_sizet(len) - || gitfo_map_ro(&p->idx_map, p->idx_fd, 0, (size_t)len)) { - gitfo_close(p->idx_fd); - return GIT_ERROR; - } + struct pack_file *a = *((struct pack_file **)a_); + struct pack_file *b = *((struct pack_file **)b_); + int st; - return GIT_SUCCESS; + /* + * Local packs tend to contain objects specific to our + * variant of the project than remote ones. In addition, + * remote ones could be on a network mounted filesystem. + * Favor local ones for these reasons. + */ + st = a->pack_local - b->pack_local; + if (st) + return -st; + + /* + * Younger packs tend to contain more recent objects, + * and more recent objects tend to get accessed more + * often. + */ + if (a->mtime < b->mtime) + return 1; + else if (a->mtime == b->mtime) + return 0; + + return -1; } -typedef struct { - git_off_t offset; - uint32_t n; -} offset_idx_info; +static struct pack_file *packfile_alloc(int extra) +{ + struct pack_file *p = git__malloc(sizeof(*p) + extra); + memset(p, 0, sizeof(*p)); + p->pack_fd = -1; + return p; +} -static int cmp_offset_idx_info(const void *lhs, const void *rhs) + +static void packfile_free(struct pack_backend *backend, struct pack_file *p) { - const offset_idx_info *a = lhs; - const offset_idx_info *b = rhs; - return (a->offset < b->offset) ? -1 : (a->offset > b->offset) ? 1 : 0; + assert(p); + + /* clear_delta_base_cache(); */ + pack_window_free_all(backend, p); + + if (p->pack_fd != -1) + gitfo_close(p->pack_fd); + + pack_index_free(p); + + free(p->bad_object_sha1); + free(p); } -static int make_offset_index(git_pack *p, offset_idx_info *data) +static int packfile_open(struct pack_file *p) { - git_off_t min_off = 3 * 4, max_off = p->pack_size - GIT_OID_RAWSZ; - uint32_t *idx, *next; - uint32_t j; + struct stat st; + struct pack_header hdr; + git_oid sha1; + unsigned char *idx_sha1; - qsort(data, p->obj_cnt, sizeof(*data), cmp_offset_idx_info); + if (!p->index_map.data && pack_index_open(p) < GIT_SUCCESS) + return GIT_ENOTFOUND; - if (data[0].offset < min_off || data[p->obj_cnt].offset > max_off) - return GIT_ERROR; + /* TODO: open with noatime */ + p->pack_fd = gitfo_open(p->pack_name, O_RDONLY); + if (p->pack_fd < 0 || gitfo_fstat(p->pack_fd, &st) < GIT_SUCCESS) + return GIT_EOSERR; + + /* If we created the struct before we had the pack we lack size. */ + if (!p->pack_size) { + if (!S_ISREG(st.st_mode)) + goto cleanup; + p->pack_size = (off_t)st.st_size; + } else if (p->pack_size != st.st_size) + goto cleanup; - if ((idx = git__malloc(sizeof(*idx) * (p->obj_cnt+1))) == NULL) - return GIT_ERROR; - if ((next = git__malloc(sizeof(*next) * p->obj_cnt)) == NULL) { - free(idx); - return GIT_ERROR; - } +#if 0 + /* We leave these file descriptors open with sliding mmap; + * there is no point keeping them open across exec(), though. + */ + fd_flag = fcntl(p->pack_fd, F_GETFD, 0); + if (fd_flag < 0) + return error("cannot determine file descriptor flags"); - for (j = 0; j < p->obj_cnt+1; j++) - idx[j] = data[j].n; + fd_flag |= FD_CLOEXEC; + if (fcntl(p->pack_fd, F_SETFD, fd_flag) == -1) + return GIT_EOSERR; +#endif - for (j = 0; j < p->obj_cnt; j++) { - assert(idx[j] < p->obj_cnt); - assert(idx[j+1] < p->obj_cnt+1); + /* Verify we recognize this pack file format. */ + if (gitfo_read(p->pack_fd, &hdr, sizeof(hdr)) < GIT_SUCCESS) + goto cleanup; - next[idx[j]] = idx[j+1]; - } + if (hdr.hdr_signature != htonl(PACK_SIGNATURE)) + goto cleanup; - p->im_off_idx = idx; - p->im_off_next = next; - return GIT_SUCCESS; -} + if (!pack_version_ok(hdr.hdr_version)) + goto cleanup; -static int idxv1_search(uint32_t *out, git_pack *p, const git_oid *id) -{ - unsigned char *data = p->im_oid; - uint32_t lo = id->id[0] ? p->im_fanout[id->id[0] - 1] : 0; - uint32_t hi = p->im_fanout[id->id[0]]; + /* Verify the pack matches its index. */ + if (p->num_objects != ntohl(hdr.hdr_entries)) + goto cleanup; - do { - uint32_t mid = (lo + hi) >> 1; - uint32_t pos = 24 * mid; - int cmp = memcmp(id->id, data + pos + 4, 20); - if (cmp < 0) - hi = mid; - else if (!cmp) { - *out = mid; - return GIT_SUCCESS; - } else - lo = mid + 1; - } while (lo < hi); - return GIT_ENOTFOUND; -} + if (gitfo_lseek(p->pack_fd, p->pack_size - GIT_OID_RAWSZ, SEEK_SET) == -1) + goto cleanup; -static int idxv1_search_offset(uint32_t *out, git_pack *p, git_off_t offset) -{ - if (offset > 0 && offset < (p->pack_size - GIT_OID_RAWSZ)) { - uint32_t lo = 0, hi = p->obj_cnt+1; - unsigned char *data = p->im_oid; - uint32_t *idx = p->im_off_idx; - do { - uint32_t mid = (lo + hi) >> 1; - uint32_t n = idx[mid]; - uint32_t pos = n * (GIT_OID_RAWSZ + 4); - git_off_t here = decode32(data + pos); - if (offset < here) - hi = mid; - else if (offset == here) { - *out = n; - return GIT_SUCCESS; - } else - lo = mid + 1; - } while (lo < hi); - } - return GIT_ENOTFOUND; -} + if (gitfo_read(p->pack_fd, sha1.id, GIT_OID_RAWSZ) < GIT_SUCCESS) + goto cleanup; -static int idxv1_get(index_entry *e, git_pack *p, uint32_t n) -{ - unsigned char *data = p->im_oid; - uint32_t *next = p->im_off_next; - - if (n < p->obj_cnt) { - uint32_t pos = n * (GIT_OID_RAWSZ + 4); - git_off_t next_off = p->pack_size - GIT_OID_RAWSZ; - e->n = n; - e->oid = data + pos + 4; - e->offset = decode32(data + pos); - if (next[n] < p->obj_cnt) { - pos = next[n] * (GIT_OID_RAWSZ + 4); - next_off = decode32(data + pos); - } - e->size = next_off - e->offset; - return GIT_SUCCESS; - } - return GIT_ENOTFOUND; + idx_sha1 = ((unsigned char *)p->index_map.data) + p->index_map.len - 40; + + if (git_oid_cmp(&sha1, (git_oid *)idx_sha1) != 0) + goto cleanup; + + return GIT_SUCCESS; + +cleanup: + gitfo_close(p->pack_fd); + p->pack_fd = -1; + return GIT_EPACKCORRUPTED; } -static int pack_openidx_v1(git_pack *p) +static int packfile_check(struct pack_file **pack_out, const char *path, int local) { - uint32_t *src_fanout = p->idx_map.data; - uint32_t *im_fanout; - offset_idx_info *info; - size_t expsz; - uint32_t j; - - - if ((im_fanout = git__malloc(sizeof(*im_fanout) * 256)) == NULL) - return GIT_ERROR; - - im_fanout[0] = decode32(&src_fanout[0]); - for (j = 1; j < 256; j++) { - im_fanout[j] = decode32(&src_fanout[j]); - if (im_fanout[j] < im_fanout[j - 1]) { - free(im_fanout); - return GIT_ERROR; - } - } - p->obj_cnt = im_fanout[255]; + struct stat st; + struct pack_file *p; + size_t path_len; + + *pack_out = NULL; + path_len = strlen(path); + p = packfile_alloc(path_len + 2); - expsz = 4 * 256 + 24 * p->obj_cnt + 2 * 20; - if (expsz != p->idx_map.len) { - free(im_fanout); - return GIT_ERROR; + /* + * Make sure a corresponding .pack file exists and that + * the index looks sane. + */ + path_len -= STRLEN(".idx"); + if (path_len < 1) { + free(p); + return GIT_ENOTFOUND; } - p->idx_search = idxv1_search; - p->idx_search_offset = idxv1_search_offset; - p->idx_get = idxv1_get; - p->im_fanout = im_fanout; - p->im_oid = (unsigned char *)(src_fanout + 256); + memcpy(p->pack_name, path, path_len); - if ((info = git__malloc(sizeof(*info) * (p->obj_cnt+1))) == NULL) { - free(im_fanout); - return GIT_ERROR; - } + strcpy(p->pack_name + path_len, ".keep"); + if (gitfo_exists(p->pack_name) == GIT_SUCCESS) + p->pack_keep = 1; - for (j = 0; j < p->obj_cnt; j++) { - uint32_t pos = j * (GIT_OID_RAWSZ + 4); - info[j].offset = decode32(p->im_oid + pos); - info[j].n = j; + strcpy(p->pack_name + path_len, ".pack"); + if (gitfo_stat(p->pack_name, &st) < GIT_SUCCESS || !S_ISREG(st.st_mode)) { + free(p); + return GIT_ENOTFOUND; } - info[p->obj_cnt].offset = p->pack_size - GIT_OID_RAWSZ; - info[p->obj_cnt].n = p->obj_cnt; - if (make_offset_index(p, info)) { - free(im_fanout); - free(info); - return GIT_ERROR; - } - free(info); + /* ok, it looks sane as far as we can check without + * actually mapping the pack file. + */ + p->pack_size = (off_t)st.st_size; + p->pack_local = local; + p->mtime = st.st_mtime; + /* see if we can parse the sha1 oid in the packfile name */ + if (path_len < 40 || + git_oid_mkstr(&p->sha1, path + path_len - GIT_OID_HEXSZ) < GIT_SUCCESS) + memset(&p->sha1, 0x0, GIT_OID_RAWSZ); + + *pack_out = p; return GIT_SUCCESS; } -static int idxv2_search(uint32_t *out, git_pack *p, const git_oid *id) +static int packfile_load__cb(void *_data, char *path) { - unsigned char *data = p->im_oid; - uint32_t lo = id->id[0] ? p->im_fanout[id->id[0] - 1] : 0; - uint32_t hi = p->im_fanout[id->id[0]]; + struct pack__dirent *data = (struct pack__dirent *)_data; + struct pack_file *pack; + int error; - do { - uint32_t mid = (lo + hi) >> 1; - uint32_t pos = 20 * mid; - int cmp = memcmp(id->id, data + pos, 20); - if (cmp < 0) - hi = mid; - else if (!cmp) { - *out = mid; - return GIT_SUCCESS; - } else - lo = mid + 1; - } while (lo < hi); - return GIT_ENOTFOUND; -} + if (git__suffixcmp(path, ".idx") != 0) + return GIT_SUCCESS; /* not an index */ -static int idxv2_search_offset(uint32_t *out, git_pack *p, git_off_t offset) -{ - if (offset > 0 && offset < (p->pack_size - GIT_OID_RAWSZ)) { - uint32_t lo = 0, hi = p->obj_cnt+1; - uint32_t *idx = p->im_off_idx; - do { - uint32_t mid = (lo + hi) >> 1; - uint32_t n = idx[mid]; - uint32_t o32 = decode32(p->im_offset32 + n); - git_off_t here = o32; - - if (o32 & 0x80000000) { - uint32_t o64_idx = (o32 & ~0x80000000); - here = decode64(p->im_offset64 + 2*o64_idx); - } + /* FIXME: git.git checks for duplicate packs. + * But that makes no fucking sense. Our dirent is not + * going to generate dupicate entries */ - if (offset < here) - hi = mid; - else if (offset == here) { - *out = n; - return GIT_SUCCESS; - } else - lo = mid + 1; - } while (lo < hi); - } - return GIT_ENOTFOUND; -} + error = packfile_check(&pack, path, data->is_pack_local); + if (error < GIT_SUCCESS) + return error; -static int idxv2_get(index_entry *e, git_pack *p, uint32_t n) -{ - unsigned char *data = p->im_oid; - uint32_t *next = p->im_off_next; - - if (n < p->obj_cnt) { - uint32_t o32 = decode32(p->im_offset32 + n); - git_off_t next_off = p->pack_size - GIT_OID_RAWSZ; - e->n = n; - e->oid = data + n * GIT_OID_RAWSZ; - e->offset = o32; - if (o32 & 0x80000000) { - uint32_t o64_idx = (o32 & ~0x80000000); - e->offset = decode64(p->im_offset64 + 2*o64_idx); - } - if (next[n] < p->obj_cnt) { - o32 = decode32(p->im_offset32 + next[n]); - next_off = o32; - if (o32 & 0x80000000) { - uint32_t o64_idx = (o32 & ~0x80000000); - next_off = decode64(p->im_offset64 + 2*o64_idx); - } - } - e->size = next_off - e->offset; - return GIT_SUCCESS; + if (git_vector_insert(&data->backend->packs, pack) < GIT_SUCCESS) { + free(pack); + return GIT_ENOMEM; } - return GIT_ENOTFOUND; + + return GIT_SUCCESS; } -static int pack_openidx_v2(git_pack *p) +static int packfile_load_all(struct pack_backend *backend, const char *odb_path, int local) { - unsigned char *data = p->idx_map.data; - uint32_t *src_fanout = (uint32_t *)(data + 8); - uint32_t *im_fanout; - offset_idx_info *info; - size_t sz, o64_sz, o64_len; - uint32_t j; - - if ((im_fanout = git__malloc(sizeof(*im_fanout) * 256)) == NULL) - return GIT_ERROR; - - im_fanout[0] = decode32(&src_fanout[0]); - for (j = 1; j < 256; j++) { - im_fanout[j] = decode32(&src_fanout[j]); - if (im_fanout[j] < im_fanout[j - 1]) { - free(im_fanout); - return GIT_ERROR; - } - } - p->obj_cnt = im_fanout[255]; + int error; + char path[GIT_PATH_MAX]; + struct pack__dirent data; - /* minimum size of .idx file (with empty 64-bit offsets table): */ - sz = 4 + 4 + 256 * 4 + p->obj_cnt * (20 + 4 + 4) + 2 * 20; - if (p->idx_map.len < sz) { - free(im_fanout); - return GIT_ERROR; - } + data.backend = backend; + data.is_pack_local = local; - p->idx_search = idxv2_search; - p->idx_search_offset = idxv2_search_offset; - p->idx_get = idxv2_get; - p->im_fanout = im_fanout; - p->im_oid = (unsigned char *)(src_fanout + 256); - p->im_crc = (uint32_t *)(p->im_oid + 20 * p->obj_cnt); - p->im_offset32 = p->im_crc + p->obj_cnt; - p->im_offset64 = p->im_offset32 + p->obj_cnt; - - if ((info = git__malloc(sizeof(*info) * (p->obj_cnt+1))) == NULL) { - free(im_fanout); - return GIT_ERROR; - } + git__joinpath(path, odb_path, "pack"); + if (gitfo_isdir(path) < GIT_SUCCESS) + return GIT_SUCCESS; - /* check 64-bit offset table index values are within bounds */ - o64_sz = p->idx_map.len - sz; - o64_len = o64_sz / 8; - for (j = 0; j < p->obj_cnt; j++) { - uint32_t o32 = decode32(p->im_offset32 + j); - git_off_t offset = o32; - if (o32 & 0x80000000) { - uint32_t o64_idx = (o32 & ~0x80000000); - if (o64_idx >= o64_len) { - free(im_fanout); - free(info); - return GIT_ERROR; - } - offset = decode64(p->im_offset64 + 2*o64_idx); - } - info[j].offset = offset; - info[j].n = j; - } - info[p->obj_cnt].offset = p->pack_size - GIT_OID_RAWSZ; - info[p->obj_cnt].n = p->obj_cnt; + error = gitfo_dirent(path, GIT_PATH_MAX, packfile_load__cb, (void *)&data); + if (error < GIT_SUCCESS) + return error; - if (make_offset_index(p, info)) { - free(im_fanout); - free(info); - return GIT_ERROR; - } - free(info); + git_vector_sort(&backend->packs); + backend->last_found = git_vector_get(&backend->packs, 0); return GIT_SUCCESS; } @@ -898,221 +895,421 @@ static int pack_openidx_v2(git_pack *p) /*********************************************************** * - * PACKFILE READING FUNCTIONS - * - * Read the contents of a packfile + * PACKFILE ENTRY SEARCH INTERNALS * ***********************************************************/ +static off_t nth_packed_object_offset(const struct pack_file *p, uint32_t n) +{ + const unsigned char *index = p->index_map.data; + index += 4 * 256; + if (p->index_version == 1) { + return ntohl(*((uint32_t *)(index + 24 * n))); + } else { + uint32_t off; + index += 8 + p->num_objects * (20 + 4); + off = ntohl(*((uint32_t *)(index + 4 * n))); + if (!(off & 0x80000000)) + return off; + index += p->num_objects * 4 + (off & 0x7fffffff) * 8; + return (((uint64_t)ntohl(*((uint32_t *)(index + 0)))) << 32) | + ntohl(*((uint32_t *)(index + 4))); + } +} -static int unpack_object(git_rawobj *out, git_pack *p, index_entry *e); - -static int unpack_object_delta(git_rawobj *out, git_pack *p, - index_entry *base_entry, - uint8_t *delta_buffer, - size_t delta_deflated_size, - size_t delta_inflated_size) +static int pack_entry_find_offset( + off_t *offset_out, + struct pack_file *p, + const git_oid *oid) { - int res = 0; - uint8_t *delta = NULL; - git_rawobj base_obj; + const uint32_t *level1_ofs = p->index_map.data; + const unsigned char *index = p->index_map.data; + unsigned hi, lo, stride; - base_obj.data = NULL; - base_obj.type = GIT_OBJ_BAD; - base_obj.len = 0; + *offset_out = 0; - if ((res = unpack_object(&base_obj, p, base_entry)) < 0) - goto cleanup; + if (index == NULL) { + int error; - delta = git__malloc(delta_inflated_size + 1); + if ((error = pack_index_open(p)) < GIT_SUCCESS) + return error; - if ((res = git_odb__inflate_buffer(delta_buffer, delta_deflated_size, - delta, delta_inflated_size)) < 0) - goto cleanup; + assert(p->index_map.data); - res = git__delta_apply(out, base_obj.data, base_obj.len, delta, delta_inflated_size); + index = p->index_map.data; + level1_ofs = p->index_map.data; + } - out->type = base_obj.type; + if (p->index_version > 1) { + level1_ofs += 2; + index += 8; + } -cleanup: - free(delta); - git_rawobj_close(&base_obj); - return res; -} + index += 4 * 256; + hi = ntohl(level1_ofs[(int)oid->id[0]]); + lo = ((oid->id[0] == 0x0) ? 0 : ntohl(level1_ofs[(int)oid->id[0] - 1])); -static int unpack_object(git_rawobj *out, git_pack *p, index_entry *e) -{ - git_otype object_type; - size_t inflated_size, deflated_size, shift; - uint8_t *buffer, byte; + if (p->index_version > 1) { + stride = 20; + } else { + stride = 24; + index += 4; + } - assert(out && p && e && git__is_sizet(e->size)); +#ifdef INDEX_DEBUG_LOOKUP + printf("%02x%02x%02x... lo %u hi %u nr %d\n", + oid->id[0], oid->id[1], oid->id[2], lo, hi, p->num_objects); +#endif - if (open_pack(p)) - return GIT_ERROR; +#ifdef GIT2_INDEX_LOOKUP /* TODO: use the advanced lookup method from git.git */ - buffer = (uint8_t *)p->pack_map.data + e->offset; - deflated_size = (size_t)e->size; + int pos = sha1_entry_pos(index, stride, 0, lo, hi, p->num_objects, oid); + if (pos < 0) + return GIT_ENOTFOUND; - if (deflated_size == 0) - deflated_size = (size_t)(p->pack_size - e->offset); + *offset_out = nth_packed_object_offset(p, pos); + return GIT_SUCCESS; - byte = *buffer++ & 0xFF; - deflated_size--; - object_type = (byte >> 4) & 0x7; - inflated_size = byte & 0xF; - shift = 4; +#else /* use an old and boring binary search */ - while (byte & 0x80) { - byte = *buffer++ & 0xFF; - deflated_size--; - inflated_size += (byte & 0x7F) << shift; - shift += 7; - } + do { + unsigned mi = (lo + hi) / 2; + int cmp = memcmp(index + mi * stride, oid->id, GIT_OID_RAWSZ); - switch (object_type) { - case GIT_OBJ_COMMIT: - case GIT_OBJ_TREE: - case GIT_OBJ_BLOB: - case GIT_OBJ_TAG: { + if (!cmp) { + *offset_out = nth_packed_object_offset(p, mi); + return GIT_SUCCESS; + } - /* Handle a normal zlib stream */ - out->len = inflated_size; - out->type = object_type; - out->data = git__malloc(inflated_size + 1); + if (cmp > 0) + hi = mi; + else + lo = mi+1; - if (git_odb__inflate_buffer(buffer, deflated_size, out->data, out->len) < 0) { - free(out->data); - out->data = NULL; + } while (lo < hi); + + return GIT_ENOTFOUND; +#endif +} + +static int pack_entry_find1( + struct pack_entry *e, + struct pack_file *p, + const git_oid *oid) +{ + off_t offset; + + assert(p); + + if (p->num_bad_objects) { + unsigned i; + for (i = 0; i < p->num_bad_objects; i++) + if (git_oid_cmp(oid, &p->bad_object_sha1[i]) == 0) return GIT_ERROR; - } + } - return GIT_SUCCESS; - } + if (pack_entry_find_offset(&offset, p, oid) < GIT_SUCCESS) + return GIT_ENOTFOUND; + + /* we found an entry in the index; + * make sure the packfile backing the index + * still exists on disk */ + if (p->pack_fd == -1 && packfile_open(p) < GIT_SUCCESS) + return GIT_EOSERR; - case GIT_OBJ_OFS_DELTA: { + e->offset = offset; + e->p = p; - git_off_t delta_offset; - index_entry entry; + git_oid_cpy(&e->sha1, oid); + return GIT_SUCCESS; +} - byte = *buffer++ & 0xFF; - delta_offset = byte & 0x7F; +static int pack_entry_find(struct pack_entry *e, struct pack_backend *backend, const git_oid *oid) +{ + size_t i; - while (byte & 0x80) { - delta_offset += 1; - byte = *buffer++ & 0xFF; - delta_offset <<= 7; - delta_offset += (byte & 0x7F); - } + if (backend->last_found && + pack_entry_find1(e, backend->last_found, oid) == GIT_SUCCESS) + return GIT_SUCCESS; - entry.n = 0; - entry.oid = NULL; - entry.offset = e->offset - delta_offset; - entry.size = 0; + for (i = 0; i < backend->packs.length; ++i) { + struct pack_file *p; - if (unpack_object_delta(out, p, &entry, - buffer, deflated_size, inflated_size) < 0) - return GIT_ERROR; + p = git_vector_get(&backend->packs, i); + if (p == backend->last_found) + continue; + if (pack_entry_find1(e, p, oid) == GIT_SUCCESS) { + backend->last_found = p; return GIT_SUCCESS; } + } + + return GIT_ENOTFOUND; +} + + - case GIT_OBJ_REF_DELTA: { - git_oid base_id; - uint32_t n; - index_entry entry; - int res = GIT_ERROR; - git_oid_mkraw(&base_id, buffer); - if (!p->idx_search(&n, p, &base_id) && - !p->idx_get(&entry, p, n)) { - res = unpack_object_delta(out, p, &entry, - buffer + GIT_OID_RAWSZ, deflated_size, inflated_size); - } - return res; - } - default: - return GIT_EOBJCORRUPTED; - } -} -static int read_packed(git_rawobj *out, const pack_location *loc) -{ - index_entry e; - int res; - assert(out && loc); - if (pack_openidx(loc->ptr) < 0) - return GIT_EPACKCORRUPTED; +/*********************************************************** + * + * PACKFILE ENTRY UNPACK INTERNALS + * + ***********************************************************/ - res = loc->ptr->idx_get(&e, loc->ptr, loc->n); +static unsigned long packfile_unpack_header1( + size_t *sizep, + git_otype *type, + const unsigned char *buf, + unsigned long len) +{ + unsigned shift; + unsigned long size, c; + unsigned long used = 0; - if (!res) - res = unpack_object(out, loc->ptr, &e); + c = buf[used++]; + *type = (c >> 4) & 7; + size = c & 15; + shift = 4; + while (c & 0x80) { + if (len <= used || bitsizeof(long) <= shift) + return 0; - pack_decidx(loc->ptr); + c = buf[used++]; + size += (c & 0x7f) << shift; + shift += 7; + } - return res; + *sizep = (size_t)size; + return used; } -static int read_header_packed(git_rawobj *out, const pack_location *loc) +static int packfile_unpack_header( + size_t *size_p, + git_otype *type_p, + struct pack_backend *backend, + struct pack_file *p, + struct pack_window **w_curs, + off_t *curpos) { - git_pack *pack; - index_entry e; - int error = GIT_SUCCESS, shift; - uint8_t *buffer, byte; + unsigned char *base; + unsigned int left; + unsigned long used; + + /* pack_window_open() assures us we have [base, base + 20) available + * as a range that we can look at at. (Its actually the hash + * size that is assured.) With our object header encoding + * the maximum deflated object size is 2^137, which is just + * insane, so we know won't exceed what we have been given. + */ + base = pack_window_open(backend, p, w_curs, *curpos, &left); + if (base == NULL) + return GIT_ENOMEM; - assert(out && loc); + used = packfile_unpack_header1(size_p, type_p, base, left); - pack = loc->ptr; + if (used == 0) + return GIT_EOBJCORRUPTED; - if (pack_openidx(pack)) - return GIT_EPACKCORRUPTED; + *curpos += used; + return GIT_SUCCESS; +} - if (pack->idx_get(&e, pack, loc->n) < 0 || - open_pack(pack) < 0) { - error = GIT_ENOTFOUND; - goto cleanup; +static int packfile_unpack_compressed( + git_rawobj *obj, + struct pack_backend *backend, + struct pack_file *p, + struct pack_window **w_curs, + off_t curpos, + size_t size, + git_otype type) +{ + int st; + z_stream stream; + unsigned char *buffer, *in; + + buffer = git__malloc(size); + + memset(&stream, 0, sizeof(stream)); + stream.next_out = buffer; + stream.avail_out = size + 1; + + st = inflateInit(&stream); + if (st != Z_OK) { + free(buffer); + return GIT_EZLIB; } - buffer = (uint8_t *)pack->pack_map.data + e.offset; + do { + in = pack_window_open(backend, p, w_curs, curpos, &stream.avail_in); + stream.next_in = in; + st = inflate(&stream, Z_FINISH); - byte = *buffer++ & 0xFF; - out->type = (byte >> 4) & 0x7; - out->len = byte & 0xF; - shift = 4; + if (!stream.avail_out) + break; /* the payload is larger than it should be */ - while (byte & 0x80) { - byte = *buffer++ & 0xFF; - out->len += (byte & 0x7F) << shift; - shift += 7; + curpos += stream.next_in - in; + } while (st == Z_OK || st == Z_BUF_ERROR); + + inflateEnd(&stream); + + if ((st != Z_STREAM_END) || stream.total_out != size) { + free(buffer); + return GIT_EZLIB; } - /* - * FIXME: if the object is not packed as a whole, - * we need to do a full load and apply the deltas before - * being able to read the header. - * - * I don't think there are any workarounds for this.' + obj->type = type; + obj->len = size; + obj->data = buffer; + return GIT_SUCCESS; +} + +static off_t get_delta_base( + struct pack_backend *backend, + struct pack_file *p, + struct pack_window **w_curs, + off_t *curpos, + git_otype type, + off_t delta_obj_offset) +{ + unsigned char *base_info = pack_window_open(backend, p, w_curs, *curpos, NULL); + off_t base_offset; + + /* pack_window_open() assured us we have [base_info, base_info + 20) + * as a range that we can look at without walking off the + * end of the mapped window. Its actually the hash size + * that is assured. An OFS_DELTA longer than the hash size + * is stupid, as then a REF_DELTA would be smaller to store. */ + if (type == GIT_OBJ_OFS_DELTA) { + unsigned used = 0; + unsigned char c = base_info[used++]; + base_offset = c & 127; + while (c & 128) { + base_offset += 1; + if (!base_offset || MSB(base_offset, 7)) + return 0; /* overflow */ + c = base_info[used++]; + base_offset = (base_offset << 7) + (c & 127); + } + base_offset = delta_obj_offset - base_offset; + if (base_offset <= 0 || base_offset >= delta_obj_offset) + return 0; /* out of bound */ + *curpos += used; + } else if (type == GIT_OBJ_REF_DELTA) { + /* The base entry _must_ be in the same pack */ + if (pack_entry_find_offset(&base_offset, p, (git_oid *)base_info) < GIT_SUCCESS) + return GIT_EPACKCORRUPTED; + *curpos += 20; + } else + return 0; - if (out->type == GIT_OBJ_OFS_DELTA || out->type == GIT_OBJ_REF_DELTA) { - error = unpack_object(out, pack, &e); - git_rawobj_close(out); + return base_offset; +} + +static int packfile_unpack_delta( + git_rawobj *obj, + struct pack_backend *backend, + struct pack_file *p, + struct pack_window **w_curs, + off_t curpos, + size_t delta_size, + git_otype delta_type, + off_t obj_offset) +{ + off_t base_offset; + git_rawobj base, delta; + int error; + + base_offset = get_delta_base(backend, p, w_curs, &curpos, delta_type, obj_offset); + if (base_offset == 0) + return GIT_EOBJCORRUPTED; + + pack_window_close(w_curs); + error = packfile_unpack(&base, backend, p, base_offset); + + /* TODO: git.git tries to load the base from other packfiles + * or loose objects */ + if (error < GIT_SUCCESS) + return error; + + error = packfile_unpack_compressed(&delta, backend, p, w_curs, curpos, delta_size, delta_type); + if (error < GIT_SUCCESS) { + git_rawobj_close(&base); + return error; } -cleanup: - pack_decidx(loc->ptr); + obj->type = base.type; + error = git__delta_apply(obj, + base.data, base.len, + delta.data, delta.len); + + git_rawobj_close(&base); + git_rawobj_close(&delta); + + /* TODO: we might want to cache this shit. eventually */ + //add_delta_base_cache(p, base_offset, base, base_size, *type); return error; } +static int packfile_unpack( + git_rawobj *obj, + struct pack_backend *backend, + struct pack_file *p, + off_t obj_offset) +{ + struct pack_window *w_curs = NULL; + off_t curpos = obj_offset; + int error; + + size_t size; + git_otype type; + /* + * TODO: optionally check the CRC on the packfile + */ + + obj->data = NULL; + obj->len = 0; + obj->type = GIT_OBJ_BAD; + + error = packfile_unpack_header(&size, &type, backend, p, &w_curs, &curpos); + if (error < GIT_SUCCESS) + return error; + + switch (type) { + case GIT_OBJ_OFS_DELTA: + case GIT_OBJ_REF_DELTA: + error = packfile_unpack_delta( + obj, backend, p, &w_curs, curpos, + size, type, obj_offset); + break; + + case GIT_OBJ_COMMIT: + case GIT_OBJ_TREE: + case GIT_OBJ_BLOB: + case GIT_OBJ_TAG: + error = packfile_unpack_compressed( + obj, backend, p, &w_curs, curpos, + size, type); + break; + + default: + error = GIT_EOBJCORRUPTED; + break; + } + + pack_window_close(&w_curs); + return error; +} @@ -1126,79 +1323,80 @@ cleanup: * ***********************************************************/ +/* int pack_backend__read_header(git_rawobj *obj, git_odb_backend *backend, const git_oid *oid) { pack_location location; assert(obj && backend && oid); - if (locate_packfile(&location, (pack_backend *)backend, oid) < 0) + if (locate_packfile(&location, (struct pack_backend *)backend, oid) < 0) return GIT_ENOTFOUND; return read_header_packed(obj, &location); } +*/ int pack_backend__read(git_rawobj *obj, git_odb_backend *backend, const git_oid *oid) { - pack_location location; - - assert(obj && backend && oid); + struct pack_entry e; + int error; - if (locate_packfile(&location, (pack_backend *)backend, oid) < 0) - return GIT_ENOTFOUND; + if ((error = pack_entry_find(&e, (struct pack_backend *)backend, oid)) < GIT_SUCCESS) + return error; - return read_packed(obj, &location); + return packfile_unpack(obj, (struct pack_backend *)backend, e.p, e.offset); } int pack_backend__exists(git_odb_backend *backend, const git_oid *oid) { - pack_location location; - assert(backend && oid); - return locate_packfile(&location, (pack_backend *)backend, oid) == GIT_SUCCESS; + struct pack_entry e; + return pack_entry_find(&e, (struct pack_backend *)backend, oid) == GIT_SUCCESS; } void pack_backend__free(git_odb_backend *_backend) { - pack_backend *backend; - git_packlist *pl; + struct pack_backend *backend; + size_t i; assert(_backend); - backend = (pack_backend *)_backend; - - gitlck_lock(&backend->lock); - - pl = backend->packlist; - backend->packlist = NULL; + backend = (struct pack_backend *)_backend; - gitlck_unlock(&backend->lock); - if (pl) - packlist_dec(backend, pl); - - gitlck_free(&backend->lock); + for (i = 0; i < backend->packs.length; ++i) { + struct pack_file *p = git_vector_get(&backend->packs, i); + packfile_free(backend, p); + } - free(backend->objects_dir); + git_vector_free(&backend->packs); free(backend); } int git_odb_backend_pack(git_odb_backend **backend_out, const char *objects_dir) { - pack_backend *backend; + int error; + struct pack_backend *backend; - backend = git__calloc(1, sizeof(pack_backend)); + backend = git__calloc(1, sizeof(struct pack_backend)); if (backend == NULL) return GIT_ENOMEM; - backend->objects_dir = git__strdup(objects_dir); - if (backend->objects_dir == NULL) { + if (git_vector_init(&backend->packs, 8, packfile_sort__cb) < GIT_SUCCESS) { free(backend); return GIT_ENOMEM; } - gitlck_init(&backend->lock); + backend->window_size = DEFAULT_WINDOW_SIZE; + backend->mapped_limit = DEFAULT_MAPPED_LIMIT; + + error = packfile_load_all(backend, objects_dir, 1); + if (error < GIT_SUCCESS) { + pack_backend__free((git_odb_backend *)backend); + return error; + } backend->parent.read = &pack_backend__read; - backend->parent.read_header = &pack_backend__read_header; + backend->parent.read_header = NULL; backend->parent.write = NULL; backend->parent.exists = &pack_backend__exists; backend->parent.free = &pack_backend__free; diff --git a/src/util.h b/src/util.h index fab93c1c2..e0dfd7b20 100644 --- a/src/util.h +++ b/src/util.h @@ -2,6 +2,8 @@ #define INCLUDE_util_h__ #define ARRAY_SIZE(x) (sizeof(x)/sizeof(x[0])) +#define bitsizeof(x) (CHAR_BIT * sizeof(x)) +#define MSB(x, bits) ((x) & (~0ULL << (bitsizeof(x) - (bits)))) /* * Don't wrap malloc/calloc. -- cgit v1.2.1 From 005718280712634486a097427212e652b0e29f36 Mon Sep 17 00:00:00 2001 From: Vicent Marti Date: Sat, 12 Mar 2011 16:04:46 +0200 Subject: Add new method `git_reference_listall` Lists all the references in a repository. Listing may be filtered by reference type. This should applease Lord Clem. --- src/refs.c | 82 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) (limited to 'src') diff --git a/src/refs.c b/src/refs.c index 2fc383e22..93897a7f9 100644 --- a/src/refs.c +++ b/src/refs.c @@ -235,6 +235,24 @@ static int loose_read(gitfo_buf *file_content, const char *name, const char *rep return error; } +static git_rtype loose_guess_rtype(const char *full_path) +{ + gitfo_buf ref_file = GITFO_BUF_INIT; + git_rtype type; + + type = GIT_REF_INVALID; + + if (gitfo_read_file(&ref_file, full_path) == GIT_SUCCESS) { + if (git__prefixcmp((const char *)(ref_file.data), GIT_SYMREF) == 0) + type = GIT_REF_SYMBOLIC; + else + type = GIT_REF_OID; + } + + gitfo_free_buf(&ref_file); + return type; +} + static int loose_lookup( git_reference **ref_out, git_repository *repo, @@ -531,6 +549,31 @@ cleanup: return error; } + + + +struct dirent_list_data { + git_vector ref_list; + size_t repo_path_len; + unsigned int list_flags; +}; + +static int _dirent_loose_listall(void *_data, char *full_path) +{ + struct dirent_list_data *data = (struct dirent_list_data *)_data; + char *file_path; + + if (gitfo_isdir(full_path) == GIT_SUCCESS) + return gitfo_dirent(full_path, GIT_PATH_MAX, _dirent_loose_listall, _data); + + if ((data->list_flags & loose_guess_rtype(full_path)) == 0) + return GIT_SUCCESS; /* we are filtering out this reference */ + + file_path = full_path + data->repo_path_len; + + return git_vector_insert(&data->ref_list, git__strdup(file_path)); +} + static int _dirent_loose_load(void *data, char *full_path) { git_repository *repository = (git_repository *)data; @@ -1292,6 +1335,45 @@ int git_reference_packall(git_repository *repo) return packed_write(repo); } +int git_reference_listall(git_strarray *array, git_repository *repo, unsigned int list_flags) +{ + int error; + struct dirent_list_data data; + char refs_path[GIT_PATH_MAX]; + + array->strings = NULL; + array->count = 0; + + git_vector_init(&data.ref_list, 8, NULL); + data.repo_path_len = strlen(repo->path_repository); + data.list_flags = list_flags; + + git__joinpath(refs_path, repo->path_repository, GIT_REFS_DIR); + error = gitfo_dirent(refs_path, GIT_PATH_MAX, _dirent_loose_listall, &data); + + if (error < GIT_SUCCESS) { + git_vector_free(&data.ref_list); + return error; + } + + if (list_flags & GIT_REF_PACKED) { + const char *ref_name; + void *_unused; + + if ((error = packed_load(repo)) < GIT_SUCCESS) { + git_vector_free(&data.ref_list); + return error; + } + + GIT_HASHTABLE_FOREACH(repo->references.packfile, ref_name, _unused, + git_vector_insert(&data.ref_list, git__strdup(ref_name)); + ); + } + + array->strings = (char **)data.ref_list.contents; + array->count = data.ref_list.length; + return GIT_SUCCESS; +} -- cgit v1.2.1 From 6b2a19418cb08e7bccefc2362f290a67555594a0 Mon Sep 17 00:00:00 2001 From: Vicent Marti Date: Sat, 12 Mar 2011 23:09:16 +0200 Subject: Fix the retarded object interdependency system It's no longer retarded. All object interdependencies are stored as OIDs instead of actual objects. This should be hundreds of times faster, specially on big repositories. Heck, who knows, maye it doesn't even segfault -- wouldn't that be awesome? What has changed on the API? `git_commit_parent`, `git_commit_tree`, `git_tag_target` now return their values through a pointer-to-pointer, and have an error code. `git_commit_set_tree` and `git_tag_set_target` now return an error code and may fail. `git_repository_free__no_gc` has been deprecated because it's stupid. Since there are no longer any interdependencies between objects, we don't need internal reference counting, and GC never fails or double-free's pointers. `git_object_close` now does a very sane thing: marks an object as unused. Closed objects will be eventually free'd from the object cache based on LRU. Please use `git_object_close` from the garbage collector `destroy` method on your bindings. It's 100% safe. `git_repository_gc` is a new method that forces a garbage collector pass through the repo, to free as many LRU objects as possible. This is useful if we are running out of memory. --- src/blob.c | 4 +- src/commit.c | 202 ++++++++++++++++++++----------------------------------- src/commit.h | 7 +- src/object.c | 22 +++--- src/refs.c | 7 +- src/repository.c | 65 +++++++----------- src/repository.h | 22 ++---- src/tag.c | 36 +++++----- src/tag.h | 3 +- 9 files changed, 133 insertions(+), 235 deletions(-) (limited to 'src') diff --git a/src/blob.c b/src/blob.c index f157f4787..1e03b6b67 100644 --- a/src/blob.c +++ b/src/blob.c @@ -130,9 +130,7 @@ int git_blob_writefile(git_oid *written_id, git_repository *repo, const char *pa git_oid_cpy(written_id, git_object_id((git_object *)blob)); - /* FIXME: maybe we don't want to free this already? - * the user may want to access it again */ - GIT_OBJECT_DECREF(repo, blob); + git_object_close((git_object*)blob); return GIT_SUCCESS; } diff --git a/src/commit.c b/src/commit.c index 974999a4a..1c5cddf7a 100644 --- a/src/commit.c +++ b/src/commit.c @@ -46,24 +46,22 @@ static void clear_parents(git_commit *commit) { unsigned int i; - for (i = 0; i < commit->parents.length; ++i) { - git_commit *parent = git_vector_get(&commit->parents, i); - GIT_OBJECT_DECREF(commit->object.repo, parent); + for (i = 0; i < commit->parent_oids.length; ++i) { + git_oid *parent = git_vector_get(&commit->parent_oids, i); + free(parent); } - git_vector_clear(&commit->parents); + git_vector_clear(&commit->parent_oids); } void git_commit__free(git_commit *commit) { clear_parents(commit); - git_vector_free(&commit->parents); + git_vector_free(&commit->parent_oids); git_signature_free(commit->author); git_signature_free(commit->committer); - GIT_OBJECT_DECREF(commit->object.repo, commit->tree); - free(commit->message); free(commit->message_short); free(commit); @@ -78,16 +76,13 @@ int git_commit__writeback(git_commit *commit, git_odb_source *src) { unsigned int i; - if (commit->tree == NULL) - return GIT_EMISSINGOBJDATA; - - git__write_oid(src, "tree", git_tree_id(commit->tree)); + git__write_oid(src, "tree", &commit->tree_oid); - for (i = 0; i < commit->parents.length; ++i) { - git_commit *parent; + for (i = 0; i < commit->parent_oids.length; ++i) { + git_oid *parent_oid; - parent = git_vector_get(&commit->parents, i); - git__write_oid(src, "parent", git_commit_id(parent)); + parent_oid = git_vector_get(&commit->parent_oids, i); + git__write_oid(src, "parent", parent_oid); } if (commit->author == NULL) @@ -103,66 +98,49 @@ int git_commit__writeback(git_commit *commit, git_odb_source *src) if (commit->message != NULL) { git__source_write(src, "\n", 1); git__source_write(src, commit->message, strlen(commit->message)); - } - - /* Mark the commit as having all attributes */ - commit->full_parse = 1; + } return GIT_SUCCESS; } -int commit_parse_buffer(git_commit *commit, void *data, size_t len, unsigned int parse_flags) +int commit_parse_buffer(git_commit *commit, void *data, size_t len) { char *buffer = (char *)data; const char *buffer_end = (char *)data + len; - git_oid oid; + git_oid parent_oid; int error; /* first parse; the vector hasn't been initialized yet */ - if (commit->parents.contents == NULL) { - git_vector_init(&commit->parents, 4, NULL); + if (commit->parent_oids.contents == NULL) { + git_vector_init(&commit->parent_oids, 4, NULL); } clear_parents(commit); - - if ((error = git__parse_oid(&oid, &buffer, buffer_end, "tree ")) < GIT_SUCCESS) - return error; - - GIT_OBJECT_DECREF(commit->object.repo, commit->tree); - if ((error = git_object_lookup((git_object **)&commit->tree, commit->object.repo, &oid, GIT_OBJ_TREE)) < GIT_SUCCESS) + if ((error = git__parse_oid(&commit->tree_oid, &buffer, buffer_end, "tree ")) < GIT_SUCCESS) return error; /* * TODO: commit grafts! */ - while (git__parse_oid(&oid, &buffer, buffer_end, "parent ") == GIT_SUCCESS) { - git_commit *parent; + while (git__parse_oid(&parent_oid, &buffer, buffer_end, "parent ") == GIT_SUCCESS) { + git_oid *new_oid; - if ((error = git_object_lookup((git_object **)&parent, commit->object.repo, &oid, GIT_OBJ_COMMIT)) < GIT_SUCCESS) - return error; + new_oid = git__malloc(sizeof(git_oid)); + git_oid_cpy(new_oid, &parent_oid); - if (git_vector_insert(&commit->parents, parent) < GIT_SUCCESS) + if (git_vector_insert(&commit->parent_oids, new_oid) < GIT_SUCCESS) return GIT_ENOMEM; } + if (commit->author) + git_signature_free(commit->author); - if (parse_flags & COMMIT_FULL_PARSE) { - if (commit->author) - git_signature_free(commit->author); - - commit->author = git__malloc(sizeof(git_signature)); - if ((error = git_signature__parse(commit->author, &buffer, buffer_end, "author ")) < GIT_SUCCESS) - return error; - - } else { - if ((buffer = memchr(buffer, '\n', buffer_end - buffer)) == NULL) - return GIT_EOBJCORRUPTED; - - buffer++; - } + commit->author = git__malloc(sizeof(git_signature)); + if ((error = git_signature__parse(commit->author, &buffer, buffer_end, "author ")) < GIT_SUCCESS) + return error; /* Always parse the committer; we need the commit time */ if (commit->committer) @@ -176,7 +154,7 @@ int commit_parse_buffer(git_commit *commit, void *data, size_t len, unsigned int while (buffer <= buffer_end && *buffer == '\n') buffer++; - if (parse_flags & COMMIT_FULL_PARSE && buffer < buffer_end) { + if (buffer < buffer_end) { const char *line_end; size_t message_len = buffer_end - buffer; @@ -203,106 +181,81 @@ int git_commit__parse(git_commit *commit) { assert(commit && commit->object.source.open); return commit_parse_buffer(commit, - commit->object.source.raw.data, commit->object.source.raw.len, COMMIT_BASIC_PARSE); + commit->object.source.raw.data, commit->object.source.raw.len); } -int git_commit__parse_full(git_commit *commit) -{ - int error; - - if (commit->full_parse) - return GIT_SUCCESS; - - if ((error = git_object__source_open((git_object *)commit)) < GIT_SUCCESS) - return error; - - error = commit_parse_buffer(commit, - commit->object.source.raw.data, commit->object.source.raw.len, COMMIT_FULL_PARSE); - - git_object__source_close((git_object *)commit); - - commit->full_parse = 1; - return error; -} - - - -#define GIT_COMMIT_GETTER(_rvalue, _name) \ - const _rvalue git_commit_##_name(git_commit *commit) \ +#define GIT_COMMIT_GETTER(_rvalue, _name, _return) \ + _rvalue git_commit_##_name(git_commit *commit) \ {\ assert(commit); \ - if (commit->_name) \ - return commit->_name; \ - if (!commit->object.in_memory) \ - git_commit__parse_full(commit); \ - return commit->_name; \ + return _return; \ } -#define CHECK_FULL_PARSE() \ - if (!commit->object.in_memory && !commit->full_parse)\ - git_commit__parse_full(commit); +GIT_COMMIT_GETTER(const git_signature *, author, commit->author) +GIT_COMMIT_GETTER(const git_signature *, committer, commit->committer) +GIT_COMMIT_GETTER(const char *, message, commit->message) +GIT_COMMIT_GETTER(const char *, message_short, commit->message_short) +GIT_COMMIT_GETTER(time_t, time, commit->committer->when.time) +GIT_COMMIT_GETTER(int, time_offset, commit->committer->when.offset) +GIT_COMMIT_GETTER(unsigned int, parentcount, commit->parent_oids.length) + + +int git_commit_tree(git_tree **tree_out, git_commit *commit) +{ + assert(commit); + return git_tree_lookup(tree_out, commit->object.repo, &commit->tree_oid); +} -const git_tree *git_commit_tree(git_commit *commit) +int git_commit_parent(git_commit **parent, git_commit *commit, unsigned int n) { + git_oid *parent_oid; assert(commit); - if (!commit->object.in_memory && commit->tree == NULL) - git_commit__parse_full(commit); + parent_oid = git_vector_get(&commit->parent_oids, n); + if (parent_oid == NULL) + return GIT_ENOTFOUND; - GIT_OBJECT_INCREF(commit->object.repo, commit->tree); - return commit->tree; + return git_commit_lookup(parent, commit->object.repo, parent_oid); } -GIT_COMMIT_GETTER(git_signature *, author) -GIT_COMMIT_GETTER(git_signature *, committer) -GIT_COMMIT_GETTER(char *, message) -GIT_COMMIT_GETTER(char *, message_short) -time_t git_commit_time(git_commit *commit) -{ - assert(commit && commit->committer); - return commit->committer->when.time; -} -int git_commit_time_offset(git_commit *commit) +int git_commit_set_tree(git_commit *commit, git_tree *tree) { - assert(commit && commit->committer); - return commit->committer->when.offset; -} + const git_oid *oid; -unsigned int git_commit_parentcount(git_commit *commit) -{ - assert(commit); - return commit->parents.length; + assert(commit && tree); + + if ((oid = git_object_id((git_object *)tree)) == NULL) + return GIT_EMISSINGOBJDATA; + + commit->object.modified = 1; + git_oid_cpy(&commit->tree_oid, oid); + return GIT_SUCCESS; } -git_commit *git_commit_parent(git_commit *commit, unsigned int n) +int git_commit_add_parent(git_commit *commit, git_commit *new_parent) { - git_commit *parent; + const git_oid *parent_oid; + git_oid *new_oid; + assert(commit && new_parent); - assert(commit); + if ((parent_oid = git_object_id((git_object *)new_parent)) == NULL) + return GIT_EMISSINGOBJDATA; - parent = git_vector_get(&commit->parents, n); - GIT_OBJECT_INCREF(commit->object.repo, parent); - return parent; -} + new_oid = git__malloc(sizeof(git_oid)); + if (new_oid == NULL) + return GIT_ENOMEM; -void git_commit_set_tree(git_commit *commit, git_tree *tree) -{ - assert(commit && tree); commit->object.modified = 1; - CHECK_FULL_PARSE(); - - GIT_OBJECT_DECREF(commit->object.repo, commit->tree); - GIT_OBJECT_INCREF(commit->object.repo, tree); - commit->tree = tree; + git_oid_cpy(new_oid, parent_oid); + return git_vector_insert(&commit->parent_oids, new_oid); } void git_commit_set_author(git_commit *commit, const git_signature *author_sig) { assert(commit && author_sig); commit->object.modified = 1; - CHECK_FULL_PARSE(); git_signature_free(commit->author); commit->author = git_signature_dup(author_sig); @@ -312,7 +265,6 @@ void git_commit_set_committer(git_commit *commit, const git_signature *committer { assert(commit && committer_sig); commit->object.modified = 1; - CHECK_FULL_PARSE(); git_signature_free(commit->committer); commit->committer = git_signature_dup(committer_sig); @@ -324,7 +276,6 @@ void git_commit_set_message(git_commit *commit, const char *message) size_t message_len; commit->object.modified = 1; - CHECK_FULL_PARSE(); if (commit->message) free(commit->message); @@ -347,12 +298,3 @@ void git_commit_set_message(git_commit *commit, const char *message) commit->message_short[message_len] = 0; } -int git_commit_add_parent(git_commit *commit, git_commit *new_parent) -{ - assert(commit && new_parent); - - CHECK_FULL_PARSE(); - commit->object.modified = 1; - GIT_OBJECT_INCREF(commit->object.repo, new_parent); - return git_vector_insert(&commit->parents, new_parent); -} diff --git a/src/commit.h b/src/commit.h index b53ee9b23..aaf349ca6 100644 --- a/src/commit.h +++ b/src/commit.h @@ -11,21 +11,18 @@ struct git_commit { git_object object; - git_vector parents; + git_vector parent_oids; + git_oid tree_oid; - git_tree *tree; git_signature *author; git_signature *committer; char *message; char *message_short; - - unsigned full_parse:1; }; void git_commit__free(git_commit *c); int git_commit__parse(git_commit *commit); -int git_commit__parse_full(git_commit *commit); int git_commit__writeback(git_commit *commit, git_odb_source *src); diff --git a/src/object.c b/src/object.c index 7891893a0..fce99153b 100644 --- a/src/object.c +++ b/src/object.c @@ -261,7 +261,7 @@ int git_object_new(git_object **object_out, git_repository *repo, git_otype type object->source.raw.type = type; - object->refcount++; + object->lru = ++repo->lru_counter; *object_out = object; return GIT_SUCCESS; } @@ -277,7 +277,8 @@ int git_object_lookup(git_object **object_out, git_repository *repo, const git_o object = git_hashtable_lookup(repo->objects, id); if (object != NULL) { *object_out = object; - GIT_OBJECT_INCREF(repo, object); + object->lru = ++repo->lru_counter; + object->can_free = 0; return GIT_SUCCESS; } @@ -330,7 +331,7 @@ int git_object_lookup(git_object **object_out, git_repository *repo, const git_o git_object__source_close(object); git_hashtable_insert(repo->objects, &object->id, object); - GIT_OBJECT_INCREF(repo, object); + object->lru = ++repo->lru_counter; *object_out = object; return GIT_SUCCESS; } @@ -412,17 +413,12 @@ void git_object_close(git_object *object) if (object == NULL) return; - if (--object->refcount <= 0) { - if (object->repo != NULL) { - if (object->in_memory) { - int idx = git_vector_search(&object->repo->memory_objects, object); - git_vector_remove(&object->repo->memory_objects, idx); - } else { - git_hashtable_remove(object->repo->objects, &object->id); - } - } - + if (object->in_memory) { + int idx = git_vector_search(&object->repo->memory_objects, object); + git_vector_remove(&object->repo->memory_objects, idx); git_object__free(object); + } else { + object->can_free = 1; } } diff --git a/src/refs.c b/src/refs.c index 93897a7f9..1ed6b567c 100644 --- a/src/refs.c +++ b/src/refs.c @@ -681,7 +681,6 @@ static int packed_write_ref(reference_oid *ref, git_filebuf *file) static int packed_find_peel(reference_oid *ref) { git_tag *tag; - const git_object *peeled_target; int error; if (ref->ref.type & GIT_REF_HAS_PEEL) @@ -706,11 +705,7 @@ static int packed_find_peel(reference_oid *ref) /* * Find the object pointed at by this tag */ - peeled_target = git_tag_target(tag); - if (peeled_target == NULL) - return GIT_EOBJCORRUPTED; - - git_oid_cpy(&ref->peel_target, git_object_id(peeled_target)); + git_oid_cpy(&ref->peel_target, git_tag_target_oid(tag)); ref->ref.type |= GIT_REF_HAS_PEEL; /* diff --git a/src/repository.c b/src/repository.c index c24560665..37aa44781 100644 --- a/src/repository.c +++ b/src/repository.c @@ -209,7 +209,6 @@ static git_repository *repository_alloc() return NULL; } - repo->gc_enabled = 1; return repo; } @@ -331,50 +330,20 @@ cleanup: return error; } -static void repository_free(git_repository *repo) -{ - assert(repo); - - free(repo->path_workdir); - free(repo->path_index); - free(repo->path_repository); - free(repo->path_odb); - - git_hashtable_free(repo->objects); - git_vector_free(&repo->memory_objects); - - git_repository__refcache_free(&repo->references); - - if (repo->db != NULL) - git_odb_close(repo->db); - - if (repo->index != NULL) - git_index_free(repo->index); - - free(repo); -} - -void git_repository_free__no_gc(git_repository *repo) +int git_repository_gc(git_repository *repo) { + int collected = 0; git_object *object; const void *_unused; - unsigned int i; - - if (repo == NULL) - return; GIT_HASHTABLE_FOREACH(repo->objects, _unused, object, - object->repo = NULL; - object->refcount = 0; + if (object->can_free) { + git_object__free(object); + collected++; + } ); - for (i = 0; i < repo->memory_objects.length; ++i) { - object = git_vector_get(&repo->memory_objects, i); - object->repo = NULL; - object->refcount = 0; - } - - repository_free(repo); + return collected; } void git_repository_free(git_repository *repo) @@ -386,8 +355,6 @@ void git_repository_free(git_repository *repo) if (repo == NULL) return; - repo->gc_enabled = 0; - /* force free all the objects */ GIT_HASHTABLE_FOREACH(repo->objects, _unused, object, git_object__free(object); @@ -398,7 +365,23 @@ void git_repository_free(git_repository *repo) git_object__free(object); } - repository_free(repo); + free(repo->path_workdir); + free(repo->path_index); + free(repo->path_repository); + free(repo->path_odb); + + git_hashtable_free(repo->objects); + git_vector_free(&repo->memory_objects); + + git_repository__refcache_free(&repo->references); + + if (repo->db != NULL) + git_odb_close(repo->db); + + if (repo->index != NULL) + git_index_free(repo->index); + + free(repo); } int git_repository_index(git_index **index_out, git_repository *repo) diff --git a/src/repository.h b/src/repository.h index 5318ed45c..48b8dae6b 100644 --- a/src/repository.h +++ b/src/repository.h @@ -27,8 +27,8 @@ struct git_object { git_oid id; git_repository *repo; git_odb_source source; - unsigned short refcount; - unsigned char in_memory, modified; + unsigned int lru; + unsigned char in_memory, modified, can_free, _pad; }; struct git_repository { @@ -45,7 +45,8 @@ struct git_repository { char *path_odb; char *path_workdir; - unsigned is_bare:1, gc_enabled:1; + unsigned is_bare:1; + unsigned int lru_counter; }; int git_object__source_open(git_object *object); @@ -61,19 +62,4 @@ int git__source_write(git_odb_source *source, const void *bytes, size_t len); int git__parse_oid(git_oid *oid, char **buffer_out, const char *buffer_end, const char *header); int git__write_oid(git_odb_source *src, const char *header, const git_oid *oid); -#define GIT_OBJECT_INCREF(repo, ob) git_object__incref((repo), (git_object *)(ob)) -#define GIT_OBJECT_DECREF(repo, ob) git_object__decref((repo), (git_object *)(ob)) - -GIT_INLINE(void) git_object__incref(git_repository *repo, struct git_object *object) -{ - if (repo && repo->gc_enabled && object) - object->refcount++; -} - -GIT_INLINE(void) git_object__decref(git_repository *repo, struct git_object *object) -{ - if (repo && repo->gc_enabled && object) - git_object_close(object); -} - #endif diff --git a/src/tag.c b/src/tag.c index 1ac2067c2..1379425a1 100644 --- a/src/tag.c +++ b/src/tag.c @@ -35,7 +35,6 @@ void git_tag__free(git_tag *tag) { git_signature_free(tag->tagger); - GIT_OBJECT_DECREF(tag->object.repo, tag->target); free(tag->message); free(tag->tag_name); free(tag); @@ -46,23 +45,31 @@ const git_oid *git_tag_id(git_tag *c) return git_object_id((git_object *)c); } -const git_object *git_tag_target(git_tag *t) +int git_tag_target(git_object **target, git_tag *t) { assert(t); - GIT_OBJECT_INCREF(t->object.repo, t->target); - return t->target; + return git_object_lookup(target, t->object.repo, &t->target, t->type); } -void git_tag_set_target(git_tag *tag, git_object *target) +const git_oid *git_tag_target_oid(git_tag *t) { + assert(t); + return &t->target; +} + +int git_tag_set_target(git_tag *tag, git_object *target) +{ + const git_oid *oid; + assert(tag && target); - GIT_OBJECT_DECREF(tag->object.repo, tag->target); - GIT_OBJECT_INCREF(tag->object.repo, target); + if ((oid = git_object_id(target)) == NULL) + return GIT_EMISSINGOBJDATA; tag->object.modified = 1; - tag->target = target; + git_oid_cpy(&tag->target, oid); tag->type = git_object_type(target); + return GIT_SUCCESS; } git_otype git_tag_type(git_tag *t) @@ -81,7 +88,6 @@ void git_tag_set_name(git_tag *tag, const char *name) { assert(tag && name); - /* TODO: sanity check? no newlines in message */ tag->object.modified = 1; if (tag->tag_name) @@ -128,12 +134,11 @@ static int parse_tag_buffer(git_tag *tag, char *buffer, const char *buffer_end) NULL, "commit\n", "tree\n", "blob\n", "tag\n" }; - git_oid target_oid; unsigned int i, text_len; char *search; int error; - if ((error = git__parse_oid(&target_oid, &buffer, buffer_end, "object ")) < 0) + if ((error = git__parse_oid(&tag->target, &buffer, buffer_end, "object ")) < 0) return error; if (buffer + 5 >= buffer_end) @@ -161,11 +166,6 @@ static int parse_tag_buffer(git_tag *tag, char *buffer, const char *buffer_end) if (tag->type == GIT_OBJ_BAD) return GIT_EOBJCORRUPTED; - git_object_close(tag->target); - error = git_object_lookup(&tag->target, tag->object.repo, &target_oid, tag->type); - if (error < 0) - return error; - if (buffer + 4 >= buffer_end) return GIT_EOBJCORRUPTED; @@ -210,10 +210,10 @@ static int parse_tag_buffer(git_tag *tag, char *buffer, const char *buffer_end) int git_tag__writeback(git_tag *tag, git_odb_source *src) { - if (tag->target == NULL || tag->tag_name == NULL || tag->tagger == NULL) + if (tag->tag_name == NULL || tag->tagger == NULL) return GIT_EMISSINGOBJDATA; - git__write_oid(src, "object", git_object_id(tag->target)); + git__write_oid(src, "object", &tag->target); git__source_printf(src, "type %s\n", git_object_type2string(tag->type)); git__source_printf(src, "tag %s\n", tag->tag_name); git_signature__write(src, "tagger", tag->tagger); diff --git a/src/tag.h b/src/tag.h index 624fcc654..a1782d064 100644 --- a/src/tag.h +++ b/src/tag.h @@ -7,8 +7,9 @@ struct git_tag { git_object object; - git_object *target; + git_oid target; git_otype type; + char *tag_name; git_signature *tagger; char *message; -- cgit v1.2.1 From 7c8a7b91431fc72a17ec579bcbca3a596071eeb1 Mon Sep 17 00:00:00 2001 From: Vicent Marti Date: Tue, 15 Mar 2011 02:45:49 +0200 Subject: Skip the optional header in packed-refs files --- src/refs.c | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) (limited to 'src') diff --git a/src/refs.c b/src/refs.c index 1ed6b567c..b9d25a769 100644 --- a/src/refs.c +++ b/src/refs.c @@ -507,22 +507,14 @@ static int packed_load(git_repository *repo) buffer_start = (const char *)packfile.data; buffer_end = (const char *)(buffer_start) + packfile.len; - /* Does the header look like valid? */ - if (git__prefixcmp((const char *)(buffer_start), GIT_PACKEDREFS_HEADER)) { - error = GIT_EPACKEDREFSCORRUPTED; - goto cleanup; - } - - /* Let's skip the header */ - buffer_start += strlen(GIT_PACKEDREFS_HEADER); - - if (*buffer_start == '\r') + while (buffer_start < buffer_end && buffer_start[0] == '#') { + buffer_start = strchr(buffer_start, '\n'); + if (buffer_start == NULL) { + error = GIT_EPACKEDREFSCORRUPTED; + goto cleanup; + } buffer_start++; - - if (*buffer_start != '\n') - return GIT_EPACKEDREFSCORRUPTED; - - buffer_start++; + } while (buffer_start < buffer_end) { reference_oid *ref = NULL; @@ -810,7 +802,9 @@ static int packed_write(git_repository *repo) if ((error = git_filebuf_open(&pack_file, pack_file_path, 0)) < GIT_SUCCESS) return error; - /* Packfiles have a header! */ + /* Packfiles have a header... apparently + * This is in fact not required, but we might as well print it + * just for kicks */ if ((error = git_filebuf_printf(&pack_file, "%s\n", GIT_PACKEDREFS_HEADER)) < GIT_SUCCESS) return error; -- cgit v1.2.1 From bd1aa741a4e7b2b83991e7bccbfe55960929af9e Mon Sep 17 00:00:00 2001 From: Ben Noordhuis Date: Mon, 14 Mar 2011 00:07:37 +0100 Subject: git_object_lookup() should also check the type if the object comes from the cache --- src/object.c | 3 +++ 1 file changed, 3 insertions(+) (limited to 'src') diff --git a/src/object.c b/src/object.c index fce99153b..c432c6da0 100644 --- a/src/object.c +++ b/src/object.c @@ -276,6 +276,9 @@ int git_object_lookup(git_object **object_out, git_repository *repo, const git_o object = git_hashtable_lookup(repo->objects, id); if (object != NULL) { + if (type != GIT_OBJ_ANY && type != object->source.raw.type) + return GIT_EINVALIDTYPE; + *object_out = object; object->lru = ++repo->lru_counter; object->can_free = 0; -- cgit v1.2.1 From 7ad96e51ca81974c417914edbc81a63e390c4301 Mon Sep 17 00:00:00 2001 From: Vicent Marti Date: Tue, 15 Mar 2011 05:38:50 +0200 Subject: Remove duplicate refs in `git_reference_listall` --- src/refs.c | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) (limited to 'src') diff --git a/src/refs.c b/src/refs.c index b9d25a769..cb11159fb 100644 --- a/src/refs.c +++ b/src/refs.c @@ -546,6 +546,7 @@ cleanup: struct dirent_list_data { git_vector ref_list; + git_repository *repo; size_t repo_path_len; unsigned int list_flags; }; @@ -553,16 +554,18 @@ struct dirent_list_data { static int _dirent_loose_listall(void *_data, char *full_path) { struct dirent_list_data *data = (struct dirent_list_data *)_data; - char *file_path; + char *file_path = full_path + data->repo_path_len; if (gitfo_isdir(full_path) == GIT_SUCCESS) return gitfo_dirent(full_path, GIT_PATH_MAX, _dirent_loose_listall, _data); + /* do not add twice a reference that exists already in the packfile */ + if (git_hashtable_lookup(data->repo->references.packfile, file_path) != NULL) + return GIT_SUCCESS; + if ((data->list_flags & loose_guess_rtype(full_path)) == 0) return GIT_SUCCESS; /* we are filtering out this reference */ - file_path = full_path + data->repo_path_len; - return git_vector_insert(&data->ref_list, git__strdup(file_path)); } @@ -1336,15 +1339,9 @@ int git_reference_listall(git_strarray *array, git_repository *repo, unsigned in git_vector_init(&data.ref_list, 8, NULL); data.repo_path_len = strlen(repo->path_repository); data.list_flags = list_flags; + data.repo = repo; - git__joinpath(refs_path, repo->path_repository, GIT_REFS_DIR); - error = gitfo_dirent(refs_path, GIT_PATH_MAX, _dirent_loose_listall, &data); - - if (error < GIT_SUCCESS) { - git_vector_free(&data.ref_list); - return error; - } - + /* list all the packed references first */ if (list_flags & GIT_REF_PACKED) { const char *ref_name; void *_unused; @@ -1359,6 +1356,16 @@ int git_reference_listall(git_strarray *array, git_repository *repo, unsigned in ); } + /* now list the loose references, trying not to + * duplicate the ref names already in the packed-refs file */ + git__joinpath(refs_path, repo->path_repository, GIT_REFS_DIR); + error = gitfo_dirent(refs_path, GIT_PATH_MAX, _dirent_loose_listall, &data); + + if (error < GIT_SUCCESS) { + git_vector_free(&data.ref_list); + return error; + } + array->strings = (char **)data.ref_list.contents; array->count = data.ref_list.length; return GIT_SUCCESS; -- cgit v1.2.1 From d40d30cb76ddcc9935471af0fb6fa6b2a66efc4c Mon Sep 17 00:00:00 2001 From: Vicent Marti Date: Tue, 15 Mar 2011 19:54:45 +0200 Subject: Debug assertion when using an initialized hashtable --- src/hashtable.c | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'src') diff --git a/src/hashtable.c b/src/hashtable.c index c36d8a8e6..ee6d3a461 100644 --- a/src/hashtable.c +++ b/src/hashtable.c @@ -184,6 +184,8 @@ int git_hashtable_insert2(git_hashtable *self, const void *key, void *value, voi int hash_id; git_hashtable_node *node; + assert(self && self->nodes); + *old_value = NULL; for (hash_id = 0; hash_id < GIT_HASHTABLE_HASHES; ++hash_id) { @@ -218,6 +220,8 @@ void *git_hashtable_lookup(git_hashtable *self, const void *key) int hash_id; git_hashtable_node *node; + assert(self && self->nodes); + for (hash_id = 0; hash_id < GIT_HASHTABLE_HASHES; ++hash_id) { node = node_with_hash(self, key, hash_id); if (node->key && self->key_equal(key, node->key) == 0) @@ -232,6 +236,8 @@ int git_hashtable_remove(git_hashtable *self, const void *key) int hash_id; git_hashtable_node *node; + assert(self && self->nodes); + for (hash_id = 0; hash_id < GIT_HASHTABLE_HASHES; ++hash_id) { node = node_with_hash(self, key, hash_id); if (node->key && self->key_equal(key, node->key) == 0) { -- cgit v1.2.1 From b5abb881a623b8b492e0375b8e9c8936079c39bb Mon Sep 17 00:00:00 2001 From: Vicent Marti Date: Tue, 15 Mar 2011 19:55:01 +0200 Subject: Do not segfault when listing unpacked references --- src/refs.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/refs.c b/src/refs.c index cb11159fb..ea39608df 100644 --- a/src/refs.c +++ b/src/refs.c @@ -560,7 +560,8 @@ static int _dirent_loose_listall(void *_data, char *full_path) return gitfo_dirent(full_path, GIT_PATH_MAX, _dirent_loose_listall, _data); /* do not add twice a reference that exists already in the packfile */ - if (git_hashtable_lookup(data->repo->references.packfile, file_path) != NULL) + if ((data->list_flags & GIT_REF_PACKED) != 0 && + git_hashtable_lookup(data->repo->references.packfile, file_path) != NULL) return GIT_SUCCESS; if ((data->list_flags & loose_guess_rtype(full_path)) == 0) -- cgit v1.2.1 From bbcc7ffc690c4065954102530447aef4e8ccf895 Mon Sep 17 00:00:00 2001 From: Vicent Marti Date: Tue, 15 Mar 2011 21:04:41 +0200 Subject: Add proper threading support to libgit2 We now depend on libpthread on all Unix platforms (should be installed by default) and use a simple wrapper for Windows threads under Win32. Signed-off-by: Vicent Marti --- src/common.h | 5 +-- src/thread-utils.h | 121 +++++++++------------------------------------------- src/win32/pthread.c | 120 +++++++++++++++++++++++++++++++++++++++++++++++++++ src/win32/pthread.h | 67 +++++++++++++++++++++++++++++ 4 files changed, 209 insertions(+), 104 deletions(-) create mode 100644 src/win32/pthread.c create mode 100644 src/win32/pthread.h (limited to 'src') diff --git a/src/common.h b/src/common.h index 1ca00471b..723085a2b 100644 --- a/src/common.h +++ b/src/common.h @@ -11,9 +11,6 @@ #include "git2/thread-utils.h" #include "cc-compat.h" -#ifdef GIT_HAS_PTHREAD -# include -#endif #ifdef GIT_HAVE_INTTYPES_H # include #endif @@ -34,6 +31,7 @@ # include # include "msvc-compat.h" # include "mingw-compat.h" +# include "win32/pthread.h" # define snprintf _snprintf @@ -43,6 +41,7 @@ typedef SSIZE_T ssize_t; # include # include +# include #endif diff --git a/src/thread-utils.h b/src/thread-utils.h index 0029e4bc1..e8372e731 100644 --- a/src/thread-utils.h +++ b/src/thread-utils.h @@ -1,108 +1,27 @@ #ifndef INCLUDE_thread_utils_h__ #define INCLUDE_thread_utils_h__ -#if defined(GIT_HAS_PTHREAD) - typedef pthread_t git_thread; -# define git_thread_create(thread, attr, start_routine, arg) pthread_create(thread, attr, start_routine, arg) -# define git_thread_kill(thread) pthread_cancel(thread) -# define git_thread_exit(status) pthread_exit(status) -# define git_thread_join(id, status) pthread_join(id, status) - /* Pthreads Mutex */ - typedef pthread_mutex_t git_lck; -# define GITLCK_INIT PTHREAD_MUTEX_INITIALIZER -# define gitlck_init(a) pthread_mutex_init(a, NULL) -# define gitlck_lock(a) pthread_mutex_lock(a) -# define gitlck_unlock(a) pthread_mutex_unlock(a) -# define gitlck_free(a) pthread_mutex_destroy(a) - - /* Pthreads condition vars */ - typedef pthread_cond_t git_cnd; -# define GITCND_INIT PTHREAD_COND_INITIALIZER -# define gitcnd_init(c, a) pthread_cond_init(c, a) -# define gitcnd_free(c) pthread_cond_destroy(c) -# define gitcnd_wait(c, l) pthread_cond_wait(c, l) -# define gitcnd_signal(c) pthread_cond_signal(c) -# define gitcnd_broadcast(c) pthread_cond_broadcast(c) - -# if defined(GIT_HAS_ASM_ATOMIC) -# include - typedef atomic_t git_refcnt; -# define gitrc_init(a, v) atomic_set(a, v) -# define gitrc_inc(a) atomic_inc_return(a) -# define gitrc_dec(a) atomic_dec_and_test(a) -# define gitrc_free(a) (void)0 -# elif defined(GIT_WIN32) - typedef long git_refcnt; -# define gitrc_init(a, v) (*a = v) -# define gitrc_inc(a) (InterlockedIncrement(a)) -# define gitrc_dec(a) (!InterlockedDecrement(a)) -# define gitrc_free(a) (void)0 -# else - typedef struct { git_lck lock; int counter; } git_refcnt; - - /** Initialize to 0. No memory barrier is issued. */ - GIT_INLINE(void) gitrc_init(git_refcnt *p, int value) - { - gitlck_init(&p->lock); - p->counter = value; - } - - /** - * Increment. - * - * Atomically increments @p by 1. A memory barrier is also - * issued before and after the operation. - * - * @param p pointer of type git_refcnt - */ - GIT_INLINE(void) gitrc_inc(git_refcnt *p) - { - gitlck_lock(&p->lock); - p->counter++; - gitlck_unlock(&p->lock); - } - - /** - * Decrement and test. - * - * Atomically decrements @p by 1 and returns true if the - * result is 0, or false for all other cases. A memory - * barrier is also issued before and after the operation. - * - * @param p pointer of type git_refcnt - */ - GIT_INLINE(int) gitrc_dec(git_refcnt *p) - { - int c; - gitlck_lock(&p->lock); - c = --p->counter; - gitlck_unlock(&p->lock); - return !c; - } - - /** Free any resources associated with the counter. */ -# define gitrc_free(p) gitlck_free(&(p)->lock) -# endif - -#elif defined(GIT_THREADS) -# error GIT_THREADS but no git_lck implementation - -#else - /* no threads support */ - typedef struct { int dummy; } git_lck; -# define GIT_MUTEX_INIT {} -# define gitlck_init(a) (void)0 -# define gitlck_lock(a) (void)0 -# define gitlck_unlock(a) (void)0 -# define gitlck_free(a) (void)0 - - typedef struct { int counter; } git_refcnt; -# define gitrc_init(a,v) ((a)->counter = v) -# define gitrc_inc(a) ((a)->counter++) -# define gitrc_dec(a) (--(a)->counter == 0) -# define gitrc_free(a) (void)0 -#endif +#define git_thread pthread_t +#define git_thread_create(thread, attr, start_routine, arg) pthread_create(thread, attr, start_routine, arg) +#define git_thread_kill(thread) pthread_cancel(thread) +#define git_thread_exit(status) pthread_exit(status) +#define git_thread_join(id, status) pthread_join(id, status) + +/* Pthreads Mutex */ +#define git_mutex pthread_mutex_t +#define git_mutex_init(a) pthread_mutex_init(a, NULL) +#define git_mutex_lock(a) pthread_mutex_lock(a) +#define git_mutex_unlock(a) pthread_mutex_unlock(a) +#define git_mutex_free(a) pthread_mutex_destroy(a) + +/* Pthreads condition vars */ +#define git_cond pthread_cond_t +#define git_cond_init(c, a) pthread_cond_init(c, a) +#define git_cond_free(c) pthread_cond_destroy(c) +#define git_cond_wait(c, l) pthread_cond_wait(c, l) +#define git_cond_signal(c) pthread_cond_signal(c) +#define git_cond_broadcast(c) pthread_cond_broadcast(c) extern int git_online_cpus(void); diff --git a/src/win32/pthread.c b/src/win32/pthread.c new file mode 100644 index 000000000..fffff81df --- /dev/null +++ b/src/win32/pthread.c @@ -0,0 +1,120 @@ +/* + * This file is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License, version 2, + * as published by the Free Software Foundation. + * + * In addition to the permissions in the GNU General Public License, + * the authors give you unlimited permission to link the compiled + * version of this file into combinations with other programs, + * and to distribute those combinations without any restriction + * coming from the use of this file. (The General Public License + * restrictions do apply in other respects; for example, they cover + * modification of the file, and distribution when not linked into + * a combined executable.) + * + * This file is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; see the file COPYING. If not, write to + * the Free Software Foundation, 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + * Original code by Ramiro Polla (Public Domain) + */ + +#include "pthread.h" + +int pthread_create(pthread_t *GIT_RESTRICT thread, + const pthread_attr_t *GIT_RESTRICT GIT_UNUSED(attr), + void *(*start_routine)(void*), void *GIT_RESTRICT arg) +{ + GIT_UNUSED_ARG(attr); + *thread = (pthread_t) CreateThread(NULL, 0, (LPTHREAD_START_ROUTINE)start_routine, arg, 0, NULL); + return *thread ? GIT_SUCCESS : GIT_EOSERR; +} + +int pthread_cond_signal(pthread_cond_t *cond) +{ + WakeConditionVariable(cond); + return 0; +} + +int pthread_cond_wait(pthread_cond_t *GIT_RESTRICT cond, + pthread_mutex_t *GIT_RESTRICT mutex) +{ + int ret; + ret = SleepConditionVariableCS(cond, mutex, INFINITE); + return -(!ret); +} + +int pthread_mutex_lock(pthread_mutex_t *mutex) +{ + EnterCriticalSection(mutex); + return 0; +} + +int pthread_mutex_unlock(pthread_mutex_t *mutex) +{ + LeaveCriticalSection(mutex); + return 0; +} + +int pthread_join(pthread_t thread, void **value_ptr) +{ + int ret; + ret = WaitForSingleObject(thread, INFINITE); + if (ret && value_ptr) + GetExitCodeThread(thread, (void*) value_ptr); + return -(!!ret); +} + +int pthread_cond_broadcast(pthread_cond_t *cond) +{ + WakeAllConditionVariable(cond); + return 0; +} + +int pthread_mutex_destroy(pthread_mutex_t *mutex) +{ + int ret; + ret = CloseHandle(mutex); + return -(!ret); +} + +int pthread_cond_destroy(pthread_cond_t *GIT_UNUSED(cond)) +{ + GIT_UNUSED_ARG(cond); + return 0; +} + +int pthread_cond_init(pthread_cond_t *GIT_RESTRICT cond, + const pthread_condattr_t *GIT_RESTRICT GIT_UNUSED(condattr)) +{ + GIT_UNUSED_ARG(condattr); + InitializeConditionVariable(cond); + return 0; +} + +int pthread_mutex_init(pthread_mutex_t *GIT_RESTRICT mutex, + const pthread_mutexattr_t *GIT_RESTRICT GIT_UNUSED(mutexattr)) +{ + GIT_UNUSED_ARG(mutexattr); + InitializeCriticalSection(mutex); + return 0; +} + +int pthread_num_processors_np(void) +{ + DWORD_PTR p, s; + int n = 0; + + if (GetProcessAffinityMask(GetCurrentProcess(), &p, &s)) + for (; p; p >>= 1) + n += p&1; + + return n ? n : 1; +} + diff --git a/src/win32/pthread.h b/src/win32/pthread.h new file mode 100644 index 000000000..ff694e303 --- /dev/null +++ b/src/win32/pthread.h @@ -0,0 +1,67 @@ +/* + * This file is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License, version 2, + * as published by the Free Software Foundation. + * + * In addition to the permissions in the GNU General Public License, + * the authors give you unlimited permission to link the compiled + * version of this file into combinations with other programs, + * and to distribute those combinations without any restriction + * coming from the use of this file. (The General Public License + * restrictions do apply in other respects; for example, they cover + * modification of the file, and distribution when not linked into + * a combined executable.) + * + * This file is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; see the file COPYING. If not, write to + * the Free Software Foundation, 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + * Original code by Ramiro Polla (Public Domain) + */ + +#ifndef GIT_PTHREAD_H +#define GIT_PTHREAD_H + +#include "../common.h" + +#if defined (_MSC_VER) +# define GIT_RESTRICT __restrict +#else +# define GIT_RESTRICT __restrict__ +#endif + +typedef int pthread_mutexattr_t; +typedef int pthread_condattr_t; +typedef int pthread_attr_t; +typedef CRITICAL_SECTION pthread_mutex_t; +typedef CONDITION_VARIABLE pthread_cond_t; +typedef HANDLE pthread_t; + +#define PTHREAD_MUTEX_INITIALIZER {(void*)-1}; + +int pthread_create(pthread_t *GIT_RESTRICT, + const pthread_attr_t *GIT_RESTRICT, + void *(*start_routine)(void*), void *__restrict); + +int pthread_join(pthread_t, void **); + +int pthread_mutex_init(pthread_mutex_t *GIT_RESTRICT, const pthread_mutexattr_t *GIT_RESTRICT); +int pthread_mutex_destroy(pthread_mutex_t *); +int pthread_mutex_lock(pthread_mutex_t *); +int pthread_mutex_unlock(pthread_mutex_t *); + +int pthread_cond_init(pthread_cond_t *GIT_RESTRICT, const pthread_condattr_t *GIT_RESTRICT); +int pthread_cond_destroy(pthread_cond_t *); +int pthread_cond_broadcast(pthread_cond_t *); +int pthread_cond_signal(pthread_cond_t *); +int pthread_cond_wait(pthread_cond_t *GIT_RESTRICT, pthread_mutex_t *GIT_RESTRICT); + +int pthread_num_processors_np(void); + +#endif \ No newline at end of file -- cgit v1.2.1 From 36b313296638f8ae349e400ba9a7f5bbfb92b96f Mon Sep 17 00:00:00 2001 From: Vicent Marti Date: Wed, 16 Mar 2011 01:04:17 +0200 Subject: Properly free commit a commit list in revwalk The commit list was not being properly free'd when a walk was stopped halfway through. --- src/revwalk.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) (limited to 'src') diff --git a/src/revwalk.c b/src/revwalk.c index edafbe73d..d17b44053 100644 --- a/src/revwalk.c +++ b/src/revwalk.c @@ -78,13 +78,17 @@ commit_list *commit_list_insert(commit_object *item, commit_list **list_p) return new_list; } -void commit_list_free(commit_list *list) +void commit_list_free(commit_list **list_p) { + commit_list *list = *list_p; + while (list) { commit_list *temp = list; list = temp->next; free(temp); } + + *list_p = NULL; } commit_object *commit_list_pop(commit_list **stack) @@ -561,9 +565,9 @@ void git_revwalk_reset(git_revwalk *walk) ); git_pqueue_free(&walk->iterator_time); - commit_list_free(walk->iterator_topo); - commit_list_free(walk->iterator_rand); - commit_list_free(walk->iterator_reverse); + commit_list_free(&walk->iterator_topo); + commit_list_free(&walk->iterator_rand); + commit_list_free(&walk->iterator_reverse); walk->walking = 0; } -- cgit v1.2.1 From 955f9ae9bd25ceb59775463e04c0276209387b67 Mon Sep 17 00:00:00 2001 From: Vicent Marti Date: Wed, 16 Mar 2011 01:06:15 +0200 Subject: Export `git_strarray_free` instead of inlining That way non-C bindings can use it. --- src/util.c | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'src') diff --git a/src/util.c b/src/util.c index c9a8e5fe9..bfc4f7b27 100644 --- a/src/util.c +++ b/src/util.c @@ -3,6 +3,15 @@ #include #include +void git_strarray_free(git_strarray *array) +{ + size_t i; + for (i = 0; i < array->count; ++i) + free(array->strings[i]); + + free(array->strings); +} + int git__fmt(char *buf, size_t buf_sz, const char *fmt, ...) { va_list va; -- cgit v1.2.1 From 36aaf1ff1a8540c364050f62f19f3021b6e121eb Mon Sep 17 00:00:00 2001 From: Vicent Marti Date: Wed, 16 Mar 2011 01:53:25 +0200 Subject: Change the Revwalk reset behavior to the old version The `reset` call now removes the pushed commits so we can reuse the revwalker. The API documentation has been updated with the details. --- src/pqueue.c | 4 ++ src/pqueue.h | 5 ++ src/revwalk.c | 179 ++++++++++++++++++++++++++++------------------------------ 3 files changed, 94 insertions(+), 94 deletions(-) (limited to 'src') diff --git a/src/pqueue.c b/src/pqueue.c index 98152cb85..6307175e3 100644 --- a/src/pqueue.c +++ b/src/pqueue.c @@ -50,6 +50,10 @@ void git_pqueue_free(git_pqueue *q) q->d = NULL; } +void git_pqueue_clear(git_pqueue *q) +{ + q->size = 1; +} size_t git_pqueue_size(git_pqueue *q) { diff --git a/src/pqueue.h b/src/pqueue.h index 6db74661d..7a1394803 100644 --- a/src/pqueue.h +++ b/src/pqueue.h @@ -53,6 +53,11 @@ int git_pqueue_init(git_pqueue *q, size_t n, git_pqueue_cmp cmppri); */ void git_pqueue_free(git_pqueue *q); +/** + * clear all the elements in the queue + * @param q the queue + */ +void git_pqueue_clear(git_pqueue *q); /** * return the size of the queue. diff --git a/src/revwalk.c b/src/revwalk.c index d17b44053..a2fc092d5 100644 --- a/src/revwalk.c +++ b/src/revwalk.c @@ -52,7 +52,6 @@ struct git_revwalk { git_repository *repo; git_hashtable *commits; - git_vector pending; commit_list *iterator_topo; commit_list *iterator_rand; @@ -159,71 +158,6 @@ static commit_object **alloc_parents(commit_object *commit, size_t n_parents) return git__malloc(n_parents * sizeof(commit_object *)); } -int git_revwalk_new(git_revwalk **revwalk_out, git_repository *repo) -{ - git_revwalk *walk; - - walk = git__malloc(sizeof(git_revwalk)); - if (walk == NULL) - return GIT_ENOMEM; - - memset(walk, 0x0, sizeof(git_revwalk)); - - walk->commits = git_hashtable_alloc(64, - object_table_hash, - (git_hash_keyeq_ptr)git_oid_cmp); - - if (walk->commits == NULL) { - free(walk); - return GIT_ENOMEM; - } - - git_vector_init(&walk->pending, 8, NULL); - git_vector_init(&walk->memory_alloc, 8, NULL); - alloc_chunk(walk); - - walk->repo = repo; - - *revwalk_out = walk; - return GIT_SUCCESS; -} - -void git_revwalk_free(git_revwalk *walk) -{ - unsigned int i; - - if (walk == NULL) - return; - - git_revwalk_reset(walk); - git_hashtable_free(walk->commits); - git_vector_free(&walk->pending); - - for (i = 0; i < walk->memory_alloc.length; ++i) { - free(git_vector_get(&walk->memory_alloc, i)); - } - - git_vector_free(&walk->memory_alloc); - free(walk); -} - -git_repository *git_revwalk_repository(git_revwalk *walk) -{ - assert(walk); - return walk->repo; -} - -int git_revwalk_sorting(git_revwalk *walk, unsigned int sort_mode) -{ - assert(walk); - - if (walk->walking) - return GIT_EBUSY; - - walk->sorting = sort_mode; - git_revwalk_reset(walk); - return GIT_SUCCESS; -} static commit_object *commit_lookup(git_revwalk *walk, const git_oid *oid) { @@ -370,10 +304,9 @@ static int push_commit(git_revwalk *walk, const git_oid *oid, int uninteresting) if (commit == NULL) return GIT_ENOTFOUND; - if (uninteresting) - mark_uninteresting(commit); + commit->uninteresting = uninteresting; - return git_vector_insert(&walk->pending, commit); + return process_commit(walk, commit); } int git_revwalk_push(git_revwalk *walk, const git_oid *oid) @@ -472,31 +405,11 @@ static int revwalk_next_reverse(commit_object **object_out, git_revwalk *walk) static int prepare_walk(git_revwalk *walk) { - unsigned int i; int error; - - if (walk->sorting & GIT_SORT_TIME) { - if ((error = git_pqueue_init(&walk->iterator_time, 32, commit_time_cmp)) < GIT_SUCCESS) - return error; - - walk->get_next = &revwalk_next_timesort; - walk->enqueue = &revwalk_enqueue_timesort; - } else { - walk->get_next = &revwalk_next_unsorted; - walk->enqueue = &revwalk_enqueue_unsorted; - } - - for (i = 0; i < walk->pending.length; ++i) { - commit_object *commit = walk->pending.contents[i]; - if ((error = process_commit(walk, commit)) < GIT_SUCCESS) { - return error; - } - } + commit_object *next; if (walk->sorting & GIT_SORT_TOPOLOGICAL) { - commit_object *next; unsigned short i; - int error; while ((error = walk->get_next(&next, walk)) == GIT_SUCCESS) { for (i = 0; i < next->out_degree; ++i) { @@ -514,8 +427,6 @@ static int prepare_walk(git_revwalk *walk) } if (walk->sorting & GIT_SORT_REVERSE) { - commit_object *next; - int error; while ((error = walk->get_next(&next, walk)) == GIT_SUCCESS) commit_list_insert(next, &walk->iterator_reverse); @@ -531,6 +442,83 @@ static int prepare_walk(git_revwalk *walk) } + + + +int git_revwalk_new(git_revwalk **revwalk_out, git_repository *repo) +{ + git_revwalk *walk; + + walk = git__malloc(sizeof(git_revwalk)); + if (walk == NULL) + return GIT_ENOMEM; + + memset(walk, 0x0, sizeof(git_revwalk)); + + walk->commits = git_hashtable_alloc(64, + object_table_hash, + (git_hash_keyeq_ptr)git_oid_cmp); + + if (walk->commits == NULL) { + free(walk); + return GIT_ENOMEM; + } + + git_pqueue_init(&walk->iterator_time, 8, commit_time_cmp); + git_vector_init(&walk->memory_alloc, 8, NULL); + alloc_chunk(walk); + + walk->get_next = &revwalk_next_unsorted; + walk->enqueue = &revwalk_enqueue_unsorted; + + walk->repo = repo; + + *revwalk_out = walk; + return GIT_SUCCESS; +} + +void git_revwalk_free(git_revwalk *walk) +{ + unsigned int i; + + if (walk == NULL) + return; + + git_revwalk_reset(walk); + git_hashtable_free(walk->commits); + git_pqueue_free(&walk->iterator_time); + + for (i = 0; i < walk->memory_alloc.length; ++i) + free(git_vector_get(&walk->memory_alloc, i)); + + git_vector_free(&walk->memory_alloc); + free(walk); +} + +git_repository *git_revwalk_repository(git_revwalk *walk) +{ + assert(walk); + return walk->repo; +} + +void git_revwalk_sorting(git_revwalk *walk, unsigned int sort_mode) +{ + assert(walk); + + if (walk->walking) + git_revwalk_reset(walk); + + walk->sorting = sort_mode; + + if (walk->sorting & GIT_SORT_TIME) { + walk->get_next = &revwalk_next_timesort; + walk->enqueue = &revwalk_enqueue_timesort; + } else { + walk->get_next = &revwalk_next_unsorted; + walk->enqueue = &revwalk_enqueue_unsorted; + } +} + int git_revwalk_next(git_oid *oid, git_revwalk *walk) { int error; @@ -544,8 +532,11 @@ int git_revwalk_next(git_oid *oid, git_revwalk *walk) } error = walk->get_next(&next, walk); - if (error < GIT_SUCCESS) + if (error < GIT_SUCCESS) { + if (error == GIT_EREVWALKOVER) + git_revwalk_reset(walk); return error; + } git_oid_cpy(oid, &next->oid); return GIT_SUCCESS; @@ -564,7 +555,7 @@ void git_revwalk_reset(git_revwalk *walk) commit->topo_delay = 0; ); - git_pqueue_free(&walk->iterator_time); + git_pqueue_clear(&walk->iterator_time); commit_list_free(&walk->iterator_topo); commit_list_free(&walk->iterator_rand); commit_list_free(&walk->iterator_reverse); -- cgit v1.2.1 From 7341bf87b111dfa0cf12761389e0e7a118363f42 Mon Sep 17 00:00:00 2001 From: Vicent Marti Date: Wed, 16 Mar 2011 23:13:28 +0200 Subject: Refs are now always in-sync on disk --- src/refs.c | 185 ++++++++++++++++++++++++++++++++++++++++++------------------- src/refs.h | 2 + 2 files changed, 131 insertions(+), 56 deletions(-) (limited to 'src') diff --git a/src/refs.c b/src/refs.c index ea39608df..40b80ec9b 100644 --- a/src/refs.c +++ b/src/refs.c @@ -59,16 +59,16 @@ static uint32_t reftable_hash(const void *key, int hash_id) static void reference_free(git_reference *reference); static int reference_create(git_reference **ref_out, git_repository *repo, const char *name, git_rtype type); +static int reference_read(gitfo_buf *file_content, time_t *mtime, const char *repo_path, const char *ref_name); /* loose refs */ static int loose_parse_symbolic(git_reference *ref, gitfo_buf *file_content); static int loose_parse_oid(git_reference *ref, gitfo_buf *file_content); -static int loose_read(gitfo_buf *file_content, const char *name, const char *repo_path); -static int loose_lookup( git_reference **ref_out, git_repository *repo, const char *name, int skip_symbolic); +static int loose_lookup(git_reference **ref_out, git_repository *repo, const char *name, int skip_symbolic); static int loose_write(git_reference *ref); +static int loose_update(git_reference *ref); /* packed refs */ -static int packed_readpack(gitfo_buf *packfile, const char *repo_path); static int packed_parse_peel(reference_oid *tag_ref, const char **buffer_out, const char *buffer_end); static int packed_parse_oid(reference_oid **ref_out, git_repository *repo, const char **buffer_out, const char *buffer_end); static int packed_load(git_repository *repo); @@ -146,12 +146,73 @@ cleanup: return error; } +static int reference_read(gitfo_buf *file_content, time_t *mtime, const char *repo_path, const char *ref_name) +{ + struct stat st; + char path[GIT_PATH_MAX]; + + /* Determine the full path of the file */ + git__joinpath(path, repo_path, ref_name); + + if (gitfo_stat(path, &st) < 0) + return GIT_ENOTFOUND; + + if (S_ISDIR(st.st_mode)) + return GIT_EOBJCORRUPTED; + + if (mtime) + *mtime = st.st_mtime; + + if (file_content) + return gitfo_read_file(file_content, path); + + return GIT_SUCCESS; +} + /***************************************** * Internal methods - Loose references *****************************************/ +static int loose_update(git_reference *ref) +{ + int error; + time_t ref_time; + gitfo_buf ref_file = GITFO_BUF_INIT; + + if (ref->type & GIT_REF_PACKED) + return packed_load(ref->owner); + + error = reference_read(NULL, &ref_time, ref->owner->path_repository, ref->name); + if (error < GIT_SUCCESS) + goto cleanup; + + if (ref_time == ref->mtime) + return GIT_SUCCESS; + + error = reference_read(&ref_file, &ref->mtime, ref->owner->path_repository, ref->name); + if (error < GIT_SUCCESS) + goto cleanup; + + if (ref->type == GIT_REF_SYMBOLIC) + error = loose_parse_symbolic(ref, &ref_file); + else if (ref->type == GIT_REF_OID) + error = loose_parse_oid(ref, &ref_file); + else + error = GIT_EINVALIDREFSTATE; + + gitfo_free_buf(&ref_file); + +cleanup: + if (error != GIT_SUCCESS) { + reference_free(ref); + git_hashtable_remove(ref->owner->references.loose_cache, ref->name); + } + + return error; +} + static int loose_parse_symbolic(git_reference *ref, gitfo_buf *file_content) { const unsigned int header_len = strlen(GIT_SYMREF); @@ -172,6 +233,7 @@ static int loose_parse_symbolic(git_reference *ref, gitfo_buf *file_content) refname_start += header_len; + free(ref_sym->target); ref_sym->target = git__strdup(refname_start); if (ref_sym->target == NULL) return GIT_ENOMEM; @@ -213,27 +275,6 @@ static int loose_parse_oid(git_reference *ref, gitfo_buf *file_content) return GIT_SUCCESS; } -static int loose_read(gitfo_buf *file_content, const char *name, const char *repo_path) -{ - int error = GIT_SUCCESS; - char ref_path[GIT_PATH_MAX]; - - /* Determine the full path of the ref */ - git__joinpath(ref_path, repo_path, name); - - /* Does it even exist ? */ - if (gitfo_exists(ref_path) < GIT_SUCCESS) - return GIT_ENOTFOUND; - - /* A ref can not be a directory */ - if (!gitfo_isdir(ref_path)) - return GIT_ENOTFOUND; - - if (file_content != NULL) - error = gitfo_read_file(file_content, ref_path); - - return error; -} static git_rtype loose_guess_rtype(const char *full_path) { @@ -262,10 +303,11 @@ static int loose_lookup( int error = GIT_SUCCESS; gitfo_buf ref_file = GITFO_BUF_INIT; git_reference *ref = NULL; + time_t ref_time; *ref_out = NULL; - error = loose_read(&ref_file, name, repo->path_repository); + error = reference_read(&ref_file, &ref_time, repo->path_repository, name); if (error < GIT_SUCCESS) goto cleanup; @@ -289,6 +331,7 @@ static int loose_lookup( if (error < GIT_SUCCESS) goto cleanup; + ref->mtime = ref_time; *ref_out = ref; return GIT_SUCCESS; @@ -304,6 +347,7 @@ static int loose_write(git_reference *ref) char ref_path[GIT_PATH_MAX]; int error, contents_size; char *ref_contents = NULL; + struct stat st; assert((ref->type & GIT_REF_PACKED) == 0); @@ -349,6 +393,9 @@ static int loose_write(git_reference *ref) error = git_filebuf_commit(&file); + if (gitfo_stat(ref_path, &st) == GIT_SUCCESS) + ref->mtime = st.st_mtime; + free(ref_contents); return error; @@ -366,19 +413,6 @@ unlock: /***************************************** * Internal methods - Packed references *****************************************/ -static int packed_readpack(gitfo_buf *packfile, const char *repo_path) -{ - char ref_path[GIT_PATH_MAX]; - - /* Determine the full path of the file */ - git__joinpath(ref_path, repo_path, GIT_PACKEDREFS_FILE); - - /* Does it even exist ? */ - if (gitfo_exists(ref_path) < GIT_SUCCESS) - return GIT_ENOTFOUND; - - return gitfo_read_file(packfile, ref_path); -} static int packed_parse_peel( reference_oid *tag_ref, @@ -483,19 +517,40 @@ static int packed_load(git_repository *repo) git_refcache *ref_cache = &repo->references; /* already loaded */ - if (repo->references.packfile != NULL) - return GIT_SUCCESS; + if (repo->references.packfile != NULL) { + time_t packed_time; - repo->references.packfile = git_hashtable_alloc( - default_table_size, - reftable_hash, - (git_hash_keyeq_ptr)strcmp); + /* check if we can read the time of the index; + * if we can read it and it matches the time of the + * index we had previously loaded, we don't need to do + * anything else. + * + * if we cannot load the time (e.g. the packfile + * has disappeared) or the time is different, we + * have to reload the packfile */ - if (repo->references.packfile == NULL) - return GIT_ENOMEM; + if (!reference_read(NULL, &packed_time, repo->path_repository, GIT_PACKEDREFS_FILE) && + packed_time == ref_cache->packfile_time) + return GIT_SUCCESS; - /* read the packfile from disk */ - error = packed_readpack(&packfile, repo->path_repository); + git_hashtable_clear(repo->references.packfile); + } else { + ref_cache->packfile = git_hashtable_alloc( + default_table_size, + reftable_hash, + (git_hash_keyeq_ptr)strcmp); + + if (ref_cache->packfile == NULL) + return GIT_ENOMEM; + } + + /* read the packfile from disk; + * store its modification time to check for future reloads */ + error = reference_read( + &packfile, + &ref_cache->packfile_time, + repo->path_repository, + GIT_PACKEDREFS_FILE); /* there is no packfile on disk; that's ok */ if (error == GIT_ENOTFOUND) @@ -536,7 +591,12 @@ static int packed_load(git_repository *repo) } } + gitfo_free_buf(&packfile); + return GIT_SUCCESS; + cleanup: + git_hashtable_free(ref_cache->packfile); + ref_cache->packfile = NULL; gitfo_free_buf(&packfile); return error; } @@ -834,8 +894,14 @@ cleanup: /* when and only when the packfile has been properly written, * we can go ahead and remove the loose refs */ - if (error == GIT_SUCCESS) + if (error == GIT_SUCCESS) { + struct stat st; + error = packed_remove_loose(repo, &packing_list); + + if (gitfo_stat(pack_file_path, &st) == GIT_SUCCESS) + repo->references.packfile_time = st.st_mtime; + } } else git_filebuf_cleanup(&pack_file); @@ -870,7 +936,7 @@ int git_reference_lookup(git_reference **ref_out, git_repository *repo, const ch /* First, check has been previously loaded and cached */ *ref_out = git_hashtable_lookup(repo->references.loose_cache, normalized_name); if (*ref_out != NULL) - return GIT_SUCCESS; + return loose_update(*ref_out); /* Then check if there is a loose file for that reference.*/ error = loose_lookup(ref_out, repo, normalized_name, 0); @@ -888,12 +954,10 @@ int git_reference_lookup(git_reference **ref_out, git_repository *repo, const ch * If we cannot find a loose reference, we look into the packfile * Load the packfile first if it hasn't been loaded */ - if (!repo->references.packfile) { - /* load all the packed references */ - error = packed_load(repo); - if (error < GIT_SUCCESS) - return error; - } + /* load all the packed references */ + error = packed_load(repo); + if (error < GIT_SUCCESS) + return error; /* Look up on the packfile */ *ref_out = git_hashtable_lookup(repo->references.packfile, normalized_name); @@ -1000,6 +1064,9 @@ const git_oid *git_reference_oid(git_reference *ref) if ((ref->type & GIT_REF_OID) == 0) return NULL; + if (loose_update(ref) < GIT_SUCCESS) + return NULL; + return &((reference_oid *)ref)->oid; } @@ -1010,6 +1077,9 @@ const char *git_reference_target(git_reference *ref) if ((ref->type & GIT_REF_SYMBOLIC) == 0) return NULL; + if (loose_update(ref) < GIT_SUCCESS) + return NULL; + return ((reference_symbolic *)ref)->target; } @@ -1293,6 +1363,9 @@ int git_reference_resolve(git_reference **resolved_ref, git_reference *ref) assert(resolved_ref && ref); *resolved_ref = NULL; + + if ((error = loose_update(ref)) < GIT_SUCCESS) + return error; repo = ref->owner; diff --git a/src/refs.h b/src/refs.h index a542ac0f2..bebb1b97d 100644 --- a/src/refs.h +++ b/src/refs.h @@ -23,11 +23,13 @@ struct git_reference { git_repository *owner; char *name; unsigned int type; + time_t mtime; }; typedef struct { git_hashtable *packfile; git_hashtable *loose_cache; + time_t packfile_time; } git_refcache; -- cgit v1.2.1 From b5c5f0f8086ee4e9bccf0703386fd5219ac380c2 Mon Sep 17 00:00:00 2001 From: Vicent Marti Date: Wed, 16 Mar 2011 23:59:09 +0200 Subject: Fix headers for the new Revision Walker The "oid.h" header is now included instead of "object.h". The old "revwalk.h" header has been removed; it was empty. --- src/commit.c | 1 - src/revwalk.c | 3 ++- src/revwalk.h | 11 ----------- src/tag.c | 1 - src/tree.c | 1 - 5 files changed, 2 insertions(+), 15 deletions(-) delete mode 100644 src/revwalk.h (limited to 'src') diff --git a/src/commit.c b/src/commit.c index 1c5cddf7a..8da170cad 100644 --- a/src/commit.c +++ b/src/commit.c @@ -30,7 +30,6 @@ #include "common.h" #include "commit.h" -#include "revwalk.h" #include "signature.h" #define COMMIT_BASIC_PARSE 0x0 diff --git a/src/revwalk.c b/src/revwalk.c index a2fc092d5..3b7ad34e6 100644 --- a/src/revwalk.c +++ b/src/revwalk.c @@ -25,10 +25,11 @@ #include "common.h" #include "commit.h" -#include "revwalk.h" #include "hashtable.h" #include "pqueue.h" +#include "git2/revwalk.h" + typedef struct commit_object { git_oid oid; uint32_t time; diff --git a/src/revwalk.h b/src/revwalk.h deleted file mode 100644 index 2970d773c..000000000 --- a/src/revwalk.h +++ /dev/null @@ -1,11 +0,0 @@ -#ifndef INCLUDE_revwalk_h__ -#define INCLUDE_revwalk_h__ - -#include "git2/common.h" -#include "git2/revwalk.h" - -#include "commit.h" -#include "repository.h" -#include "hashtable.h" - -#endif /* INCLUDE_revwalk_h__ */ diff --git a/src/tag.c b/src/tag.c index 1379425a1..0489e7e4a 100644 --- a/src/tag.c +++ b/src/tag.c @@ -27,7 +27,6 @@ #include "commit.h" #include "tag.h" #include "signature.h" -#include "revwalk.h" #include "git2/object.h" #include "git2/repository.h" #include "git2/signature.h" diff --git a/src/tree.c b/src/tree.c index 702cccbce..307d41b0d 100644 --- a/src/tree.c +++ b/src/tree.c @@ -25,7 +25,6 @@ #include "common.h" #include "commit.h" -#include "revwalk.h" #include "tree.h" #include "git2/repository.h" #include "git2/object.h" -- cgit v1.2.1 From bb3de0c472b2d5d6b8091c190bee3db79c4b5e27 Mon Sep 17 00:00:00 2001 From: Vicent Marti Date: Wed, 16 Mar 2011 21:35:51 +0200 Subject: Thread safe cache --- src/cache.c | 175 ++++++++++++++++++++++++++++++++++++++++++++++++++++ src/cache.h | 38 ++++++++++++ src/common.h | 9 ++- src/thread-utils.h | 82 ++++++++++++++++++++++-- src/win32/pthread.c | 50 +++------------ src/win32/pthread.h | 9 +-- 6 files changed, 304 insertions(+), 59 deletions(-) create mode 100644 src/cache.c create mode 100644 src/cache.h (limited to 'src') diff --git a/src/cache.c b/src/cache.c new file mode 100644 index 000000000..5a9b8415c --- /dev/null +++ b/src/cache.c @@ -0,0 +1,175 @@ +/* + * This file is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License, version 2, + * as published by the Free Software Foundation. + * + * In addition to the permissions in the GNU General Public License, + * the authors give you unlimited permission to link the compiled + * version of this file into combinations with other programs, + * and to distribute those combinations without any restriction + * coming from the use of this file. (The General Public License + * restrictions do apply in other respects; for example, they cover + * modification of the file, and distribution when not linked into + * a combined executable.) + * + * This file is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; see the file COPYING. If not, write to + * the Free Software Foundation, 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#include "common.h" +#include "repository.h" +#include "commit.h" +#include "thread-utils.h" +#include "cache.h" + +#define GIT_CACHE_OPENADR 3 + + +GIT_INLINE(int) cached_obj_compare(git_cached_obj *obj, const git_oid *oid) +{ + return git_oid_cmp(&obj->oid, oid); +} + +GIT_INLINE(void) cached_obj_incref(git_cached_obj *obj) +{ + git_atomic_inc(&obj->refcount); +} + +GIT_INLINE(void) cached_obj_decref(git_cached_obj *obj, git_cached_obj_freeptr free_obj) +{ + if (git_atomic_dec(&obj->refcount) == 0) + free_obj(obj); +} + + +void git_cache_init(git_cache *cache, size_t size, git_cached_obj_freeptr free_ptr) +{ + size_t i; + + if (size < 8) + size = 8; + + /* round up size to closest power of 2 */ + size--; + size |= size >> 1; + size |= size >> 2; + size |= size >> 4; + size |= size >> 8; + size |= size >> 16; + + cache->size_mask = size; + cache->lru_count = 0; + cache->free_obj = free_ptr; + + cache->nodes = git__malloc((size + 1) * sizeof(cache_node)); + + for (i = 0; i < (size + 1); ++i) { + git_mutex_init(&cache->nodes[i].lock); + cache->nodes[i].ptr = NULL; + cache->nodes[i].lru = 0; + } +} + +void git_cache_free(git_cache *cache) +{ + size_t i; + + for (i = 0; i < (cache->size_mask + 1); ++i) { + cached_obj_decref(cache->nodes[i].ptr, cache->free_obj); + git_mutex_free(&cache->nodes[i].lock); + } + + free(cache->nodes); +} + +void *git_cache_get(git_cache *cache, const git_oid *oid) +{ + const uint32_t *hash; + size_t i, pos, found = 0; + cache_node *node; + + hash = (const uint32_t *)oid->id; + + for (i = 0; !found && i < GIT_CACHE_OPENADR; ++i) { + pos = hash[i] & cache->size_mask; + node = &cache->nodes[pos]; + + git_mutex_lock(&node->lock); + { + if (cached_obj_compare(node->ptr, oid) == 0) { + cached_obj_incref(node->ptr); + node->lru = ++cache->lru_count; + found = 1; + } + } + git_mutex_unlock(&node->lock); + } + + + return found ? node->ptr : NULL; +} + +void *git_cache_try_store(git_cache *cache, void *entry) +{ + cache_node *nodes[GIT_CACHE_OPENADR], *lru_node; + const uint32_t *hash; + const git_oid *oid; + size_t i, stored = 0; + + oid = &((git_cached_obj*)entry)->oid; + hash = (const uint32_t *)oid->id; + + /* increase the refcount on this object, because + * the cache now owns it */ + cached_obj_incref(entry); + + for (i = 0; i < GIT_CACHE_OPENADR; ++i) { + size_t pos = hash[i] & cache->size_mask; + + nodes[i] = &cache->nodes[pos]; + git_mutex_lock(&nodes[i]->lock); + } + + lru_node = nodes[0]; + + for (i = 0; !stored && entry && i < GIT_CACHE_OPENADR; ++i) { + + if (nodes[i]->ptr == NULL) { + nodes[i]->ptr = entry; + nodes[i]->lru = ++cache->lru_count; + stored = 1; + } else if (cached_obj_compare(nodes[i]->ptr, oid) == 0) { + cached_obj_decref(entry, cache->free_obj); + entry = nodes[i]->ptr; + stored = 1; + } + + if (nodes[i]->lru < lru_node->lru) + lru_node = nodes[i]; + } + + if (!stored) { + void *old_entry = lru_node->ptr; + assert(old_entry); + + cached_obj_decref(old_entry, cache->free_obj); + lru_node->ptr = entry; + lru_node->lru = ++cache->lru_count; + } + + /* increase the refcount again, because we are + * returning it to the user */ + cached_obj_incref(entry); + + for (i = 0; i < GIT_CACHE_OPENADR; ++i) + git_mutex_unlock(&nodes[i]->lock); + + return entry; +} diff --git a/src/cache.h b/src/cache.h new file mode 100644 index 000000000..9f525e68c --- /dev/null +++ b/src/cache.h @@ -0,0 +1,38 @@ +#ifndef INCLUDE_cache_h__ +#define INCLUDE_cache_h__ + +#include "git2/common.h" +#include "git2/oid.h" +#include "git2/odb.h" + +#include "thread-utils.h" + +typedef void (*git_cached_obj_freeptr)(void *); + +typedef struct { + git_oid oid; + git_atomic refcount; +} git_cached_obj; + +typedef struct { + git_cached_obj *ptr; + git_mutex lock; + unsigned int lru; +} cache_node; + +typedef struct { + cache_node *nodes; + + unsigned int lru_count; + size_t size_mask; + git_cached_obj_freeptr free_obj; +} git_cache; + + +void git_cache_init(git_cache *cache, size_t size, git_cached_obj_freeptr free_ptr); +void git_cache_free(git_cache *cache); + +void *git_cache_try_store(git_cache *cache, void *entry); +void *git_cache_get(git_cache *cache, const git_oid *oid); + +#endif diff --git a/src/common.h b/src/common.h index 723085a2b..5ad878e26 100644 --- a/src/common.h +++ b/src/common.h @@ -31,18 +31,21 @@ # include # include "msvc-compat.h" # include "mingw-compat.h" -# include "win32/pthread.h" +# ifdef GIT_THREADS +# include "win32/pthread.h" +#endif # define snprintf _snprintf typedef SSIZE_T ssize_t; #else - # include # include -# include +# ifdef GIT_THREADS +# include +# endif #endif #include "git2/common.h" diff --git a/src/thread-utils.h b/src/thread-utils.h index e8372e731..1cf0e3407 100644 --- a/src/thread-utils.h +++ b/src/thread-utils.h @@ -2,6 +2,19 @@ #define INCLUDE_thread_utils_h__ + +/* Common operations even if threading has been disabled */ +typedef struct { + volatile int val; +} git_atomic; + +static inline void git_atomic_set(git_atomic *a, int val) +{ + a->val = val; +} + +#ifdef GIT_THREADS + #define git_thread pthread_t #define git_thread_create(thread, attr, start_routine, arg) pthread_create(thread, attr, start_routine, arg) #define git_thread_kill(thread) pthread_cancel(thread) @@ -15,13 +28,70 @@ #define git_mutex_unlock(a) pthread_mutex_unlock(a) #define git_mutex_free(a) pthread_mutex_destroy(a) +/* Pthreads condition vars -- disabled by now */ +#define git_cond unsigned int //pthread_cond_t +#define git_cond_init(c, a) (void)0 //pthread_cond_init(c, a) +#define git_cond_free(c) (void)0 //pthread_cond_destroy(c) +#define git_cond_wait(c, l) (void)0 //pthread_cond_wait(c, l) +#define git_cond_signal(c) (void)0 //pthread_cond_signal(c) +#define git_cond_broadcast(c) (void)0 //pthread_cond_broadcast(c) + +static inline int git_atomic_inc(git_atomic *a) +{ +#ifdef __GNUC__ + return __sync_add_and_fetch(&a->val, 1); +#elif defined(_MSC_VER) + return InterlockedIncrement(&a->val); +#else +# error "Unsupported architecture for atomic operations" +#endif +} + +static inline int git_atomic_dec(git_atomic *a) +{ +#ifdef __GNUC__ + return __sync_sub_and_fetch(&a->val, 1); +#elif defined(_MSC_VER) + return InterlockedDecrement(&a->val); +#else +# error "Unsupported architecture for atomic operations" +#endif +} + +#else + +#define git_thread unsigned int +#define git_thread_create(thread, attr, start_routine, arg) (void)0 +#define git_thread_kill(thread) (void)0 +#define git_thread_exit(status) (void)0 +#define git_thread_join(id, status) (void)0 + +/* Pthreads Mutex */ +#define git_mutex unsigned int +#define git_mutex_init(a) (void)0 +#define git_mutex_lock(a) (void)0 +#define git_mutex_unlock(a) (void)0 +#define git_mutex_free(a) (void)0 + /* Pthreads condition vars */ -#define git_cond pthread_cond_t -#define git_cond_init(c, a) pthread_cond_init(c, a) -#define git_cond_free(c) pthread_cond_destroy(c) -#define git_cond_wait(c, l) pthread_cond_wait(c, l) -#define git_cond_signal(c) pthread_cond_signal(c) -#define git_cond_broadcast(c) pthread_cond_broadcast(c) +#define git_cond unsigned int +#define git_cond_init(c, a) (void)0 +#define git_cond_free(c) (void)0 +#define git_cond_wait(c, l) (void)0 +#define git_cond_signal(c) (void)0 +#define git_cond_broadcast(c) (void)0 + +static inline int git_atomic_inc(git_atomic *a) +{ + return ++a->val; +} + +static inline int git_atomic_dec(git_atomic *a) +{ + return --a->val; +} + +#endif extern int git_online_cpus(void); diff --git a/src/win32/pthread.c b/src/win32/pthread.c index fffff81df..f47364a76 100644 --- a/src/win32/pthread.c +++ b/src/win32/pthread.c @@ -36,32 +36,6 @@ int pthread_create(pthread_t *GIT_RESTRICT thread, return *thread ? GIT_SUCCESS : GIT_EOSERR; } -int pthread_cond_signal(pthread_cond_t *cond) -{ - WakeConditionVariable(cond); - return 0; -} - -int pthread_cond_wait(pthread_cond_t *GIT_RESTRICT cond, - pthread_mutex_t *GIT_RESTRICT mutex) -{ - int ret; - ret = SleepConditionVariableCS(cond, mutex, INFINITE); - return -(!ret); -} - -int pthread_mutex_lock(pthread_mutex_t *mutex) -{ - EnterCriticalSection(mutex); - return 0; -} - -int pthread_mutex_unlock(pthread_mutex_t *mutex) -{ - LeaveCriticalSection(mutex); - return 0; -} - int pthread_join(pthread_t thread, void **value_ptr) { int ret; @@ -71,9 +45,11 @@ int pthread_join(pthread_t thread, void **value_ptr) return -(!!ret); } -int pthread_cond_broadcast(pthread_cond_t *cond) +int pthread_mutex_init(pthread_mutex_t *GIT_RESTRICT mutex, + const pthread_mutexattr_t *GIT_RESTRICT GIT_UNUSED(mutexattr)) { - WakeAllConditionVariable(cond); + GIT_UNUSED_ARG(mutexattr); + InitializeCriticalSection(mutex); return 0; } @@ -84,25 +60,15 @@ int pthread_mutex_destroy(pthread_mutex_t *mutex) return -(!ret); } -int pthread_cond_destroy(pthread_cond_t *GIT_UNUSED(cond)) -{ - GIT_UNUSED_ARG(cond); - return 0; -} - -int pthread_cond_init(pthread_cond_t *GIT_RESTRICT cond, - const pthread_condattr_t *GIT_RESTRICT GIT_UNUSED(condattr)) +int pthread_mutex_lock(pthread_mutex_t *mutex) { - GIT_UNUSED_ARG(condattr); - InitializeConditionVariable(cond); + EnterCriticalSection(mutex); return 0; } -int pthread_mutex_init(pthread_mutex_t *GIT_RESTRICT mutex, - const pthread_mutexattr_t *GIT_RESTRICT GIT_UNUSED(mutexattr)) +int pthread_mutex_unlock(pthread_mutex_t *mutex) { - GIT_UNUSED_ARG(mutexattr); - InitializeCriticalSection(mutex); + LeaveCriticalSection(mutex); return 0; } diff --git a/src/win32/pthread.h b/src/win32/pthread.h index ff694e303..10949f1eb 100644 --- a/src/win32/pthread.h +++ b/src/win32/pthread.h @@ -40,7 +40,6 @@ typedef int pthread_mutexattr_t; typedef int pthread_condattr_t; typedef int pthread_attr_t; typedef CRITICAL_SECTION pthread_mutex_t; -typedef CONDITION_VARIABLE pthread_cond_t; typedef HANDLE pthread_t; #define PTHREAD_MUTEX_INITIALIZER {(void*)-1}; @@ -56,12 +55,6 @@ int pthread_mutex_destroy(pthread_mutex_t *); int pthread_mutex_lock(pthread_mutex_t *); int pthread_mutex_unlock(pthread_mutex_t *); -int pthread_cond_init(pthread_cond_t *GIT_RESTRICT, const pthread_condattr_t *GIT_RESTRICT); -int pthread_cond_destroy(pthread_cond_t *); -int pthread_cond_broadcast(pthread_cond_t *); -int pthread_cond_signal(pthread_cond_t *); -int pthread_cond_wait(pthread_cond_t *GIT_RESTRICT, pthread_mutex_t *GIT_RESTRICT); - int pthread_num_processors_np(void); -#endif \ No newline at end of file +#endif -- cgit v1.2.1 From 72a3fe42fb7208712bbe8f0981f4c6274c05e9c3 Mon Sep 17 00:00:00 2001 From: Vicent Marti Date: Fri, 18 Mar 2011 19:38:49 +0200 Subject: I broke your bindings Hey. Apologies in advance -- I broke your bindings. This is a major commit that includes a long-overdue redesign of the whole object-database structure. This is expected to be the last major external API redesign of the library until the first non-alpha release. Please get your bindings up to date with these changes. They will be included in the next minor release. Sorry again! Major features include: - Real caching and refcounting on parsed objects - Real caching and refcounting on objects read from the ODB - Streaming writes & reads from the ODB - Single-method writes for all object types - The external API is now partially thread-safe The speed increases are significant in all aspects, specially when reading an object several times from the ODB (revwalking) and when writing big objects to the ODB. Here's a full changelog for the external API: blob.h ------ - Remove `git_blob_new` - Remove `git_blob_set_rawcontent` - Remove `git_blob_set_rawcontent_fromfile` - Rename `git_blob_writefile` -> `git_blob_create_fromfile` - Change `git_blob_create_fromfile`: The `path` argument is now relative to the repository's working dir - Add `git_blob_create_frombuffer` commit.h -------- - Remove `git_commit_new` - Remove `git_commit_add_parent` - Remove `git_commit_set_message` - Remove `git_commit_set_committer` - Remove `git_commit_set_author` - Remove `git_commit_set_tree` - Add `git_commit_create` - Add `git_commit_create_v` - Add `git_commit_create_o` - Add `git_commit_create_ov` tag.h ----- - Remove `git_tag_new` - Remove `git_tag_set_target` - Remove `git_tag_set_name` - Remove `git_tag_set_tagger` - Remove `git_tag_set_message` - Add `git_tag_create` - Add `git_tag_create_o` tree.h ------ - Change `git_tree_entry_2object`: New signature is `(git_object **object_out, git_repository *repo, git_tree_entry *entry)` - Remove `git_tree_new` - Remove `git_tree_add_entry` - Remove `git_tree_remove_entry_byindex` - Remove `git_tree_remove_entry_byname` - Remove `git_tree_clearentries` - Remove `git_tree_entry_set_id` - Remove `git_tree_entry_set_name` - Remove `git_tree_entry_set_attributes` object.h ------------ - Remove `git_object_new - Remove `git_object_write` - Change `git_object_close`: This method is now *mandatory*. Not closing an object causes a memory leak. odb.h ----- - Remove type `git_rawobj` - Remove `git_rawobj_close` - Rename `git_rawobj_hash` -> `git_odb_hash` - Change `git_odb_hash`: New signature is `(git_oid *id, const void *data, size_t len, git_otype type)` - Add type `git_odb_object` - Add `git_odb_object_close` - Change `git_odb_read`: New signature is `(git_odb_object **out, git_odb *db, const git_oid *id)` - Change `git_odb_read_header`: New signature is `(size_t *len_p, git_otype *type_p, git_odb *db, const git_oid *id)` - Remove `git_odb_write` - Add `git_odb_open_wstream` - Add `git_odb_open_rstream` odb_backend.h ------------- - Change type `git_odb_backend`: New internal signatures are as follows int (* read)(void **, size_t *, git_otype *, struct git_odb_backend *, const git_oid *) int (* read_header)(size_t *, git_otype *, struct git_odb_backend *, const git_oid *) int (* writestream)(struct git_odb_stream **, struct git_odb_backend *, size_t, git_otype) int (* readstream)( struct git_odb_stream **, struct git_odb_backend *, const git_oid *) - Add type `git_odb_stream` - Add enum `git_odb_streammode` Signed-off-by: Vicent Marti --- src/backends/sqlite.c | 9 ++ src/blob.c | 113 +++++++++------------ src/blob.h | 6 +- src/cache.c | 46 +++------ src/cache.h | 21 ++++ src/commit.c | 268 +++++++++++++++++++++++++++--------------------- src/commit.h | 4 +- src/delta-apply.h | 2 + src/errors.c | 3 +- src/filebuf.c | 188 ++++++++++++++++++++++++++-------- src/filebuf.h | 22 ++-- src/fileops.c | 90 +++++++++++++++- src/fileops.h | 2 + src/index.c | 2 +- src/object.c | 276 +++++--------------------------------------------- src/odb.c | 133 +++++++++++++++--------- src/odb.h | 18 +++- src/odb_loose.c | 264 +++++++++++++++++++++++------------------------ src/odb_pack.c | 19 ++-- src/oid.c | 14 ++- src/repository.c | 72 +------------ src/repository.h | 28 ++--- src/revwalk.c | 13 +-- src/signature.c | 23 +++-- src/signature.h | 2 +- src/tag.c | 144 ++++++++++++++------------ src/tag.h | 4 +- src/thread-utils.h | 10 +- src/tree.c | 191 ++-------------------------------- src/tree.h | 7 +- src/util.h | 2 + 31 files changed, 918 insertions(+), 1078 deletions(-) (limited to 'src') diff --git a/src/backends/sqlite.c b/src/backends/sqlite.c index b4c941a59..72d7b4d8e 100644 --- a/src/backends/sqlite.c +++ b/src/backends/sqlite.c @@ -272,4 +272,13 @@ cleanup: return GIT_ERROR; } +#else + +int git_odb_backend_sqlite(git_odb_backend **GIT_UNUSED(backend_out), const char *GIT_UNUSED(sqlite_db)) +{ + GIT_UNUSED_ARG(backend_out); + GIT_UNUSED_ARG(sqlite_db); + return GIT_ENOTIMPLEMENTED; +} + #endif /* HAVE_SQLITE3 */ diff --git a/src/blob.c b/src/blob.c index 1e03b6b67..bc0a08a8a 100644 --- a/src/blob.c +++ b/src/blob.c @@ -33,104 +33,89 @@ const void *git_blob_rawcontent(git_blob *blob) { assert(blob); - - if (blob->content.data != NULL) - return blob->content.data; - - if (blob->object.in_memory) - return NULL; - - if (!blob->object.source.open && git_object__source_open((git_object *)blob) < GIT_SUCCESS) - return NULL; - - return blob->object.source.raw.data; + return blob->odb_object->raw.data; } int git_blob_rawsize(git_blob *blob) { assert(blob); - - if (blob->content.data != NULL) - return blob->content.len; - - return blob->object.source.raw.len; + return blob->odb_object->raw.len; } void git_blob__free(git_blob *blob) { - gitfo_free_buf(&blob->content); + git_odb_object_close(blob->odb_object); free(blob); } -int git_blob__parse(git_blob *blob) +int git_blob__parse(git_blob *blob, git_odb_object *odb_obj) { assert(blob); + git_cached_obj_incref((git_cached_obj *)odb_obj); + blob->odb_object = odb_obj; return GIT_SUCCESS; } -int git_blob__writeback(git_blob *blob, git_odb_source *src) -{ - assert(blob->object.modified); - - if (blob->content.data == NULL) - return GIT_EMISSINGOBJDATA; - - return git__source_write(src, blob->content.data, blob->content.len); -} - -int git_blob_set_rawcontent(git_blob *blob, const void *buffer, size_t len) +int git_blob_create_frombuffer(git_oid *oid, git_repository *repo, const void *buffer, size_t len) { - assert(blob && buffer); - - blob->object.modified = 1; - - git_object__source_close((git_object *)blob); - - if (blob->content.data != NULL) - gitfo_free_buf(&blob->content); + int error; + git_odb_stream *stream; - blob->content.data = git__malloc(len); - blob->content.len = len; + if ((error = git_odb_open_wstream(&stream, repo->db, len, GIT_OBJ_BLOB)) < GIT_SUCCESS) + return error; - if (blob->content.data == NULL) - return GIT_ENOMEM; + stream->write(stream, buffer, len); - memcpy(blob->content.data, buffer, len); + error = stream->finalize_write(oid, stream); + stream->free(stream); - return GIT_SUCCESS; + return error; } -int git_blob_set_rawcontent_fromfile(git_blob *blob, const char *filename) +int git_blob_create_fromfile(git_oid *oid, git_repository *repo, const char *path) { - assert(blob && filename); - blob->object.modified = 1; + int error, fd; + char full_path[GIT_PATH_MAX]; + char buffer[2048]; + git_off_t size; + git_odb_stream *stream; - if (blob->content.data != NULL) - gitfo_free_buf(&blob->content); - - return gitfo_read_file(&blob->content, filename); -} + if (repo->path_workdir == NULL) + return GIT_ENOTFOUND; -int git_blob_writefile(git_oid *written_id, git_repository *repo, const char *path) -{ - int error; - git_blob *blob; + git__joinpath(full_path, repo->path_workdir, path); - if (gitfo_exists(path) < 0) + if ((fd = gitfo_open(full_path, O_RDONLY)) < 0) return GIT_ENOTFOUND; - if ((error = git_blob_new(&blob, repo)) < GIT_SUCCESS) - return error; + if ((size = gitfo_size(fd)) < 0 || !git__is_sizet(size)) { + gitfo_close(fd); + return GIT_EOSERR; + } - if ((error = git_blob_set_rawcontent_fromfile(blob, path)) < GIT_SUCCESS) + if ((error = git_odb_open_wstream(&stream, repo->db, (size_t)size, GIT_OBJ_BLOB)) < GIT_SUCCESS) { + gitfo_close(fd); return error; + } - if ((error = git_object_write((git_object *)blob)) < GIT_SUCCESS) - return error; + while (size > 0) { + ssize_t read_len; - git_oid_cpy(written_id, git_object_id((git_object *)blob)); + read_len = read(fd, buffer, sizeof(buffer)); - git_object_close((git_object*)blob); - return GIT_SUCCESS; + if (read_len < 0) { + gitfo_close(fd); + stream->free(stream); + return GIT_EOSERR; + } + + stream->write(stream, buffer, read_len); + size -= read_len; + } + + error = stream->finalize_write(oid, stream); + stream->free(stream); + + return error; } diff --git a/src/blob.h b/src/blob.h index febc296fe..4300d7e54 100644 --- a/src/blob.h +++ b/src/blob.h @@ -3,15 +3,15 @@ #include "git2/blob.h" #include "repository.h" +#include "odb.h" #include "fileops.h" struct git_blob { git_object object; - gitfo_buf content; + git_odb_object *odb_object; }; void git_blob__free(git_blob *blob); -int git_blob__parse(git_blob *blob); -int git_blob__writeback(git_blob *blob, git_odb_source *src); +int git_blob__parse(git_blob *blob, git_odb_object *obj); #endif diff --git a/src/cache.c b/src/cache.c index 5a9b8415c..fdf5c1b80 100644 --- a/src/cache.c +++ b/src/cache.c @@ -32,23 +32,6 @@ #define GIT_CACHE_OPENADR 3 -GIT_INLINE(int) cached_obj_compare(git_cached_obj *obj, const git_oid *oid) -{ - return git_oid_cmp(&obj->oid, oid); -} - -GIT_INLINE(void) cached_obj_incref(git_cached_obj *obj) -{ - git_atomic_inc(&obj->refcount); -} - -GIT_INLINE(void) cached_obj_decref(git_cached_obj *obj, git_cached_obj_freeptr free_obj) -{ - if (git_atomic_dec(&obj->refcount) == 0) - free_obj(obj); -} - - void git_cache_init(git_cache *cache, size_t size, git_cached_obj_freeptr free_ptr) { size_t i; @@ -82,7 +65,9 @@ void git_cache_free(git_cache *cache) size_t i; for (i = 0; i < (cache->size_mask + 1); ++i) { - cached_obj_decref(cache->nodes[i].ptr, cache->free_obj); + if (cache->nodes[i].ptr) + git_cached_obj_decref(cache->nodes[i].ptr, cache->free_obj); + git_mutex_free(&cache->nodes[i].lock); } @@ -103,8 +88,8 @@ void *git_cache_get(git_cache *cache, const git_oid *oid) git_mutex_lock(&node->lock); { - if (cached_obj_compare(node->ptr, oid) == 0) { - cached_obj_incref(node->ptr); + if (node->ptr && git_cached_obj_compare(node->ptr, oid) == 0) { + git_cached_obj_incref(node->ptr); node->lru = ++cache->lru_count; found = 1; } @@ -121,14 +106,14 @@ void *git_cache_try_store(git_cache *cache, void *entry) cache_node *nodes[GIT_CACHE_OPENADR], *lru_node; const uint32_t *hash; const git_oid *oid; - size_t i, stored = 0; + size_t i; oid = &((git_cached_obj*)entry)->oid; hash = (const uint32_t *)oid->id; /* increase the refcount on this object, because * the cache now owns it */ - cached_obj_incref(entry); + git_cached_obj_incref(entry); for (i = 0; i < GIT_CACHE_OPENADR; ++i) { size_t pos = hash[i] & cache->size_mask; @@ -139,34 +124,35 @@ void *git_cache_try_store(git_cache *cache, void *entry) lru_node = nodes[0]; - for (i = 0; !stored && entry && i < GIT_CACHE_OPENADR; ++i) { + for (i = 0; i < GIT_CACHE_OPENADR; ++i) { if (nodes[i]->ptr == NULL) { nodes[i]->ptr = entry; nodes[i]->lru = ++cache->lru_count; - stored = 1; - } else if (cached_obj_compare(nodes[i]->ptr, oid) == 0) { - cached_obj_decref(entry, cache->free_obj); + break; + } else if (git_cached_obj_compare(nodes[i]->ptr, oid) == 0) { + git_cached_obj_decref(entry, cache->free_obj); entry = nodes[i]->ptr; - stored = 1; + nodes[i]->lru = ++cache->lru_count; + break; } if (nodes[i]->lru < lru_node->lru) lru_node = nodes[i]; } - if (!stored) { + if (i == GIT_CACHE_OPENADR) { void *old_entry = lru_node->ptr; assert(old_entry); - cached_obj_decref(old_entry, cache->free_obj); + git_cached_obj_decref(old_entry, cache->free_obj); lru_node->ptr = entry; lru_node->lru = ++cache->lru_count; } /* increase the refcount again, because we are * returning it to the user */ - cached_obj_incref(entry); + git_cached_obj_incref(entry); for (i = 0; i < GIT_CACHE_OPENADR; ++i) git_mutex_unlock(&nodes[i]->lock); diff --git a/src/cache.h b/src/cache.h index 9f525e68c..975aaff7e 100644 --- a/src/cache.h +++ b/src/cache.h @@ -7,6 +7,8 @@ #include "thread-utils.h" +#define GIT_DEFAULT_CACHE_SIZE 128 + typedef void (*git_cached_obj_freeptr)(void *); typedef struct { @@ -35,4 +37,23 @@ void git_cache_free(git_cache *cache); void *git_cache_try_store(git_cache *cache, void *entry); void *git_cache_get(git_cache *cache, const git_oid *oid); + +GIT_INLINE(int) git_cached_obj_compare(git_cached_obj *obj, const git_oid *oid) +{ + return git_oid_cmp(&obj->oid, oid); +} + +GIT_INLINE(void) git_cached_obj_incref(git_cached_obj *obj) +{ + git_atomic_inc(&obj->refcount); +} + +GIT_INLINE(void) git_cached_obj_decref(git_cached_obj *obj, git_cached_obj_freeptr free_obj) +{ + if (git_atomic_dec(&obj->refcount) == 0) + free_obj(obj); +} + + + #endif diff --git a/src/commit.c b/src/commit.c index 8da170cad..2670c2e0a 100644 --- a/src/commit.c +++ b/src/commit.c @@ -29,9 +29,12 @@ #include "git2/signature.h" #include "common.h" +#include "odb.h" #include "commit.h" #include "signature.h" +#include + #define COMMIT_BASIC_PARSE 0x0 #define COMMIT_FULL_PARSE 0x1 @@ -71,35 +74,165 @@ const git_oid *git_commit_id(git_commit *c) return git_object_id((git_object *)c); } -int git_commit__writeback(git_commit *commit, git_odb_source *src) + +int git_commit_create_v( + git_oid *oid, + git_repository *repo, + const char *update_ref, + const git_signature *author, + const git_signature *committer, + const char *message, + const git_oid *tree_oid, + int parent_count, + ...) { - unsigned int i; + va_list ap; + int i, error; + const git_oid **oids; - git__write_oid(src, "tree", &commit->tree_oid); + oids = git__malloc(parent_count * sizeof(git_oid *)); - for (i = 0; i < commit->parent_oids.length; ++i) { - git_oid *parent_oid; + va_start(ap, parent_count); + for (i = 0; i < parent_count; ++i) + oids[i] = va_arg(ap, const git_oid *); + va_end(ap); - parent_oid = git_vector_get(&commit->parent_oids, i); - git__write_oid(src, "parent", parent_oid); - } + error = git_commit_create( + oid, repo, update_ref, author, committer, message, + tree_oid, parent_count, oids); + + free(oids); + return error; +} + +int git_commit_create_ov( + git_oid *oid, + git_repository *repo, + const char *update_ref, + const git_signature *author, + const git_signature *committer, + const char *message, + const git_tree *tree, + int parent_count, + ...) +{ + va_list ap; + int i, error; + const git_oid **oids; + + oids = git__malloc(parent_count * sizeof(git_oid *)); + + va_start(ap, parent_count); + for (i = 0; i < parent_count; ++i) + oids[i] = git_object_id(va_arg(ap, const git_object *)); + va_end(ap); + + error = git_commit_create( + oid, repo, update_ref, author, committer, message, + git_object_id((git_object *)tree), + parent_count, oids); + + free(oids); + return error; +} + +int git_commit_create_o( + git_oid *oid, + git_repository *repo, + const char *update_ref, + const git_signature *author, + const git_signature *committer, + const char *message, + const git_tree *tree, + int parent_count, + const git_commit *parents[]) +{ + int i, error; + const git_oid **oids; + + oids = git__malloc(parent_count * sizeof(git_oid *)); + + for (i = 0; i < parent_count; ++i) + oids[i] = git_object_id((git_object *)parents[i]); + + error = git_commit_create( + oid, repo, update_ref, author, committer, message, + git_object_id((git_object *)tree), + parent_count, oids); + + free(oids); + return error; +} - if (commit->author == NULL) - return GIT_EMISSINGOBJDATA; +int git_commit_create( + git_oid *oid, + git_repository *repo, + const char *update_ref, + const git_signature *author, + const git_signature *committer, + const char *message, + const git_oid *tree_oid, + int parent_count, + const git_oid *parents[]) +{ + size_t final_size = 0; + int message_length, author_length, committer_length; + + char *author_str, *committer_str; + + int error, i; + git_odb_stream *stream; + + message_length = strlen(message); + author_length = git_signature__write(&author_str, "author", author); + committer_length = git_signature__write(&committer_str, "committer", committer); + + if (author_length < 0 || committer_length < 0) + return GIT_ENOMEM; + + final_size += GIT_OID_LINE_LENGTH("tree"); + final_size += GIT_OID_LINE_LENGTH("parent") * parent_count; + final_size += author_length; + final_size += committer_length; + final_size += 1 + message_length; + + if ((error = git_odb_open_wstream(&stream, repo->db, final_size, GIT_OBJ_COMMIT)) < GIT_SUCCESS) + return error; + + git__write_oid(stream, "tree", tree_oid); - git_signature__write(src, "author", commit->author); + for (i = 0; i < parent_count; ++i) + git__write_oid(stream, "parent", parents[i]); - if (commit->committer == NULL) - return GIT_EMISSINGOBJDATA; + stream->write(stream, author_str, author_length); + free(author_str); - git_signature__write(src, "committer", commit->committer); + stream->write(stream, committer_str, committer_length); + free(committer_str); - if (commit->message != NULL) { - git__source_write(src, "\n", 1); - git__source_write(src, commit->message, strlen(commit->message)); + + stream->write(stream, "\n", 1); + stream->write(stream, message, message_length); + + error = stream->finalize_write(oid, stream); + stream->free(stream); + + if (error == GIT_SUCCESS && update_ref != NULL) { + git_reference *head; + + error = git_reference_lookup(&head, repo, update_ref); + if (error < GIT_SUCCESS) + return error; + + if (git_reference_type(head) == GIT_REF_SYMBOLIC) { + if ((error = git_reference_resolve(&head, head)) < GIT_SUCCESS) + return error; + } + + error = git_reference_set_oid(head, oid); } - return GIT_SUCCESS; + return error; } int commit_parse_buffer(git_commit *commit, void *data, size_t len) @@ -110,12 +243,7 @@ int commit_parse_buffer(git_commit *commit, void *data, size_t len) git_oid parent_oid; int error; - /* first parse; the vector hasn't been initialized yet */ - if (commit->parent_oids.contents == NULL) { - git_vector_init(&commit->parent_oids, 4, NULL); - } - - clear_parents(commit); + git_vector_init(&commit->parent_oids, 4, NULL); if ((error = git__parse_oid(&commit->tree_oid, &buffer, buffer_end, "tree ")) < GIT_SUCCESS) return error; @@ -134,17 +262,11 @@ int commit_parse_buffer(git_commit *commit, void *data, size_t len) return GIT_ENOMEM; } - if (commit->author) - git_signature_free(commit->author); - commit->author = git__malloc(sizeof(git_signature)); if ((error = git_signature__parse(commit->author, &buffer, buffer_end, "author ")) < GIT_SUCCESS) return error; /* Always parse the committer; we need the commit time */ - if (commit->committer) - git_signature_free(commit->committer); - commit->committer = git__malloc(sizeof(git_signature)); if ((error = git_signature__parse(commit->committer, &buffer, buffer_end, "committer ")) < GIT_SUCCESS) return error; @@ -176,11 +298,10 @@ int commit_parse_buffer(git_commit *commit, void *data, size_t len) return GIT_SUCCESS; } -int git_commit__parse(git_commit *commit) +int git_commit__parse(git_commit *commit, git_odb_object *obj) { - assert(commit && commit->object.source.open); - return commit_parse_buffer(commit, - commit->object.source.raw.data, commit->object.source.raw.len); + assert(commit); + return commit_parse_buffer(commit, obj->raw.data, obj->raw.len); } #define GIT_COMMIT_GETTER(_rvalue, _name, _return) \ @@ -218,82 +339,3 @@ int git_commit_parent(git_commit **parent, git_commit *commit, unsigned int n) } - -int git_commit_set_tree(git_commit *commit, git_tree *tree) -{ - const git_oid *oid; - - assert(commit && tree); - - if ((oid = git_object_id((git_object *)tree)) == NULL) - return GIT_EMISSINGOBJDATA; - - commit->object.modified = 1; - git_oid_cpy(&commit->tree_oid, oid); - return GIT_SUCCESS; -} - -int git_commit_add_parent(git_commit *commit, git_commit *new_parent) -{ - const git_oid *parent_oid; - git_oid *new_oid; - assert(commit && new_parent); - - if ((parent_oid = git_object_id((git_object *)new_parent)) == NULL) - return GIT_EMISSINGOBJDATA; - - new_oid = git__malloc(sizeof(git_oid)); - if (new_oid == NULL) - return GIT_ENOMEM; - - commit->object.modified = 1; - git_oid_cpy(new_oid, parent_oid); - return git_vector_insert(&commit->parent_oids, new_oid); -} - -void git_commit_set_author(git_commit *commit, const git_signature *author_sig) -{ - assert(commit && author_sig); - commit->object.modified = 1; - - git_signature_free(commit->author); - commit->author = git_signature_dup(author_sig); -} - -void git_commit_set_committer(git_commit *commit, const git_signature *committer_sig) -{ - assert(commit && committer_sig); - commit->object.modified = 1; - - git_signature_free(commit->committer); - commit->committer = git_signature_dup(committer_sig); -} - -void git_commit_set_message(git_commit *commit, const char *message) -{ - const char *line_end; - size_t message_len; - - commit->object.modified = 1; - - if (commit->message) - free(commit->message); - - if (commit->message_short) - free(commit->message_short); - - commit->message = git__strdup(message); - - /* Short message */ - if((line_end = strchr(message, '\n')) == NULL) { - commit->message_short = git__strdup(message); - return; - } - - message_len = line_end - message; - - commit->message_short = git__malloc(message_len + 1); - memcpy(commit->message_short, message, message_len); - commit->message_short[message_len] = 0; -} - diff --git a/src/commit.h b/src/commit.h index aaf349ca6..3d15c5044 100644 --- a/src/commit.h +++ b/src/commit.h @@ -22,8 +22,6 @@ struct git_commit { }; void git_commit__free(git_commit *c); -int git_commit__parse(git_commit *commit); - -int git_commit__writeback(git_commit *commit, git_odb_source *src); +int git_commit__parse(git_commit *commit, git_odb_object *obj); #endif diff --git a/src/delta-apply.h b/src/delta-apply.h index 642442de0..36c5cc60d 100644 --- a/src/delta-apply.h +++ b/src/delta-apply.h @@ -1,6 +1,8 @@ #ifndef INCLUDE_delta_apply_h__ #define INCLUDE_delta_apply_h__ +#include "odb.h" + /** * Apply a git binary delta to recover the original content. * diff --git a/src/errors.c b/src/errors.c index 880163f78..f6b964837 100644 --- a/src/errors.c +++ b/src/errors.c @@ -27,7 +27,8 @@ static struct { {GIT_EPACKEDREFSCORRUPTED, "The pack-refs file is either corrupted of its format is not currently supported"}, {GIT_EINVALIDPATH, "The path is invalid" }, {GIT_EREVWALKOVER, "The revision walker is empty; there are no more commits left to iterate"}, - {GIT_EINVALIDREFSTATE, "The state of the reference is not valid"} + {GIT_EINVALIDREFSTATE, "The state of the reference is not valid"}, + {GIT_ENOTIMPLEMENTED, "This feature has not been implemented yet"} }; const char *git_strerror(int num) diff --git a/src/filebuf.c b/src/filebuf.c index 4fc4f1486..607ad618d 100644 --- a/src/filebuf.c +++ b/src/filebuf.c @@ -77,43 +77,81 @@ void git_filebuf_cleanup(git_filebuf *file) if (file->fd >= 0) gitfo_close(file->fd); - if (gitfo_exists(file->path_lock) == GIT_SUCCESS) + if (file->path_lock && gitfo_exists(file->path_lock) == GIT_SUCCESS) gitfo_unlink(file->path_lock); if (file->digest) git_hash_free_ctx(file->digest); free(file->buffer); + free(file->z_buf); -#ifdef GIT_FILEBUF_THREADS - free(file->buffer_back); -#endif + deflateEnd(&file->zs); free(file->path_original); free(file->path_lock); } -static int flush_buffer(git_filebuf *file) +GIT_INLINE(int) flush_buffer(git_filebuf *file) { - int result = GIT_SUCCESS; + int result = file->write(file, file->buffer, file->buf_pos); + file->buf_pos = 0; + return result; +} - if (file->buf_pos > 0) { - result = gitfo_write(file->fd, file->buffer, file->buf_pos); - if (file->digest) - git_hash_update(file->digest, file->buffer, file->buf_pos); +static int write_normal(git_filebuf *file, const void *source, size_t len) +{ + int result = 0; - file->buf_pos = 0; + if (len > 0) { + result = gitfo_write(file->fd, (void *)source, len); + if (file->digest) + git_hash_update(file->digest, source, len); } return result; } +static int write_deflate(git_filebuf *file, const void *source, size_t len) +{ + int result = Z_OK; + z_stream *zs = &file->zs; + + if (len > 0 || file->flush_mode == Z_FINISH) { + zs->next_in = (void *)source; + zs->avail_in = len; + + do { + int have; + + zs->next_out = file->z_buf; + zs->avail_out = file->buf_size; + + result = deflate(zs, file->flush_mode); + assert(result != Z_STREAM_ERROR); + + have = file->buf_size - zs->avail_out; + + if (gitfo_write(file->fd, file->z_buf, have) < GIT_SUCCESS) + return GIT_EOSERR; + + } while (zs->avail_out == 0); + + assert(zs->avail_in == 0); + + if (file->digest) + git_hash_update(file->digest, source, len); + } + + return GIT_SUCCESS; +} + int git_filebuf_open(git_filebuf *file, const char *path, int flags) { int error; size_t path_len; - if (file == NULL || path == NULL) + if (file == NULL) return GIT_ERROR; memset(file, 0x0, sizeof(git_filebuf)); @@ -122,46 +160,93 @@ int git_filebuf_open(git_filebuf *file, const char *path, int flags) file->buf_pos = 0; file->fd = -1; - path_len = strlen(path); - - file->path_original = git__strdup(path); - if (file->path_original == NULL) { + /* Allocate the main cache buffer */ + file->buffer = git__malloc(file->buf_size); + if (file->buffer == NULL){ error = GIT_ENOMEM; goto cleanup; } - file->path_lock = git__malloc(path_len + GIT_FILELOCK_EXTLENGTH); - if (file->path_lock == NULL) { - error = GIT_ENOMEM; - goto cleanup; + /* If we are hashing on-write, allocate a new hash context */ + if (flags & GIT_FILEBUF_HASH_CONTENTS) { + if ((file->digest = git_hash_new_ctx()) == NULL) { + error = GIT_ENOMEM; + goto cleanup; + } } - memcpy(file->path_lock, file->path_original, path_len); - memcpy(file->path_lock + path_len, GIT_FILELOCK_EXTENSION, GIT_FILELOCK_EXTLENGTH); + /* If we are deflating on-write, */ + if (flags & GIT_FILEBUF_DEFLATE_CONTENTS) { - file->buffer = git__malloc(file->buf_size); - if (file->buffer == NULL){ - error = GIT_ENOMEM; - goto cleanup; - } + /* Initialize the ZLib stream */ + if (deflateInit(&file->zs, Z_DEFAULT_COMPRESSION) != Z_OK) { + error = GIT_EZLIB; + goto cleanup; + } -#ifdef GIT_FILEBUF_THREADS - file->buffer_back = git__malloc(file->buf_size); - if (file->buffer_back == NULL){ - error = GIT_ENOMEM; - goto cleanup; + /* Allocate the Zlib cache buffer */ + file->z_buf = git__malloc(file->buf_size); + if (file->z_buf == NULL){ + error = GIT_ENOMEM; + goto cleanup; + } + + /* Never flush */ + file->flush_mode = Z_NO_FLUSH; + file->write = &write_deflate; + } else { + file->write = &write_normal; } -#endif - if (flags & GIT_FILEBUF_HASH_CONTENTS) { - if ((file->digest = git_hash_new_ctx()) == NULL) { + /* If we are writing to a temp file */ + if (flags & GIT_FILEBUF_TEMPORARY) { + char tmp_path[GIT_PATH_MAX]; + + /* Open the file as temporary for locking */ + file->fd = gitfo_creat_tmp(tmp_path, "_filebuf_"); + if (file->fd < 0) { + error = GIT_EOSERR; + goto cleanup; + } + + /* No original path */ + file->path_original = NULL; + file->path_lock = git__strdup(tmp_path); + + if (file->path_lock == NULL) { error = GIT_ENOMEM; goto cleanup; } - } + } else { + /* If the file is not temporary, make sure we have a path */ + if (path == NULL) { + error = GIT_ERROR; + goto cleanup; + } - if ((error = lock_file(file, flags)) < GIT_SUCCESS) - goto cleanup; + path_len = strlen(path); + + /* Save the original path of the file */ + file->path_original = git__strdup(path); + if (file->path_original == NULL) { + error = GIT_ENOMEM; + goto cleanup; + } + + /* create the locking path by appending ".lock" to the original */ + file->path_lock = git__malloc(path_len + GIT_FILELOCK_EXTLENGTH); + if (file->path_lock == NULL) { + error = GIT_ENOMEM; + goto cleanup; + } + + memcpy(file->path_lock, file->path_original, path_len); + memcpy(file->path_lock + path_len, GIT_FILELOCK_EXTENSION, GIT_FILELOCK_EXTLENGTH); + + /* open the file for locking */ + if ((error = lock_file(file, flags)) < GIT_SUCCESS) + goto cleanup; + } return GIT_SUCCESS; @@ -187,10 +272,25 @@ int git_filebuf_hash(git_oid *oid, git_filebuf *file) return GIT_SUCCESS; } +int git_filebuf_commit_at(git_filebuf *file, const char *path) +{ + free(file->path_original); + file->path_original = git__strdup(path); + if (file->path_original == NULL) + return GIT_ENOMEM; + + return git_filebuf_commit(file); +} + int git_filebuf_commit(git_filebuf *file) { int error; + /* tmp file cannot be commited */ + if (file->path_original == NULL) + return GIT_EOSERR; + + file->flush_mode = Z_FINISH; if ((error = flush_buffer(file)) < GIT_SUCCESS) goto cleanup; @@ -204,16 +304,16 @@ cleanup: return error; } -GIT_INLINE(void) add_to_cache(git_filebuf *file, void *buf, size_t len) +GIT_INLINE(void) add_to_cache(git_filebuf *file, const void *buf, size_t len) { memcpy(file->buffer + file->buf_pos, buf, len); file->buf_pos += len; } -int git_filebuf_write(git_filebuf *file, void *buff, size_t len) +int git_filebuf_write(git_filebuf *file, const void *buff, size_t len) { int error; - unsigned char *buf = buff; + const unsigned char *buf = buff; for (;;) { size_t space_left = file->buf_size - file->buf_pos; @@ -237,9 +337,9 @@ int git_filebuf_write(git_filebuf *file, void *buff, size_t len) /* write too-large chunks immediately */ if (len > file->buf_size) { - error = gitfo_write(file->fd, buf, len); - if (file->digest) - git_hash_update(file->digest, buf, len); + error = file->write(file, buf, len); + if (error < GIT_SUCCESS) + return error; } } } diff --git a/src/filebuf.h b/src/filebuf.h index 9db615fbd..37cb36784 100644 --- a/src/filebuf.h +++ b/src/filebuf.h @@ -3,14 +3,17 @@ #include "fileops.h" #include "hash.h" +#include "git2/zlib.h" #ifdef GIT_THREADS # define GIT_FILEBUF_THREADS #endif -#define GIT_FILEBUF_HASH_CONTENTS 0x1 -#define GIT_FILEBUF_APPEND 0x2 -#define GIT_FILEBUF_FORCE 0x4 +#define GIT_FILEBUF_HASH_CONTENTS (1 << 0) +#define GIT_FILEBUF_APPEND (1 << 2) +#define GIT_FILEBUF_FORCE (1 << 3) +#define GIT_FILEBUF_TEMPORARY (1 << 4) +#define GIT_FILEBUF_DEFLATE_CONTENTS (1 << 5) #define GIT_FILELOCK_EXTENSION ".lock\0" #define GIT_FILELOCK_EXTLENGTH 6 @@ -19,12 +22,16 @@ struct git_filebuf { char *path_original; char *path_lock; + int (*write)(struct git_filebuf *file, + const void *source, size_t len); + git_hash_ctx *digest; unsigned char *buffer; -#ifdef GIT_FILEBUF_THREADS - unsigned char *buffer_back; -#endif + unsigned char *z_buf; + + z_stream zs; + int flush_mode; size_t buf_size, buf_pos; git_file fd; @@ -32,12 +39,13 @@ struct git_filebuf { typedef struct git_filebuf git_filebuf; -int git_filebuf_write(git_filebuf *lock, void *buff, size_t len); +int git_filebuf_write(git_filebuf *lock, const void *buff, size_t len); int git_filebuf_reserve(git_filebuf *file, void **buff, size_t len); int git_filebuf_printf(git_filebuf *file, const char *format, ...); int git_filebuf_open(git_filebuf *lock, const char *path, int flags); int git_filebuf_commit(git_filebuf *lock); +int git_filebuf_commit_at(git_filebuf *lock, const char *path); void git_filebuf_cleanup(git_filebuf *lock); int git_filebuf_hash(git_oid *oid, git_filebuf *file); diff --git a/src/fileops.c b/src/fileops.c index 76e689e8a..237a16a9f 100644 --- a/src/fileops.c +++ b/src/fileops.c @@ -2,13 +2,13 @@ #include "fileops.h" #include -static int force_path(const char *to) +int gitfo_mkdir_2file(const char *file_path) { const int mode = 0755; /* or 0777 ? */ int error = GIT_SUCCESS; char target_folder_path[GIT_PATH_MAX]; - error = git__dirname_r(target_folder_path, sizeof(target_folder_path), to); + error = git__dirname_r(target_folder_path, sizeof(target_folder_path), file_path); if (error < GIT_SUCCESS) return error; @@ -25,6 +25,87 @@ static int force_path(const char *to) return GIT_SUCCESS; } +static int creat_tempfile(char *path_out, const char *tmp_dir, const char *filename) +{ + int fd; + + git__joinpath(path_out, tmp_dir, filename); + strcat(path_out, "_git2_XXXXXX"); + +#ifdef GIT_WIN32 + /* FIXME: there may be race conditions when multi-threading + * with the library */ + if (_mktemp_s(path_out, GIT_PATH_MAX) != 0) + return GIT_EOSERR; + + fd = gitfo_creat(path_out, 0744); +#else + fd = mkstemp(path_out); +#endif + + return fd >= 0 ? fd : GIT_EOSERR; +} + +static const char *find_tmpdir(void) +{ + static int tmpdir_not_found = 0; + static char temp_dir[GIT_PATH_MAX]; + static const char *env_vars[] = { + "TEMP", "TMP", "TMPDIR" + }; + + unsigned int i, j; + char test_file[GIT_PATH_MAX]; + + if (tmpdir_not_found) + return NULL; + + if (temp_dir[0] != '\0') + return temp_dir; + + for (i = 0; i < ARRAY_SIZE(env_vars); ++i) { + char *env_path; + + env_path = getenv(env_vars[i]); + if (env_path == NULL) + continue; + + strcpy(temp_dir, env_path); + + /* Fix backslashes because Windows environment vars + * are probably fucked up */ + for (j = 0; j < strlen(temp_dir); ++j) + if (temp_dir[j] == '\\') + temp_dir[j] = '/'; + + if (creat_tempfile(test_file, temp_dir, "writetest") >= 0) { + gitfo_unlink(test_file); + return temp_dir; + } + } + + /* last resort: current folder. */ + strcpy(temp_dir, "./"); + if (creat_tempfile(test_file, temp_dir, "writetest") >= 0) { + gitfo_unlink(test_file); + return temp_dir; + } + + tmpdir_not_found = 1; + return NULL; +} + +int gitfo_creat_tmp(char *path_out, const char *filename) +{ + const char *tmp_dir; + + tmp_dir = find_tmpdir(); + if (tmp_dir == NULL) + return GIT_EOSERR; + + return creat_tempfile(path_out, tmp_dir, filename); +} + int gitfo_open(const char *path, int flags) { int fd = open(path, flags | O_BINARY); @@ -39,7 +120,7 @@ int gitfo_creat(const char *path, int mode) int gitfo_creat_force(const char *path, int mode) { - if (force_path(path) < GIT_SUCCESS) + if (gitfo_mkdir_2file(path) < GIT_SUCCESS) return GIT_EOSERR; return gitfo_creat(path, mode); @@ -117,6 +198,7 @@ int gitfo_isdir(const char *path) int gitfo_exists(const char *path) { + assert(path); return access(path, F_OK); } @@ -198,7 +280,7 @@ int gitfo_mv(const char *from, const char *to) int gitfo_mv_force(const char *from, const char *to) { - if (force_path(to) < GIT_SUCCESS) + if (gitfo_mkdir_2file(to) < GIT_SUCCESS) return GIT_EOSERR; return gitfo_mv(from, to); diff --git a/src/fileops.h b/src/fileops.h index 5aa302b54..bc636fc38 100644 --- a/src/fileops.h +++ b/src/fileops.h @@ -58,8 +58,10 @@ extern int gitfo_exists(const char *path); extern int gitfo_open(const char *path, int flags); extern int gitfo_creat(const char *path, int mode); extern int gitfo_creat_force(const char *path, int mode); +extern int gitfo_creat_tmp(char *path_out, const char *filename); extern int gitfo_isdir(const char *path); extern int gitfo_mkdir_recurs(const char *path, int mode); +extern int gitfo_mkdir_2file(const char *path); #define gitfo_close(fd) close(fd) extern int gitfo_read(git_file fd, void *buf, size_t cnt); diff --git a/src/index.c b/src/index.c index 95e56b7d5..6a355e11b 100644 --- a/src/index.c +++ b/src/index.c @@ -324,7 +324,7 @@ int git_index_add(git_index *index, const char *rel_path, int stage) entry.file_size = st.st_size; /* write the blob to disk and get the oid */ - if ((error = git_blob_writefile(&entry.oid, index->repository, full_path)) < GIT_SUCCESS) + if ((error = git_blob_create_fromfile(&entry.oid, index->repository, rel_path)) < GIT_SUCCESS) return error; entry.flags |= (stage << GIT_IDXENTRY_STAGESHIFT); diff --git a/src/object.c b/src/object.c index c432c6da0..0572663eb 100644 --- a/src/object.c +++ b/src/object.c @@ -66,153 +66,6 @@ static struct { { "REF_DELTA", 0, 0 } }; -/* - * Object source methods - * - * Abstract buffer methods that allow the writeback system - * to prepare the contents of any git file in-memory before - * writing them to disk. - */ -static int source_resize(git_odb_source *src) -{ - size_t write_offset, new_size; - void *new_data; - - write_offset = (size_t)((char *)src->write_ptr - (char *)src->raw.data); - - new_size = src->raw.len * 2; - if ((new_data = git__malloc(new_size)) == NULL) - return GIT_ENOMEM; - - memcpy(new_data, src->raw.data, src->written_bytes); - free(src->raw.data); - - src->raw.data = new_data; - src->raw.len = new_size; - src->write_ptr = (char *)new_data + write_offset; - - return GIT_SUCCESS; -} - -int git__source_printf(git_odb_source *source, const char *format, ...) -{ - va_list arglist; - int len; - - assert(source->open && source->write_ptr); - - va_start(arglist, format); - - len = vsnprintf(source->write_ptr, source->raw.len - source->written_bytes, format, arglist); - - while (source->written_bytes + len >= source->raw.len) { - if (source_resize(source) < GIT_SUCCESS) - return GIT_ENOMEM; - - len = vsnprintf(source->write_ptr, source->raw.len - source->written_bytes, format, arglist); - } - - source->write_ptr = (char *)source->write_ptr + len; - source->written_bytes += len; - - return GIT_SUCCESS; -} - -int git__source_write(git_odb_source *source, const void *bytes, size_t len) -{ - assert(source); - - assert(source->open && source->write_ptr); - - while (source->written_bytes + len >= source->raw.len) { - if (source_resize(source) < GIT_SUCCESS) - return GIT_ENOMEM; - } - - memcpy(source->write_ptr, bytes, len); - source->write_ptr = (char *)source->write_ptr + len; - source->written_bytes += len; - - return GIT_SUCCESS; -} - -static void prepare_write(git_object *object) -{ - if (object->source.write_ptr != NULL || object->source.open) - git_object__source_close(object); - - /* TODO: proper size calculation */ - object->source.raw.data = git__malloc(OBJECT_BASE_SIZE); - object->source.raw.len = OBJECT_BASE_SIZE; - - object->source.write_ptr = object->source.raw.data; - object->source.written_bytes = 0; - - object->source.open = 1; -} - -static int write_back(git_object *object) -{ - int error; - git_oid new_id; - - assert(object); - - assert(object->source.open); - assert(object->modified); - - object->source.raw.len = object->source.written_bytes; - - if ((error = git_odb_write(&new_id, object->repo->db, &object->source.raw)) < GIT_SUCCESS) - return error; - - if (object->in_memory) { - int idx = git_vector_search(&object->repo->memory_objects, object); - git_vector_remove(&object->repo->memory_objects, idx); - } else { - git_hashtable_remove(object->repo->objects, &object->id); - } - - git_oid_cpy(&object->id, &new_id); - git_hashtable_insert(object->repo->objects, &object->id, object); - - object->source.write_ptr = NULL; - object->source.written_bytes = 0; - - object->modified = 0; - object->in_memory = 0; - - git_object__source_close(object); - return GIT_SUCCESS; -} - -int git_object__source_open(git_object *object) -{ - int error; - - assert(object && !object->in_memory); - - if (object->source.open) - git_object__source_close(object); - - error = git_odb_read(&object->source.raw, object->repo->db, &object->id); - if (error < GIT_SUCCESS) - return error; - - object->source.open = 1; - return GIT_SUCCESS; -} - -void git_object__source_close(git_object *object) -{ - assert(object); - - if (object->source.open) { - git_rawobj_close(&object->source.raw); - object->source.open = 0; - } -} - static int create_object(git_object **object_out, git_otype type) { git_object *object = NULL; @@ -225,43 +78,19 @@ static int create_object(git_object **object_out, git_otype type) case GIT_OBJ_COMMIT: case GIT_OBJ_TAG: case GIT_OBJ_BLOB: + case GIT_OBJ_TREE: object = git__malloc(git_object__size(type)); if (object == NULL) return GIT_ENOMEM; memset(object, 0x0, git_object__size(type)); break; - - case GIT_OBJ_TREE: - object = (git_object *)git_tree__new(); - if (object == NULL) - return GIT_ENOMEM; - break; default: return GIT_EINVALIDTYPE; } - *object_out = object; - return GIT_SUCCESS; -} - -int git_object_new(git_object **object_out, git_repository *repo, git_otype type) -{ - git_object *object = NULL; - int error; - - assert(object_out && repo); - - if ((error = create_object(&object, type)) < GIT_SUCCESS) - return error; - - object->repo = repo; - object->in_memory = 1; - object->modified = 1; - - object->source.raw.type = type; + object->type = type; - object->lru = ++repo->lru_counter; *object_out = object; return GIT_SUCCESS; } @@ -269,126 +98,77 @@ int git_object_new(git_object **object_out, git_repository *repo, git_otype type int git_object_lookup(git_object **object_out, git_repository *repo, const git_oid *id, git_otype type) { git_object *object = NULL; - git_rawobj obj_file; + git_odb_object *odb_obj; int error = GIT_SUCCESS; assert(repo && object_out && id); - object = git_hashtable_lookup(repo->objects, id); + object = git_cache_get(&repo->objects, id); if (object != NULL) { - if (type != GIT_OBJ_ANY && type != object->source.raw.type) + if (type != GIT_OBJ_ANY && type != object->type) return GIT_EINVALIDTYPE; *object_out = object; - object->lru = ++repo->lru_counter; - object->can_free = 0; return GIT_SUCCESS; } - error = git_odb_read(&obj_file, repo->db, id); + error = git_odb_read(&odb_obj, repo->db, id); if (error < GIT_SUCCESS) return error; - if (type != GIT_OBJ_ANY && type != obj_file.type) { - git_rawobj_close(&obj_file); + if (type != GIT_OBJ_ANY && type != odb_obj->raw.type) { + git_odb_object_close(odb_obj); return GIT_EINVALIDTYPE; } - type = obj_file.type; + type = odb_obj->raw.type; if ((error = create_object(&object, type)) < GIT_SUCCESS) return error; /* Initialize parent object */ - git_oid_cpy(&object->id, id); + git_oid_cpy(&object->cached.oid, id); object->repo = repo; - memcpy(&object->source.raw, &obj_file, sizeof(git_rawobj)); - object->source.open = 1; switch (type) { case GIT_OBJ_COMMIT: - error = git_commit__parse((git_commit *)object); + error = git_commit__parse((git_commit *)object, odb_obj); break; case GIT_OBJ_TREE: - error = git_tree__parse((git_tree *)object); + error = git_tree__parse((git_tree *)object, odb_obj); break; case GIT_OBJ_TAG: - error = git_tag__parse((git_tag *)object); + error = git_tag__parse((git_tag *)object, odb_obj); break; case GIT_OBJ_BLOB: - error = git_blob__parse((git_blob *)object); + error = git_blob__parse((git_blob *)object, odb_obj); break; default: break; } + git_odb_object_close(odb_obj); + if (error < GIT_SUCCESS) { git_object__free(object); return error; } - git_object__source_close(object); - git_hashtable_insert(repo->objects, &object->id, object); - - object->lru = ++repo->lru_counter; - *object_out = object; + *object_out = git_cache_try_store(&repo->objects, object); return GIT_SUCCESS; } -int git_object_write(git_object *object) +void git_object__free(void *_obj) { - int error; - git_odb_source *source; + git_object *object = (git_object *)_obj; assert(object); - if (object->modified == 0) - return GIT_SUCCESS; - - prepare_write(object); - source = &object->source; - - switch (source->raw.type) { - case GIT_OBJ_COMMIT: - error = git_commit__writeback((git_commit *)object, source); - break; - - case GIT_OBJ_TREE: - error = git_tree__writeback((git_tree *)object, source); - break; - - case GIT_OBJ_TAG: - error = git_tag__writeback((git_tag *)object, source); - break; - - case GIT_OBJ_BLOB: - error = git_blob__writeback((git_blob *)object, source); - break; - - default: - error = GIT_ERROR; - break; - } - - if (error < GIT_SUCCESS) { - git_object__source_close(object); - return error; - } - - return write_back(object); -} - -void git_object__free(git_object *object) -{ - assert(object); - - git_object__source_close(object); - - switch (object->source.raw.type) { + switch (object->type) { case GIT_OBJ_COMMIT: git_commit__free((git_commit *)object); break; @@ -416,29 +196,19 @@ void git_object_close(git_object *object) if (object == NULL) return; - if (object->in_memory) { - int idx = git_vector_search(&object->repo->memory_objects, object); - git_vector_remove(&object->repo->memory_objects, idx); - git_object__free(object); - } else { - object->can_free = 1; - } + git_cached_obj_decref((git_cached_obj *)object, git_object__free); } const git_oid *git_object_id(const git_object *obj) { assert(obj); - - if (obj->in_memory) - return NULL; - - return &obj->id; + return &obj->cached.oid; } git_otype git_object_type(const git_object *obj) { assert(obj); - return obj->source.raw.type; + return obj->type; } git_repository *git_object_owner(const git_object *obj) diff --git a/src/odb.c b/src/odb.c index 2013ac24c..9aeaa8a23 100644 --- a/src/odb.c +++ b/src/odb.c @@ -87,56 +87,49 @@ int git_odb__hash_obj(git_oid *id, char *hdr, size_t n, int *len, git_rawobj *ob return GIT_SUCCESS; } -void git_rawobj_close(git_rawobj *obj) -{ - free(obj->data); - obj->data = NULL; -} -int git_rawobj_hash(git_oid *id, git_rawobj *obj) +static git_odb_object *new_odb_object(const git_oid *oid, git_rawobj *source) { - char hdr[64]; - int hdrlen; + git_odb_object *object = git__malloc(sizeof(git_odb_object)); + memset(object, 0x0, sizeof(git_odb_object)); - assert(id && obj); + git_oid_cpy(&object->cached.oid, oid); + memcpy(&object->raw, source, sizeof(git_rawobj)); - return git_odb__hash_obj(id, hdr, sizeof(hdr), &hdrlen, obj); + return object; } -int git_odb__inflate_buffer(void *in, size_t inlen, void *out, size_t outlen) +static void free_odb_object(void *o) { - z_stream zs; - int status = Z_OK; - - memset(&zs, 0x0, sizeof(zs)); - - zs.next_out = out; - zs.avail_out = outlen; - - zs.next_in = in; - zs.avail_in = inlen; + git_odb_object *object = (git_odb_object *)o; - if (inflateInit(&zs) < Z_OK) - return GIT_ERROR; + if (object != NULL) { + free(object->raw.data); + free(object); + } +} - while (status == Z_OK) - status = inflate(&zs, Z_FINISH); +void git_odb_object_close(git_odb_object *object) +{ + git_cached_obj_decref((git_cached_obj *)object, &free_odb_object); +} - inflateEnd(&zs); +int git_odb_hash(git_oid *id, const void *data, size_t len, git_otype type) +{ + char hdr[64]; + int hdrlen; + git_rawobj raw; - if ((status != Z_STREAM_END) /*|| (zs.avail_in != 0) */) - return GIT_ERROR; + assert(id); - if (zs.total_out != outlen) - return GIT_ERROR; + raw.data = (void *)data; + raw.len = len; + raw.type = type; - return GIT_SUCCESS; + return git_odb__hash_obj(id, hdr, sizeof(hdr), &hdrlen, &raw); } - - - /*********************************************************** * * OBJECT DATABASE PUBLIC API @@ -162,6 +155,8 @@ int git_odb_new(git_odb **out) if (!db) return GIT_ENOMEM; + git_cache_init(&db->cache, GIT_DEFAULT_CACHE_SIZE, &free_odb_object); + if (git_vector_init(&db->backends, 4, backend_sort_cmp) < 0) { free(db); return GIT_ENOMEM; @@ -306,16 +301,23 @@ void git_odb_close(git_odb *db) } git_vector_free(&db->backends); + git_cache_free(&db->cache); free(db); } int git_odb_exists(git_odb *db, const git_oid *id) { + git_odb_object *object; unsigned int i; int found = 0; assert(db && id); + if ((object = git_cache_get(&db->cache, id)) != NULL) { + git_odb_object_close(object); + return 1; + } + for (i = 0; i < db->backends.length && !found; ++i) { backend_internal *internal = git_vector_get(&db->backends, i); git_odb_backend *b = internal->backend; @@ -327,19 +329,27 @@ int git_odb_exists(git_odb *db, const git_oid *id) return found; } -int git_odb_read_header(git_rawobj *out, git_odb *db, const git_oid *id) +int git_odb_read_header(size_t *len_p, git_otype *type_p, git_odb *db, const git_oid *id) { unsigned int i; int error = GIT_ENOTFOUND; + git_odb_object *object; - assert(out && db && id); + assert(db && id); + + if ((object = git_cache_get(&db->cache, id)) != NULL) { + *len_p = object->raw.len; + *type_p = object->raw.type; + git_odb_object_close(object); + return GIT_SUCCESS; + } for (i = 0; i < db->backends.length && error < 0; ++i) { backend_internal *internal = git_vector_get(&db->backends, i); git_odb_backend *b = internal->backend; if (b->read_header != NULL) - error = b->read_header(out, b, id); + error = b->read_header(len_p, type_p, b, id); } /* @@ -347,37 +357,50 @@ int git_odb_read_header(git_rawobj *out, git_odb *db, const git_oid *id) * try reading the whole object and freeing the contents */ if (error < 0) { - error = git_odb_read(out, db, id); - git_rawobj_close(out); + if ((error = git_odb_read(&object, db, id)) < GIT_SUCCESS) + return error; + + *len_p = object->raw.len; + *type_p = object->raw.len; + git_odb_object_close(object); } - return error; + return GIT_SUCCESS; } -int git_odb_read(git_rawobj *out, git_odb *db, const git_oid *id) +int git_odb_read(git_odb_object **out, git_odb *db, const git_oid *id) { unsigned int i; int error = GIT_ENOTFOUND; + git_rawobj raw; assert(out && db && id); + *out = git_cache_get(&db->cache, id); + if (*out != NULL) + return GIT_SUCCESS; + for (i = 0; i < db->backends.length && error < 0; ++i) { backend_internal *internal = git_vector_get(&db->backends, i); git_odb_backend *b = internal->backend; if (b->read != NULL) - error = b->read(out, b, id); + error = b->read(&raw.data, &raw.len, &raw.type, b, id); + } + + if (error == GIT_SUCCESS) { + *out = git_cache_try_store(&db->cache, new_odb_object(id, &raw)); } return error; } -int git_odb_write(git_oid *id, git_odb *db, git_rawobj *obj) +int git_odb_open_wstream(git_odb_stream **stream, git_odb *db, size_t size, git_otype type) { unsigned int i; int error = GIT_ERROR; - assert(obj && db && id); + assert(stream && db); for (i = 0; i < db->backends.length && error < 0; ++i) { backend_internal *internal = git_vector_get(&db->backends, i); @@ -387,8 +410,26 @@ int git_odb_write(git_oid *id, git_odb *db, git_rawobj *obj) if (internal->is_alternate) continue; - if (b->write != NULL) - error = b->write(id, b, obj); + if (b->writestream != NULL) + error = b->writestream(stream, b, size, type); + } + + return error; +} + +int git_odb_open_rstream(git_odb_stream **stream, git_odb *db, const git_oid *oid) +{ + unsigned int i; + int error = GIT_ERROR; + + assert(stream && db); + + for (i = 0; i < db->backends.length && error < 0; ++i) { + backend_internal *internal = git_vector_get(&db->backends, i); + git_odb_backend *b = internal->backend; + + if (b->readstream != NULL) + error = b->readstream(stream, b, oid); } return error; diff --git a/src/odb.h b/src/odb.h index c3d0a17ab..f3685834e 100644 --- a/src/odb.h +++ b/src/odb.h @@ -3,15 +3,31 @@ #include "git2/odb.h" #include "git2/oid.h" +#include "git2/types.h" #include "vector.h" +#include "cache.h" +/* DO NOT EXPORT */ +typedef struct { + void *data; /**< Raw, decompressed object data. */ + size_t len; /**< Total number of bytes in data. */ + git_otype type; /**< Type of this object. */ +} git_rawobj; + +/* EXPORT */ +struct git_odb_object { + git_cached_obj cached; + git_rawobj raw; +}; + +/* EXPORT */ struct git_odb { void *_internal; git_vector backends; + git_cache cache; }; int git_odb__hash_obj(git_oid *id, char *hdr, size_t n, int *len, git_rawobj *obj); -int git_odb__inflate_buffer(void *in, size_t inlen, void *out, size_t outlen); #endif diff --git a/src/odb_loose.c b/src/odb_loose.c index 4e2d9a639..4ab1128f3 100644 --- a/src/odb_loose.c +++ b/src/odb_loose.c @@ -30,14 +30,22 @@ #include "hash.h" #include "odb.h" #include "delta-apply.h" +#include "filebuf.h" #include "git2/odb_backend.h" +#include "git2/types.h" typedef struct { /* object header data */ git_otype type; /* object type */ size_t size; /* object size */ } obj_hdr; +typedef struct { + git_odb_stream stream; + git_filebuf fbuf; + int finished; +} loose_writestream; + typedef struct loose_backend { git_odb_backend parent; @@ -53,38 +61,6 @@ typedef struct loose_backend { * ***********************************************************/ -static int make_temp_file(git_file *fd, char *tmp, size_t n, char *file) -{ - char *template = "/tmp_obj_XXXXXX"; - size_t tmplen = strlen(template); - int dirlen; - - if ((dirlen = git__dirname_r(tmp, n, file)) < 0) - return GIT_ERROR; - - if ((dirlen + tmplen) >= n) - return GIT_ERROR; - - strcpy(tmp + dirlen, (dirlen) ? template : template + 1); - - *fd = gitfo_mkstemp(tmp); - if (*fd < 0 && dirlen) { - /* create directory if it doesn't exist */ - tmp[dirlen] = '\0'; - if ((gitfo_exists(tmp) < 0) && gitfo_mkdir(tmp, 0755)) - return GIT_ERROR; - /* try again */ - strcpy(tmp + dirlen, template); - *fd = gitfo_mkstemp(tmp); - } - if (*fd < 0) - return GIT_ERROR; - - return GIT_SUCCESS; -} - - - static size_t object_file_name(char *name, size_t n, char *dir, const git_oid *id) { size_t len = strlen(dir); @@ -236,72 +212,44 @@ static int finish_inflate(z_stream *s) return GIT_SUCCESS; } -static int deflate_buf(z_stream *s, void *in, size_t len, int flush) +static int is_zlib_compressed_data(unsigned char *data) { - int status = Z_OK; + unsigned int w; - set_stream_input(s, in, len); - while (status == Z_OK) { - status = deflate(s, flush); - if (s->avail_in == 0) - break; - } - return status; + w = ((unsigned int)(data[0]) << 8) + data[1]; + return data[0] == 0x78 && !(w % 31); } -static int deflate_obj(gitfo_buf *buf, char *hdr, int hdrlen, git_rawobj *obj, int level) +static int inflate_buffer(void *in, size_t inlen, void *out, size_t outlen) { z_stream zs; - int status; - size_t size; - - assert(buf && !buf->data && hdr && obj); - assert(level == Z_DEFAULT_COMPRESSION || (level >= 0 && level <= 9)); + int status = Z_OK; - buf->data = NULL; - buf->len = 0; - init_stream(&zs, NULL, 0); + memset(&zs, 0x0, sizeof(zs)); - if (deflateInit(&zs, level) < Z_OK) - return GIT_ERROR; + zs.next_out = out; + zs.avail_out = outlen; - size = deflateBound(&zs, hdrlen + obj->len); + zs.next_in = in; + zs.avail_in = inlen; - if ((buf->data = git__malloc(size)) == NULL) { - deflateEnd(&zs); + if (inflateInit(&zs) < Z_OK) return GIT_ERROR; - } - set_stream_output(&zs, buf->data, size); - - /* compress the header */ - status = deflate_buf(&zs, hdr, hdrlen, Z_NO_FLUSH); + while (status == Z_OK) + status = inflate(&zs, Z_FINISH); - /* if header compressed OK, compress the object */ - if (status == Z_OK) - status = deflate_buf(&zs, obj->data, obj->len, Z_FINISH); + inflateEnd(&zs); - if (status != Z_STREAM_END) { - deflateEnd(&zs); - free(buf->data); - buf->data = NULL; + if ((status != Z_STREAM_END) /*|| (zs.avail_in != 0) */) return GIT_ERROR; - } - buf->len = zs.total_out; - deflateEnd(&zs); + if (zs.total_out != outlen) + return GIT_ERROR; return GIT_SUCCESS; } -static int is_zlib_compressed_data(unsigned char *data) -{ - unsigned int w; - - w = ((unsigned int)(data[0]) << 8) + data[1]; - return data[0] == 0x78 && !(w % 31); -} - static void *inflate_tail(z_stream *s, void *hb, size_t used, obj_hdr *hdr) { unsigned char *buf, *head = hb; @@ -371,7 +319,7 @@ static int inflate_packlike_loose_disk_obj(git_rawobj *out, gitfo_buf *obj) in = ((unsigned char *)obj->data) + used; len = obj->len - used; - if (git_odb__inflate_buffer(in, len, buf, hdr.size)) { + if (inflate_buffer(in, len, buf, hdr.size)) { free(buf); return GIT_ERROR; } @@ -505,37 +453,6 @@ cleanup: return error; } -static int write_obj(gitfo_buf *buf, git_oid *id, loose_backend *backend) -{ - char file[GIT_PATH_MAX]; - char temp[GIT_PATH_MAX]; - git_file fd; - - if (object_file_name(file, sizeof(file), backend->objects_dir, id)) - return GIT_EOSERR; - - if (make_temp_file(&fd, temp, sizeof(temp), file) < 0) - return GIT_EOSERR; - - if (gitfo_write(fd, buf->data, buf->len) < 0) { - gitfo_close(fd); - gitfo_unlink(temp); - return GIT_EOSERR; - } - - if (backend->fsync_object_files) - gitfo_fsync(fd); - gitfo_close(fd); - gitfo_chmod(temp, 0444); - - if (gitfo_mv(temp, file) < 0) { - gitfo_unlink(temp); - return GIT_EOSERR; - } - - return GIT_SUCCESS; -} - static int locate_object(char *object_location, loose_backend *backend, const git_oid *oid) { object_file_name(object_location, GIT_PATH_MAX, backend->objects_dir, oid); @@ -558,29 +475,44 @@ static int locate_object(char *object_location, loose_backend *backend, const gi * ***********************************************************/ -int loose_backend__read_header(git_rawobj *obj, git_odb_backend *backend, const git_oid *oid) +int loose_backend__read_header(size_t *len_p, git_otype *type_p, git_odb_backend *backend, const git_oid *oid) { char object_path[GIT_PATH_MAX]; + git_rawobj raw; + int error; - assert(obj && backend && oid); + assert(backend && oid); if (locate_object(object_path, (loose_backend *)backend, oid) < 0) return GIT_ENOTFOUND; - return read_header_loose(obj, object_path); -} + if ((error = read_header_loose(&raw, object_path)) < GIT_SUCCESS) + return error; + *len_p = raw.len; + *type_p = raw.type; + return GIT_SUCCESS; +} -int loose_backend__read(git_rawobj *obj, git_odb_backend *backend, const git_oid *oid) +int loose_backend__read(void **buffer_p, size_t *len_p, git_otype *type_p, git_odb_backend *backend, const git_oid *oid) { char object_path[GIT_PATH_MAX]; + git_rawobj raw; + int error; - assert(obj && backend && oid); + assert(backend && oid); if (locate_object(object_path, (loose_backend *)backend, oid) < 0) return GIT_ENOTFOUND; - return read_loose(obj, object_path); + if ((error = read_loose(&raw, object_path)) < GIT_SUCCESS) + return error; + + *buffer_p = raw.data; + *len_p = raw.len; + *type_p = raw.type; + + return GIT_SUCCESS; } int loose_backend__exists(git_odb_backend *backend, const git_oid *oid) @@ -592,32 +524,104 @@ int loose_backend__exists(git_odb_backend *backend, const git_oid *oid) return locate_object(object_path, (loose_backend *)backend, oid) == GIT_SUCCESS; } +int loose_backend__stream_fwrite(git_oid *oid, git_odb_stream *_stream) +{ + loose_writestream *stream = (loose_writestream *)_stream; + loose_backend *backend = (loose_backend *)_stream->backend; + + int error; + char final_path[GIT_PATH_MAX]; + + if ((error = git_filebuf_hash(oid, &stream->fbuf)) < GIT_SUCCESS) + return error; + + if (object_file_name(final_path, sizeof(final_path), backend->objects_dir, oid)) + return GIT_ENOMEM; -int loose_backend__write(git_oid *id, git_odb_backend *_backend, git_rawobj *obj) + if ((error = gitfo_mkdir_2file(final_path)) < GIT_SUCCESS) + return error; + + stream->finished = 1; + return git_filebuf_commit_at(&stream->fbuf, final_path); +} + +int loose_backend__stream_write(git_odb_stream *_stream, const char *data, size_t len) { + loose_writestream *stream = (loose_writestream *)_stream; + return git_filebuf_write(&stream->fbuf, data, len); +} + +void loose_backend__stream_free(git_odb_stream *_stream) +{ + loose_writestream *stream = (loose_writestream *)_stream; + + if (!stream->finished) + git_filebuf_cleanup(&stream->fbuf); + + free(stream); +} + +static int format_object_header(char *hdr, size_t n, size_t obj_len, git_otype obj_type) +{ + const char *type_str = git_object_type2string(obj_type); + int len = snprintf(hdr, n, "%s %"PRIuZ, type_str, obj_len); + + assert(len > 0); /* otherwise snprintf() is broken */ + assert(((size_t) len) < n); /* otherwise the caller is broken! */ + + if (len < 0 || ((size_t) len) >= n) + return GIT_ERROR; + return len+1; +} + +int loose_backend__stream(git_odb_stream **stream_out, git_odb_backend *_backend, size_t length, git_otype type) +{ + loose_backend *backend; + loose_writestream *stream; + char hdr[64]; int hdrlen; - gitfo_buf buf = GITFO_BUF_INIT; int error; - loose_backend *backend; - assert(id && _backend && obj); + assert(_backend); backend = (loose_backend *)_backend; + *stream_out = NULL; - if ((error = git_odb__hash_obj(id, hdr, sizeof(hdr), &hdrlen, obj)) < 0) - return error; + hdrlen = format_object_header(hdr, sizeof(hdr), length, type); + if (hdrlen < GIT_SUCCESS) + return GIT_EOBJCORRUPTED; + + stream = git__calloc(1, sizeof(loose_writestream)); + if (stream == NULL) + return GIT_ENOMEM; - if (git_odb_exists(_backend->odb, id)) - return GIT_SUCCESS; + stream->stream.backend = _backend; + stream->stream.read = NULL; /* read only */ + stream->stream.write = &loose_backend__stream_write; + stream->stream.finalize_write = &loose_backend__stream_fwrite; + stream->stream.free = &loose_backend__stream_free; + stream->stream.mode = GIT_STREAM_WRONLY; - if ((error = deflate_obj(&buf, hdr, hdrlen, obj, backend->object_zlib_level)) < 0) + error = git_filebuf_open(&stream->fbuf, NULL, + GIT_FILEBUF_HASH_CONTENTS | + GIT_FILEBUF_DEFLATE_CONTENTS | + GIT_FILEBUF_TEMPORARY); + + if (error < GIT_SUCCESS) { + free(stream); return error; + } - error = write_obj(&buf, id, backend); + error = stream->stream.write((git_odb_stream *)stream, hdr, hdrlen); + if (error < GIT_SUCCESS) { + git_filebuf_cleanup(&stream->fbuf); + free(stream); + return error; + } - gitfo_free_buf(&buf); - return error; + *stream_out = (git_odb_stream *)stream; + return GIT_SUCCESS; } void loose_backend__free(git_odb_backend *_backend) @@ -649,7 +653,7 @@ int git_odb_backend_loose(git_odb_backend **backend_out, const char *objects_dir backend->parent.read = &loose_backend__read; backend->parent.read_header = &loose_backend__read_header; - backend->parent.write = &loose_backend__write; + backend->parent.writestream = &loose_backend__stream; backend->parent.exists = &loose_backend__exists; backend->parent.free = &loose_backend__free; diff --git a/src/odb_pack.c b/src/odb_pack.c index 3067179be..65210f0b0 100644 --- a/src/odb_pack.c +++ b/src/odb_pack.c @@ -1243,7 +1243,7 @@ static int packfile_unpack_delta( error = packfile_unpack_compressed(&delta, backend, p, w_curs, curpos, delta_size, delta_type); if (error < GIT_SUCCESS) { - git_rawobj_close(&base); + free(base.data); return error; } @@ -1252,8 +1252,8 @@ static int packfile_unpack_delta( base.data, base.len, delta.data, delta.len); - git_rawobj_close(&base); - git_rawobj_close(&delta); + free(base.data); + free(delta.data); /* TODO: we might want to cache this shit. eventually */ //add_delta_base_cache(p, base_offset, base, base_size, *type); @@ -1337,15 +1337,23 @@ int pack_backend__read_header(git_rawobj *obj, git_odb_backend *backend, const g } */ -int pack_backend__read(git_rawobj *obj, git_odb_backend *backend, const git_oid *oid) +int pack_backend__read(void **buffer_p, size_t *len_p, git_otype *type_p, git_odb_backend *backend, const git_oid *oid) { struct pack_entry e; + git_rawobj raw; int error; if ((error = pack_entry_find(&e, (struct pack_backend *)backend, oid)) < GIT_SUCCESS) return error; - return packfile_unpack(obj, (struct pack_backend *)backend, e.p, e.offset); + if ((error = packfile_unpack(&raw, (struct pack_backend *)backend, e.p, e.offset)) < GIT_SUCCESS) + return error; + + *buffer_p = raw.data; + *len_p = raw.len; + *type_p = raw.type; + + return GIT_SUCCESS; } int pack_backend__exists(git_odb_backend *backend, const git_oid *oid) @@ -1397,7 +1405,6 @@ int git_odb_backend_pack(git_odb_backend **backend_out, const char *objects_dir) backend->parent.read = &pack_backend__read; backend->parent.read_header = NULL; - backend->parent.write = NULL; backend->parent.exists = &pack_backend__exists; backend->parent.free = &pack_backend__free; diff --git a/src/oid.c b/src/oid.c index 81b7d6005..eb167a685 100644 --- a/src/oid.c +++ b/src/oid.c @@ -143,14 +143,18 @@ int git__parse_oid(git_oid *oid, char **buffer_out, return GIT_SUCCESS; } -int git__write_oid(git_odb_source *src, const char *header, const git_oid *oid) +int git__write_oid(git_odb_stream *stream, const char *header, const git_oid *oid) { - char hex_oid[41]; + char hex_oid[42]; - git_oid_fmt(hex_oid, oid); - hex_oid[40] = 0; + git_oid_fmt(hex_oid + 1, oid); - return git__source_printf(src, "%s %s\n", header, hex_oid); + hex_oid[0] = ' '; + hex_oid[41] = '\n'; + + stream->write(stream, header, strlen(header)); + stream->write(stream, hex_oid, 42); + return GIT_SUCCESS; } void git_oid_mkraw(git_oid *out, const unsigned char *raw) diff --git a/src/repository.c b/src/repository.c index 37aa44781..132969402 100644 --- a/src/repository.c +++ b/src/repository.c @@ -40,29 +40,11 @@ #define GIT_BRANCH_MASTER "master" -static const int OBJECT_TABLE_SIZE = 32; - typedef struct { char *path_repository; unsigned is_bare:1, has_been_reinit:1; } repo_init; -/* - * Hash table methods - * - * Callbacks for the ODB cache, implemented - * as a hash table - */ -static uint32_t object_table_hash(const void *key, int hash_id) -{ - uint32_t r; - git_oid *id; - - id = (git_oid *)key; - memcpy(&r, id->id + (hash_id * sizeof(uint32_t)), sizeof(r)); - return r; -} - /* * Git repository open methods * @@ -186,25 +168,9 @@ static git_repository *repository_alloc() memset(repo, 0x0, sizeof(git_repository)); - repo->objects = git_hashtable_alloc( - OBJECT_TABLE_SIZE, - object_table_hash, - (git_hash_keyeq_ptr)git_oid_cmp); - - if (repo->objects == NULL) { - free(repo); - return NULL; - } + git_cache_init(&repo->objects, GIT_DEFAULT_CACHE_SIZE, &git_object__free); if (git_repository__refcache_init(&repo->references) < GIT_SUCCESS) { - git_hashtable_free(repo->objects); - free(repo); - return NULL; - } - - if (git_vector_init(&repo->memory_objects, 16, NULL) < GIT_SUCCESS) { - git_hashtable_free(repo->objects); - git_repository__refcache_free(&repo->references); free(repo); return NULL; } @@ -330,51 +296,19 @@ cleanup: return error; } -int git_repository_gc(git_repository *repo) -{ - int collected = 0; - git_object *object; - const void *_unused; - - GIT_HASHTABLE_FOREACH(repo->objects, _unused, object, - if (object->can_free) { - git_object__free(object); - collected++; - } - ); - - return collected; -} - void git_repository_free(git_repository *repo) { - git_object *object; - const void *_unused; - unsigned int i; - if (repo == NULL) return; - /* force free all the objects */ - GIT_HASHTABLE_FOREACH(repo->objects, _unused, object, - git_object__free(object); - ); - - for (i = 0; i < repo->memory_objects.length; ++i) { - object = git_vector_get(&repo->memory_objects, i); - git_object__free(object); - } + git_cache_free(&repo->objects); + git_repository__refcache_free(&repo->references); free(repo->path_workdir); free(repo->path_index); free(repo->path_repository); free(repo->path_odb); - git_hashtable_free(repo->objects); - git_vector_free(&repo->memory_objects); - - git_repository__refcache_free(&repo->references); - if (repo->db != NULL) git_odb_close(repo->db); diff --git a/src/repository.h b/src/repository.h index 48b8dae6b..fef1c7da0 100644 --- a/src/repository.h +++ b/src/repository.h @@ -9,6 +9,7 @@ #include "hashtable.h" #include "index.h" +#include "cache.h" #include "refs.h" #define DOT_GIT ".git" @@ -16,28 +17,17 @@ #define GIT_OBJECTS_DIR "objects/" #define GIT_INDEX_FILE "index" -typedef struct { - git_rawobj raw; - void *write_ptr; - size_t written_bytes; - int open:1; -} git_odb_source; - struct git_object { - git_oid id; + git_cached_obj cached; git_repository *repo; - git_odb_source source; - unsigned int lru; - unsigned char in_memory, modified, can_free, _pad; + git_otype type; }; struct git_repository { git_odb *db; git_index *index; - git_hashtable *objects; - git_vector memory_objects; - + git_cache objects; git_refcache references; char *path_repository; @@ -49,17 +39,11 @@ struct git_repository { unsigned int lru_counter; }; -int git_object__source_open(git_object *object); -void git_object__source_close(git_object *object); - /* fully free the object; internal method, do not * export */ -void git_object__free(git_object *object); - -int git__source_printf(git_odb_source *source, const char *format, ...); -int git__source_write(git_odb_source *source, const void *bytes, size_t len); +void git_object__free(void *object); int git__parse_oid(git_oid *oid, char **buffer_out, const char *buffer_end, const char *header); -int git__write_oid(git_odb_source *src, const char *header, const git_oid *oid); +int git__write_oid(git_odb_stream *src, const char *header, const git_oid *oid); #endif diff --git a/src/revwalk.c b/src/revwalk.c index 3b7ad34e6..a9d4f8734 100644 --- a/src/revwalk.c +++ b/src/revwalk.c @@ -25,6 +25,7 @@ #include "common.h" #include "commit.h" +#include "odb.h" #include "hashtable.h" #include "pqueue.h" @@ -236,22 +237,22 @@ static int commit_quick_parse(git_revwalk *walk, commit_object *commit, git_rawo static int commit_parse(git_revwalk *walk, commit_object *commit) { - git_rawobj data; + git_odb_object *obj; int error; if (commit->parsed) return GIT_SUCCESS; - if ((error = git_odb_read(&data, walk->repo->db, &commit->oid)) < GIT_SUCCESS) + if ((error = git_odb_read(&obj, walk->repo->db, &commit->oid)) < GIT_SUCCESS) return error; - if (data.type != GIT_OBJ_COMMIT) { - git_rawobj_close(&data); + if (obj->raw.type != GIT_OBJ_COMMIT) { + git_odb_object_close(obj); return GIT_EOBJTYPE; } - error = commit_quick_parse(walk, commit, &data); - git_rawobj_close(&data); + error = commit_quick_parse(walk, commit, &obj->raw); + git_odb_object_close(obj); return error; } diff --git a/src/signature.c b/src/signature.c index 5c9f15973..13816c396 100644 --- a/src/signature.c +++ b/src/signature.c @@ -46,13 +46,7 @@ git_signature *git_signature_new(const char *name, const char *email, time_t tim goto cleanup; p->name = git__strdup(name); - if (p->name == NULL) - goto cleanup; - p->email = git__strdup(email); - if (p->email == NULL) - goto cleanup; - p->when.time = time; p->when.offset = offset; @@ -179,10 +173,12 @@ int git_signature__parse(git_signature *sig, char **buffer_out, return GIT_SUCCESS; } -int git_signature__write(git_odb_source *src, const char *header, const git_signature *sig) +int git_signature__write(char **signature, const char *header, const git_signature *sig) { - char sign; int offset, hours, mins; + char sig_buffer[2048]; + int sig_buffer_len; + char sign; offset = sig->when.offset; sign = (sig->when.offset < 0) ? '-' : '+'; @@ -193,7 +189,16 @@ int git_signature__write(git_odb_source *src, const char *header, const git_sign hours = offset / 60; mins = offset % 60; - return git__source_printf(src, "%s %s <%s> %u %c%02d%02d\n", header, sig->name, sig->email, (unsigned)sig->when.time, sign, hours, mins); + sig_buffer_len = snprintf(sig_buffer, sizeof(sig_buffer), + "%s %s <%s> %u %c%02d%02d\n", + header, sig->name, sig->email, + (unsigned)sig->when.time, sign, hours, mins); + + if (sig_buffer_len < 0 || (size_t)sig_buffer_len > sizeof(sig_buffer)) + return GIT_ENOMEM; + + *signature = git__strdup(sig_buffer); + return sig_buffer_len; } diff --git a/src/signature.h b/src/signature.h index ee212c2dc..3534cb21f 100644 --- a/src/signature.h +++ b/src/signature.h @@ -7,6 +7,6 @@ #include int git_signature__parse(git_signature *sig, char **buffer_out, const char *buffer_end, const char *header); -int git_signature__write(git_odb_source *src, const char *header, const git_signature *sig); +int git_signature__write(char **signature, const char *header, const git_signature *sig); #endif diff --git a/src/tag.c b/src/tag.c index 0489e7e4a..7baababbf 100644 --- a/src/tag.c +++ b/src/tag.c @@ -56,21 +56,6 @@ const git_oid *git_tag_target_oid(git_tag *t) return &t->target; } -int git_tag_set_target(git_tag *tag, git_object *target) -{ - const git_oid *oid; - - assert(tag && target); - - if ((oid = git_object_id(target)) == NULL) - return GIT_EMISSINGOBJDATA; - - tag->object.modified = 1; - git_oid_cpy(&tag->target, oid); - tag->type = git_object_type(target); - return GIT_SUCCESS; -} - git_otype git_tag_type(git_tag *t) { assert(t); @@ -83,50 +68,17 @@ const char *git_tag_name(git_tag *t) return t->tag_name; } -void git_tag_set_name(git_tag *tag, const char *name) -{ - assert(tag && name); - - tag->object.modified = 1; - - if (tag->tag_name) - free(tag->tag_name); - - tag->tag_name = git__strdup(name); -} - const git_signature *git_tag_tagger(git_tag *t) { return t->tagger; } -void git_tag_set_tagger(git_tag *tag, const git_signature *tagger_sig) -{ - assert(tag && tagger_sig); - tag->object.modified = 1; - - git_signature_free(tag->tagger); - tag->tagger = git_signature_dup(tagger_sig); -} - const char *git_tag_message(git_tag *t) { assert(t); return t->message; } -void git_tag_set_message(git_tag *tag, const char *message) -{ - assert(tag && message); - - tag->object.modified = 1; - - if (tag->message) - free(tag->message); - - tag->message = git__strdup(message); -} - static int parse_tag_buffer(git_tag *tag, char *buffer, const char *buffer_end) { static const char *tag_types[] = { @@ -187,9 +139,6 @@ static int parse_tag_buffer(git_tag *tag, char *buffer, const char *buffer_end) buffer = search + 1; - if (tag->tagger != NULL) - git_signature_free(tag->tagger); - tag->tagger = git__malloc(sizeof(git_signature)); if ((error = git_signature__parse(tag->tagger, &buffer, buffer_end, "tagger ")) != 0) @@ -197,9 +146,6 @@ static int parse_tag_buffer(git_tag *tag, char *buffer, const char *buffer_end) text_len = buffer_end - ++buffer; - if (tag->message != NULL) - free(tag->message); - tag->message = git__malloc(text_len + 1); memcpy(tag->message, buffer, text_len); tag->message[text_len] = '\0'; @@ -207,26 +153,90 @@ static int parse_tag_buffer(git_tag *tag, char *buffer, const char *buffer_end) return GIT_SUCCESS; } -int git_tag__writeback(git_tag *tag, git_odb_source *src) +int git_tag_create_o( + git_oid *oid, + git_repository *repo, + const char *tag_name, + const git_object *target, + const git_signature *tagger, + const char *message) { - if (tag->tag_name == NULL || tag->tagger == NULL) - return GIT_EMISSINGOBJDATA; + return git_tag_create( + oid, repo, tag_name, + git_object_id(target), + git_object_type(target), + tagger, message); +} - git__write_oid(src, "object", &tag->target); - git__source_printf(src, "type %s\n", git_object_type2string(tag->type)); - git__source_printf(src, "tag %s\n", tag->tag_name); - git_signature__write(src, "tagger", tag->tagger); +int git_tag_create( + git_oid *oid, + git_repository *repo, + const char *tag_name, + const git_oid *target, + git_otype target_type, + const git_signature *tagger, + const char *message) +{ + size_t final_size = 0; + git_odb_stream *stream; - if (tag->message != NULL) - git__source_printf(src, "\n%s", tag->message); + const char *type_str; + char *tagger_str; - return GIT_SUCCESS; + int type_str_len, tag_name_len, tagger_str_len, message_len; + int error; + + + type_str = git_object_type2string(target_type); + + tagger_str_len = git_signature__write(&tagger_str, "tagger", tagger); + + type_str_len = strlen(type_str); + tag_name_len = strlen(tag_name); + message_len = strlen(message); + + final_size += GIT_OID_LINE_LENGTH("object"); + final_size += STRLEN("type ") + type_str_len + 1; + final_size += STRLEN("tag ") + tag_name_len + 1; + final_size += tagger_str_len; + final_size += 1 + message_len; + + if ((error = git_odb_open_wstream(&stream, repo->db, final_size, GIT_OBJ_TAG)) < GIT_SUCCESS) + return error; + + git__write_oid(stream, "object", target); + + stream->write(stream, "type ", STRLEN("type ")); + stream->write(stream, type_str, type_str_len); + + stream->write(stream, "\ntag ", STRLEN("\ntag ")); + stream->write(stream, tag_name, tag_name_len); + stream->write(stream, "\n", 1); + + stream->write(stream, tagger_str, tagger_str_len); + free(tagger_str); + + stream->write(stream, "\n", 1); + stream->write(stream, message, message_len); + + + error = stream->finalize_write(oid, stream); + stream->free(stream); + + if (error == GIT_SUCCESS) { + char ref_name[512]; + git_reference *new_ref; + git__joinpath(ref_name, GIT_REFS_TAGS_DIR, tag_name); + error = git_reference_create_oid(&new_ref, repo, ref_name, oid); + } + + return error; } -int git_tag__parse(git_tag *tag) +int git_tag__parse(git_tag *tag, git_odb_object *obj) { - assert(tag && tag->object.source.open); - return parse_tag_buffer(tag, tag->object.source.raw.data, (char *)tag->object.source.raw.data + tag->object.source.raw.len); + assert(tag); + return parse_tag_buffer(tag, obj->raw.data, (char *)obj->raw.data + obj->raw.len); } diff --git a/src/tag.h b/src/tag.h index a1782d064..eddf8fa3a 100644 --- a/src/tag.h +++ b/src/tag.h @@ -3,6 +3,7 @@ #include "git2/tag.h" #include "repository.h" +#include "odb.h" struct git_tag { git_object object; @@ -16,7 +17,6 @@ struct git_tag { }; void git_tag__free(git_tag *tag); -int git_tag__parse(git_tag *tag); -int git_tag__writeback(git_tag *tag, git_odb_source *src); +int git_tag__parse(git_tag *tag, git_odb_object *obj); #endif diff --git a/src/thread-utils.h b/src/thread-utils.h index 1cf0e3407..e542639c8 100644 --- a/src/thread-utils.h +++ b/src/thread-utils.h @@ -8,7 +8,7 @@ typedef struct { volatile int val; } git_atomic; -static inline void git_atomic_set(git_atomic *a, int val) +GIT_INLINE(void) git_atomic_set(git_atomic *a, int val) { a->val = val; } @@ -36,7 +36,7 @@ static inline void git_atomic_set(git_atomic *a, int val) #define git_cond_signal(c) (void)0 //pthread_cond_signal(c) #define git_cond_broadcast(c) (void)0 //pthread_cond_broadcast(c) -static inline int git_atomic_inc(git_atomic *a) +GIT_INLINE(int) git_atomic_inc(git_atomic *a) { #ifdef __GNUC__ return __sync_add_and_fetch(&a->val, 1); @@ -47,7 +47,7 @@ static inline int git_atomic_inc(git_atomic *a) #endif } -static inline int git_atomic_dec(git_atomic *a) +GIT_INLINE(int) git_atomic_dec(git_atomic *a) { #ifdef __GNUC__ return __sync_sub_and_fetch(&a->val, 1); @@ -81,12 +81,12 @@ static inline int git_atomic_dec(git_atomic *a) #define git_cond_signal(c) (void)0 #define git_cond_broadcast(c) (void)0 -static inline int git_atomic_inc(git_atomic *a) +GIT_INLINE(int) git_atomic_inc(git_atomic *a) { return ++a->val; } -static inline int git_atomic_dec(git_atomic *a) +GIT_INLINE(int) git_atomic_dec(git_atomic *a) { return --a->val; } diff --git a/src/tree.c b/src/tree.c index 307d41b0d..31b286e69 100644 --- a/src/tree.c +++ b/src/tree.c @@ -41,9 +41,11 @@ int entry_search_cmp(const void *key, const void *array_member) return strcmp(filename, entry->filename); } +#if 0 static int valid_attributes(const int attributes) { return attributes >= 0 && attributes <= MAX_FILEMODE; } +#endif int entry_sort_cmp(const void *a, const void *b) { @@ -56,13 +58,10 @@ int entry_sort_cmp(const void *a, const void *b) entry_b->attr & 040000); } -void git_tree_clear_entries(git_tree *tree) +void git_tree__free(git_tree *tree) { unsigned int i; - if (tree == NULL) - return; - for (i = 0; i < tree->entries.length; ++i) { git_tree_entry *e; e = git_vector_get(&tree->entries, i); @@ -71,32 +70,6 @@ void git_tree_clear_entries(git_tree *tree) free(e); } - git_vector_clear(&tree->entries); - tree->object.modified = 1; -} - - -git_tree *git_tree__new(void) -{ - git_tree *tree; - - tree = git__malloc(sizeof(struct git_tree)); - if (tree == NULL) - return NULL; - - memset(tree, 0x0, sizeof(struct git_tree)); - - if (git_vector_init(&tree->entries, DEFAULT_TREE_SIZE, entry_sort_cmp) < GIT_SUCCESS) { - free(tree); - return NULL; - } - - return tree; -} - -void git_tree__free(git_tree *tree) -{ - git_tree_clear_entries(tree); git_vector_free(&tree->entries); free(tree); } @@ -106,37 +79,6 @@ const git_oid *git_tree_id(git_tree *c) return git_object_id((git_object *)c); } -int git_tree_entry_set_attributes(git_tree_entry *entry, unsigned int attr) -{ - assert(entry && entry->owner); - - if (!valid_attributes(attr)) { - return GIT_ERROR; - } - - entry->attr = attr; - entry->owner->object.modified = 1; - return GIT_SUCCESS; -} - -void git_tree_entry_set_name(git_tree_entry *entry, const char *name) -{ - assert(entry && entry->owner); - - free(entry->filename); - entry->filename = git__strdup(name); - git_vector_sort(&entry->owner->entries); - entry->owner->object.modified = 1; -} - -void git_tree_entry_set_id(git_tree_entry *entry, const git_oid *oid) -{ - assert(entry && entry->owner); - - git_oid_cpy(&entry->oid, oid); - entry->owner->object.modified = 1; -} - unsigned int git_tree_entry_attributes(git_tree_entry *entry) { return entry->attr; @@ -154,15 +96,10 @@ const git_oid *git_tree_entry_id(git_tree_entry *entry) return &entry->oid; } -int git_tree_entry_2object(git_object **object_out, git_tree_entry *entry) +int git_tree_entry_2object(git_object **object_out, git_repository *repo, git_tree_entry *entry) { assert(entry && object_out); - return git_object_lookup(object_out, entry->owner->object.repo, &entry->oid, GIT_OBJ_ANY); -} - -static void sort_entries(git_tree *tree) -{ - git_vector_sort(&tree->entries); + return git_object_lookup(object_out, repo, &entry->oid, GIT_OBJ_ANY); } git_tree_entry *git_tree_entry_byname(git_tree *tree, const char *filename) @@ -171,8 +108,6 @@ git_tree_entry *git_tree_entry_byname(git_tree *tree, const char *filename) assert(tree && filename); - sort_entries(tree); - idx = git_vector_bsearch2(&tree->entries, entry_search_cmp, filename); if (idx == GIT_ENOTFOUND) return NULL; @@ -183,9 +118,6 @@ git_tree_entry *git_tree_entry_byname(git_tree *tree, const char *filename) git_tree_entry *git_tree_entry_byindex(git_tree *tree, int idx) { assert(tree); - - sort_entries(tree); - return git_vector_get(&tree->entries, (unsigned int)idx); } @@ -195,107 +127,12 @@ size_t git_tree_entrycount(git_tree *tree) return tree->entries.length; } -int git_tree_add_entry(git_tree_entry **entry_out, git_tree *tree, const git_oid *id, const char *filename, int attributes) -{ - git_tree_entry *entry; - - assert(tree && id && filename); - if (!valid_attributes(attributes)) { - return GIT_ERROR; - } - - if ((entry = git__malloc(sizeof(git_tree_entry))) == NULL) - return GIT_ENOMEM; - - memset(entry, 0x0, sizeof(git_tree_entry)); - - entry->filename = git__strdup(filename); - git_oid_cpy(&entry->oid, id); - entry->attr = attributes; - entry->owner = tree; - - if (git_vector_insert(&tree->entries, entry) < 0) - return GIT_ENOMEM; - - if (entry_out != NULL) - *entry_out = entry; - - tree->object.modified = 1; - return GIT_SUCCESS; -} - -int git_tree_remove_entry_byindex(git_tree *tree, int idx) -{ - git_tree_entry *remove_ptr; - - assert(tree); - - sort_entries(tree); - - remove_ptr = git_vector_get(&tree->entries, (unsigned int)idx); - if (remove_ptr == NULL) - return GIT_ENOTFOUND; - - free(remove_ptr->filename); - free(remove_ptr); - - tree->object.modified = 1; - - return git_vector_remove(&tree->entries, (unsigned int)idx); -} - -int git_tree_remove_entry_byname(git_tree *tree, const char *filename) -{ - int idx; - - assert(tree && filename); - - sort_entries(tree); - - idx = git_vector_bsearch2(&tree->entries, entry_search_cmp, filename); - if (idx == GIT_ENOTFOUND) - return GIT_ENOTFOUND; - - return git_tree_remove_entry_byindex(tree, idx); -} - -int git_tree__writeback(git_tree *tree, git_odb_source *src) -{ - size_t i; - char filemode[MAX_FILEMODE_BYTES + 1 + 1]; - - assert(tree && src); - - if (tree->entries.length == 0) - return GIT_EMISSINGOBJDATA; - - sort_entries(tree); - - for (i = 0; i < tree->entries.length; ++i) { - git_tree_entry *entry; - - entry = git_vector_get(&tree->entries, i); - - snprintf(filemode, sizeof(filemode), "%o ", entry->attr); - - git__source_write(src, filemode, strlen(filemode)); - git__source_write(src, entry->filename, strlen(entry->filename) + 1); - git__source_write(src, entry->oid.id, GIT_OID_RAWSZ); - } - - return GIT_SUCCESS; -} - - static int tree_parse_buffer(git_tree *tree, char *buffer, char *buffer_end) { - static const size_t avg_entry_size = 40; - unsigned int expected_size; int error = GIT_SUCCESS; - expected_size = (tree->object.source.raw.len / avg_entry_size) + 1; - - git_tree_clear_entries(tree); + if (git_vector_init(&tree->entries, DEFAULT_TREE_SIZE, entry_sort_cmp) < GIT_SUCCESS) + return GIT_ENOMEM; while (buffer < buffer_end) { git_tree_entry *entry; @@ -309,7 +146,6 @@ static int tree_parse_buffer(git_tree *tree, char *buffer, char *buffer_end) if (git_vector_insert(&tree->entries, entry) < GIT_SUCCESS) return GIT_ENOMEM; - entry->owner = tree; entry->attr = strtol(buffer, &buffer, 8); if (*buffer++ != ' ') { @@ -336,16 +172,9 @@ static int tree_parse_buffer(git_tree *tree, char *buffer, char *buffer_end) return error; } -int git_tree__parse(git_tree *tree) +int git_tree__parse(git_tree *tree, git_odb_object *obj) { - char *buffer, *buffer_end; - - assert(tree && tree->object.source.open); - assert(!tree->object.in_memory); - - buffer = tree->object.source.raw.data; - buffer_end = buffer + tree->object.source.raw.len; - - return tree_parse_buffer(tree, buffer, buffer_end); + assert(tree); + return tree_parse_buffer(tree, (char *)obj->raw.data, (char *)obj->raw.data + obj->raw.len); } diff --git a/src/tree.h b/src/tree.h index 78500c471..b4e910a9f 100644 --- a/src/tree.h +++ b/src/tree.h @@ -3,14 +3,13 @@ #include "git2/tree.h" #include "repository.h" +#include "odb.h" #include "vector.h" struct git_tree_entry { unsigned int attr; char *filename; git_oid oid; - - git_tree *owner; }; struct git_tree { @@ -19,8 +18,6 @@ struct git_tree { }; void git_tree__free(git_tree *tree); -git_tree *git_tree__new(void); -int git_tree__parse(git_tree *tree); -int git_tree__writeback(git_tree *tree, git_odb_source *src); +int git_tree__parse(git_tree *tree, git_odb_object *obj); #endif diff --git a/src/util.h b/src/util.h index e0dfd7b20..653b34d02 100644 --- a/src/util.h +++ b/src/util.h @@ -97,6 +97,8 @@ extern char *git__strtok_keep(char *output, char *src, char *delimit); #define STRLEN(str) (sizeof(str) - 1) +#define GIT_OID_LINE_LENGTH(header) (STRLEN(header) + 1 + GIT_OID_HEXSZ + 1) + /* * Realloc the buffer pointed at by variable 'x' so that it can hold * at least 'nr' entries; the number of entries currently allocated -- cgit v1.2.1 From 99baacfb5687c266df78be4442d30f2f2dc19a28 Mon Sep 17 00:00:00 2001 From: Vicent Marti Date: Mon, 21 Mar 2011 19:27:45 +0200 Subject: Fix MSVC warnings --- src/cache.c | 2 +- src/commit.c | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) (limited to 'src') diff --git a/src/cache.c b/src/cache.c index fdf5c1b80..fd42e2c5b 100644 --- a/src/cache.c +++ b/src/cache.c @@ -78,7 +78,7 @@ void *git_cache_get(git_cache *cache, const git_oid *oid) { const uint32_t *hash; size_t i, pos, found = 0; - cache_node *node; + cache_node *node = NULL; hash = (const uint32_t *)oid->id; diff --git a/src/commit.c b/src/commit.c index 2670c2e0a..d5d6ebd8a 100644 --- a/src/commit.c +++ b/src/commit.c @@ -101,7 +101,7 @@ int git_commit_create_v( oid, repo, update_ref, author, committer, message, tree_oid, parent_count, oids); - free(oids); + free((void *)oids); return error; } @@ -132,7 +132,7 @@ int git_commit_create_ov( git_object_id((git_object *)tree), parent_count, oids); - free(oids); + free((void *)oids); return error; } @@ -160,7 +160,7 @@ int git_commit_create_o( git_object_id((git_object *)tree), parent_count, oids); - free(oids); + free((void *)oids); return error; } -- cgit v1.2.1 From 567fc1d20c734c34dd1c4c19be12abcc1622aa5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Mart=C3=ADn=20Nieto?= Date: Tue, 22 Mar 2011 12:47:53 +0100 Subject: refs loose_lookup: also free the buffer on success Free the ref_file buffer at the end of the function also on success. This fixes a small memory leak. --- src/refs.c | 1 + 1 file changed, 1 insertion(+) (limited to 'src') diff --git a/src/refs.c b/src/refs.c index 40b80ec9b..8e2965528 100644 --- a/src/refs.c +++ b/src/refs.c @@ -333,6 +333,7 @@ static int loose_lookup( ref->mtime = ref_time; *ref_out = ref; + gitfo_free_buf(&ref_file); return GIT_SUCCESS; cleanup: -- cgit v1.2.1 From b932ef5b3d77b26b8f558bafa3ee416595279dce Mon Sep 17 00:00:00 2001 From: Vicent Marti Date: Mon, 21 Mar 2011 19:44:43 +0200 Subject: Fix MSVC warnings when building threads --- src/thread-utils.h | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'src') diff --git a/src/thread-utils.h b/src/thread-utils.h index e542639c8..20d6022ea 100644 --- a/src/thread-utils.h +++ b/src/thread-utils.h @@ -5,7 +5,11 @@ /* Common operations even if threading has been disabled */ typedef struct { +#if defined(_MSC_VER) + volatile long val; +#else volatile int val; +#endif } git_atomic; GIT_INLINE(void) git_atomic_set(git_atomic *a, int val) -- cgit v1.2.1 From 1881f0783b6dd23fb40ba0b43a4ec98bb08a637c Mon Sep 17 00:00:00 2001 From: Vicent Marti Date: Mon, 21 Mar 2011 20:28:02 +0200 Subject: Add getters for `git_odb_object` --- src/odb.c | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) (limited to 'src') diff --git a/src/odb.c b/src/odb.c index 9aeaa8a23..d825fd95c 100644 --- a/src/odb.c +++ b/src/odb.c @@ -109,6 +109,26 @@ static void free_odb_object(void *o) } } +const git_oid *git_odb_object_id(git_odb_object *object) +{ + return &object->cached.oid; +} + +const void *git_odb_object_data(git_odb_object *object) +{ + return object->raw.data; +} + +size_t git_odb_object_size(git_odb_object *object) +{ + return object->raw.len; +} + +git_otype git_odb_object_type(git_odb_object *object) +{ + return object->raw.type; +} + void git_odb_object_close(git_odb_object *object) { git_cached_obj_decref((git_cached_obj *)object, &free_odb_object); -- cgit v1.2.1 From 21d73e7195a427b39c1c3742c85254e8384f5802 Mon Sep 17 00:00:00 2001 From: Vicent Marti Date: Tue, 22 Mar 2011 20:26:01 +0200 Subject: Always free the parents of a revwalk commit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Thanks to Carlos Martín Nieto for spotting this. --- src/revwalk.c | 11 +++++++++++ 1 file changed, 11 insertions(+) (limited to 'src') diff --git a/src/revwalk.c b/src/revwalk.c index a9d4f8734..73bb060f5 100644 --- a/src/revwalk.c +++ b/src/revwalk.c @@ -482,11 +482,22 @@ int git_revwalk_new(git_revwalk **revwalk_out, git_repository *repo) void git_revwalk_free(git_revwalk *walk) { unsigned int i; + const void *_unused; + commit_object *commit; if (walk == NULL) return; git_revwalk_reset(walk); + + /* if the parent has more than PARENTS_PER_COMMIT parents, + * we had to allocate a separate array for those parents. + * make sure it's being free'd */ + GIT_HASHTABLE_FOREACH(walk->commits, _unused, commit, { + if (commit->out_degree > PARENTS_PER_COMMIT) + free(commit->parents); + }); + git_hashtable_free(walk->commits); git_pqueue_free(&walk->iterator_time); -- cgit v1.2.1 From 56d8ca266c92e51a1b38ac68b7aa6a24193f5812 Mon Sep 17 00:00:00 2001 From: nulltoken Date: Sun, 20 Mar 2011 18:36:25 +0100 Subject: Switch from time_t to git_time_t git_time_t is defined as a signed 64 integer. This allows a true predictable multiplatform behavior. --- src/commit.c | 2 +- src/index.c | 12 ++++++------ src/odb_pack.c | 4 ++-- src/signature.c | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) (limited to 'src') diff --git a/src/commit.c b/src/commit.c index d5d6ebd8a..03b111da5 100644 --- a/src/commit.c +++ b/src/commit.c @@ -315,7 +315,7 @@ GIT_COMMIT_GETTER(const git_signature *, author, commit->author) GIT_COMMIT_GETTER(const git_signature *, committer, commit->committer) GIT_COMMIT_GETTER(const char *, message, commit->message) GIT_COMMIT_GETTER(const char *, message_short, commit->message_short) -GIT_COMMIT_GETTER(time_t, time, commit->committer->when.time) +GIT_COMMIT_GETTER(git_time_t, time, commit->committer->when.time) GIT_COMMIT_GETTER(int, time_offset, commit->committer->when.offset) GIT_COMMIT_GETTER(unsigned int, parentcount, commit->parent_oids.length) diff --git a/src/index.c b/src/index.c index 6a355e11b..34208731c 100644 --- a/src/index.c +++ b/src/index.c @@ -312,8 +312,8 @@ int git_index_add(git_index *index, const char *rel_path, int stage) memset(&entry, 0x0, sizeof(git_index_entry)); - entry.ctime.seconds = st.st_ctime; - entry.mtime.seconds = st.st_mtime; + entry.ctime.seconds = (git_time_t)st.st_ctime; + entry.mtime.seconds = (git_time_t)st.st_mtime; /* entry.mtime.nanoseconds = st.st_mtimensec; */ /* entry.ctime.nanoseconds = st.st_ctimensec; */ entry.dev= st.st_rdev; @@ -491,10 +491,10 @@ static size_t read_entry(git_index_entry *dest, const void *buffer, size_t buffe source = (const struct entry_short *)(buffer); - dest->ctime.seconds = (time_t)ntohl(source->ctime.seconds); - dest->ctime.nanoseconds = (time_t)ntohl(source->ctime.nanoseconds); - dest->mtime.seconds = (time_t)ntohl(source->mtime.seconds); - dest->mtime.nanoseconds = (time_t)ntohl(source->mtime.nanoseconds); + dest->ctime.seconds = (git_time_t)ntohl(source->ctime.seconds); + dest->ctime.nanoseconds = ntohl(source->ctime.nanoseconds); + dest->mtime.seconds = (git_time_t)ntohl(source->mtime.seconds); + dest->mtime.nanoseconds = ntohl(source->mtime.nanoseconds); dest->dev = ntohl(source->dev); dest->ino = ntohl(source->ino); dest->mode = ntohl(source->mode); diff --git a/src/odb_pack.c b/src/odb_pack.c index 65210f0b0..8c527bcf3 100644 --- a/src/odb_pack.c +++ b/src/odb_pack.c @@ -93,7 +93,7 @@ struct pack_file { git_oid *bad_object_sha1; /* array of git_oid */ int index_version; - time_t mtime; + git_time_t mtime; int pack_fd; unsigned pack_local:1, pack_keep:1; git_oid sha1; @@ -827,7 +827,7 @@ static int packfile_check(struct pack_file **pack_out, const char *path, int loc */ p->pack_size = (off_t)st.st_size; p->pack_local = local; - p->mtime = st.st_mtime; + p->mtime = (git_time_t)st.st_mtime; /* see if we can parse the sha1 oid in the packfile name */ if (path_len < 40 || diff --git a/src/signature.c b/src/signature.c index 13816c396..412637600 100644 --- a/src/signature.c +++ b/src/signature.c @@ -38,7 +38,7 @@ void git_signature_free(git_signature *sig) free(sig); } -git_signature *git_signature_new(const char *name, const char *email, time_t time, int offset) +git_signature *git_signature_new(const char *name, const char *email, git_time_t time, int offset) { git_signature *p = NULL; -- cgit v1.2.1 From ae6ba7f713804e1c04febee81a8f52ea0abf443b Mon Sep 17 00:00:00 2001 From: nulltoken Date: Tue, 22 Mar 2011 19:45:01 +0100 Subject: Fix gitfo_mv() behavior when running on Windows When the system temporary folder is located on a different volume than the working directory into which libgit2 is executing, MoveFileEx() requires an additional flag. --- src/fileops.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/fileops.c b/src/fileops.c index 237a16a9f..a884b4002 100644 --- a/src/fileops.c +++ b/src/fileops.c @@ -263,7 +263,7 @@ int gitfo_mv(const char *from, const char *to) * file exists, the `rename` call fails. This is as * close as it gets with the Win32 API. */ - return MoveFileEx(from, to, MOVEFILE_REPLACE_EXISTING) ? GIT_SUCCESS : GIT_EOSERR; + return MoveFileEx(from, to, MOVEFILE_REPLACE_EXISTING | MOVEFILE_COPY_ALLOWED) ? GIT_SUCCESS : GIT_EOSERR; #else /* Don't even try this on Win32 */ if (!link(from, to)) { -- cgit v1.2.1 From 677a3c07f47f46dcbab4a3ee8a1e2aace6f999b0 Mon Sep 17 00:00:00 2001 From: nulltoken Date: Tue, 15 Mar 2011 22:07:01 +0100 Subject: Add failing test for issue 84 see https://github.com/libgit2/libgit2/issues#issue/84 --- src/fileops.c | 31 +++++++++++++++++++++++++++++++ src/fileops.h | 2 ++ 2 files changed, 33 insertions(+) (limited to 'src') diff --git a/src/fileops.c b/src/fileops.c index a884b4002..4ed53bc97 100644 --- a/src/fileops.c +++ b/src/fileops.c @@ -633,3 +633,34 @@ int gitfo_cmp_path(const char *name1, int len1, int isdir1, return 0; } +static void posixify_path(char *path) +{ + while (*path) { + if (*path == '\\') + *path = '/'; + + path++; + } +} + +int gitfo_getcwd(char *buffer_out, size_t size) +{ + char *cwd_buffer; + + assert(buffer_out && size > 0); + +#ifdef GIT_WIN32 + cwd_buffer = _getcwd(buffer_out, size); +#else + cwd_buffer = getcwd(buffer_out, size); //TODO: Fixme. Ensure the required headers are correctly included +#endif + + if (cwd_buffer == NULL) + return GIT_EOSERR; + + posixify_path(buffer_out); + + git__joinpath(buffer_out, buffer_out, ""); //Ensure the path ends with a trailing slash + + return GIT_SUCCESS; +} diff --git a/src/fileops.h b/src/fileops.h index bc636fc38..bd59011d5 100644 --- a/src/fileops.h +++ b/src/fileops.h @@ -144,6 +144,8 @@ extern int gitfo_close_cached(gitfo_cache *ioc); extern int gitfo_cmp_path(const char *name1, int len1, int isdir1, const char *name2, int len2, int isdir2); +extern int gitfo_getcwd(char *buffer_out, size_t size); + /** * Clean up a provided absolute or relative directory path. * -- cgit v1.2.1 From 6279abda7370ebb046571437012dc6bb96e1edba Mon Sep 17 00:00:00 2001 From: nulltoken Date: Thu, 17 Mar 2011 06:56:58 +0100 Subject: Make gitfo_prettify_dir_path() and gitfo_prettify_file_path() no longer externalized --- src/fileops.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/fileops.h b/src/fileops.h index bd59011d5..c2c038b26 100644 --- a/src/fileops.h +++ b/src/fileops.h @@ -168,7 +168,7 @@ extern int gitfo_getcwd(char *buffer_out, size_t size); * - GIT_SUCCESS on success; * - GIT_ERROR when the input path is invalid or escapes the current directory. */ -GIT_EXTERN(int) gitfo_prettify_dir_path(char *buffer_out, const char *path); +int gitfo_prettify_dir_path(char *buffer_out, const char *path); /** * Clean up a provided absolute or relative file path. @@ -190,6 +190,6 @@ GIT_EXTERN(int) gitfo_prettify_dir_path(char *buffer_out, const char *path); * - GIT_SUCCESS on success; * - GIT_ERROR when the input path is invalid or escapes the current directory. */ -GIT_EXTERN(int) gitfo_prettify_file_path(char *buffer_out, const char *path); +int gitfo_prettify_file_path(char *buffer_out, const char *path); #endif /* INCLUDE_fileops_h__ */ -- cgit v1.2.1 From c90292ce4f852ff9f0e8ec4ddc7cdc438745790d Mon Sep 17 00:00:00 2001 From: nulltoken Date: Fri, 18 Mar 2011 16:56:43 +0100 Subject: Change gitfo_prettify_dir_path() and gitfo_prettify_file_path() behavior Those functions now return prettified rooted path. --- src/fileops.c | 60 ++++++++++++++++++++++++++++++++++++++------------------ src/fileops.h | 7 +++++-- src/repository.c | 10 +++++----- 3 files changed, 51 insertions(+), 26 deletions(-) (limited to 'src') diff --git a/src/fileops.c b/src/fileops.c index 4ed53bc97..41239091b 100644 --- a/src/fileops.c +++ b/src/fileops.c @@ -443,22 +443,29 @@ int gitfo_dirent( return GIT_SUCCESS; } -#ifdef GIT_WIN32 -static int is_windows_rooted_path(const char *path) +int retrieve_path_root_offset(const char *path) { + int offset = 0; + +#ifdef GIT_WIN32 + /* Does the root of the path look like a windows drive ? */ if (isalpha(path[0]) && (path[1] == ':')) - return GIT_SUCCESS; + offset += 2; + +#endif + + if (*(path + offset) == '/') + return offset; return GIT_ERROR; } -#endif int gitfo_mkdir_recurs(const char *path, int mode) { - int error; + int error, root_path_offset; char *pp, *sp; char *path_copy = git__strdup(path); @@ -468,12 +475,9 @@ int gitfo_mkdir_recurs(const char *path, int mode) error = GIT_SUCCESS; pp = path_copy; -#ifdef GIT_WIN32 - - if (!is_windows_rooted_path(pp)) - pp += 2; /* Skip the drive name (eg. C: or D:) */ - -#endif + root_path_offset = retrieve_path_root_offset(pp); + if (root_path_offset > 0) + pp += root_path_offset; /* On Windows, will skip the drive name (eg. C: or D:) */ while (error == GIT_SUCCESS && (sp = strchr(pp, '/')) != 0) { if (sp != pp && gitfo_isdir(path_copy) < GIT_SUCCESS) { @@ -504,6 +508,8 @@ static int retrieve_previous_path_component_start(const char *path) len = strlen(path); offset = len - 1; + //TODO: Deal with Windows rooted path + /* Skip leading slash */ if (path[start] == '/') start++; @@ -522,15 +528,25 @@ static int retrieve_previous_path_component_start(const char *path) return offset; } -int gitfo_prettify_dir_path(char *buffer_out, const char *path) +int gitfo_prettify_dir_path(char *buffer_out, size_t size, const char *path) { - int len = 0, segment_len, only_dots; + int len = 0, segment_len, only_dots, root_path_offset, error = GIT_SUCCESS; char *current; const char *buffer_out_start, *buffer_end; - buffer_out_start = buffer_out; current = (char *)path; buffer_end = path + strlen(path); + buffer_out_start = buffer_out; + + root_path_offset = retrieve_path_root_offset(path); + if (root_path_offset < 0) { + error = gitfo_getcwd(buffer_out, size); + if (error < GIT_SUCCESS) + return error; + + len = strlen(buffer_out); + buffer_out += len; + } while (current < buffer_end) { /* Prevent multiple slashes from being added to the output */ @@ -543,7 +559,7 @@ int gitfo_prettify_dir_path(char *buffer_out, const char *path) segment_len = 0; /* Copy path segment to the output */ - while (current < buffer_end && *current !='/') + while (current < buffer_end && *current != '/') { only_dots &= (*current == '.'); *buffer_out++ = *current++; @@ -568,7 +584,9 @@ int gitfo_prettify_dir_path(char *buffer_out, const char *path) *buffer_out ='\0'; len = retrieve_previous_path_component_start(buffer_out_start); - if (len < GIT_SUCCESS) + + /* Are we escaping out of the root dir? */ + if (len < 0) return GIT_EINVALIDPATH; buffer_out = (char *)buffer_out_start + len; @@ -576,7 +594,7 @@ int gitfo_prettify_dir_path(char *buffer_out, const char *path) } /* Guard against potential multiple dot path traversal (cf http://cwe.mitre.org/data/definitions/33.html) */ - if (only_dots &&segment_len > 0) + if (only_dots && segment_len > 0) return GIT_EINVALIDPATH; *buffer_out++ = '/'; @@ -588,20 +606,24 @@ int gitfo_prettify_dir_path(char *buffer_out, const char *path) return GIT_SUCCESS; } -int gitfo_prettify_file_path(char *buffer_out, const char *path) +int gitfo_prettify_file_path(char *buffer_out, size_t size, const char *path) { int error, path_len, i; const char* pattern = "/.."; path_len = strlen(path); + /* Let's make sure the filename isn't empty nor a dot */ + if (path_len == 0 || (path_len == 1 && *path == '.')) + return GIT_EINVALIDPATH; + /* Let's make sure the filename doesn't end with "/", "/." or "/.." */ for (i = 1; path_len > i && i < 4; i++) { if (!strncmp(path + path_len - i, pattern, i)) return GIT_EINVALIDPATH; } - error = gitfo_prettify_dir_path(buffer_out, path); + error = gitfo_prettify_dir_path(buffer_out, size, path); if (error < GIT_SUCCESS) return error; diff --git a/src/fileops.h b/src/fileops.h index c2c038b26..ce236f608 100644 --- a/src/fileops.h +++ b/src/fileops.h @@ -163,12 +163,13 @@ extern int gitfo_getcwd(char *buffer_out, size_t size); * the file system perspective. * * @param buffer_out buffer to populate with the normalized path. + * @param size buffer size. * @param path directory path to clean up. * @return * - GIT_SUCCESS on success; * - GIT_ERROR when the input path is invalid or escapes the current directory. */ -int gitfo_prettify_dir_path(char *buffer_out, const char *path); +int gitfo_prettify_dir_path(char *buffer_out, size_t size, const char *path); /** * Clean up a provided absolute or relative file path. @@ -185,11 +186,13 @@ int gitfo_prettify_dir_path(char *buffer_out, const char *path); * the file system perspective. * * @param buffer_out buffer to populate with the normalized path. + * @param size buffer size. * @param path file path to clean up. * @return * - GIT_SUCCESS on success; * - GIT_ERROR when the input path is invalid or escapes the current directory. */ -int gitfo_prettify_file_path(char *buffer_out, const char *path); +int gitfo_prettify_file_path(char *buffer_out, size_t size, const char *path); +int retrieve_path_root_offset(const char *path); #endif /* INCLUDE_fileops_h__ */ diff --git a/src/repository.c b/src/repository.c index 132969402..91b95a881 100644 --- a/src/repository.c +++ b/src/repository.c @@ -66,7 +66,7 @@ static int assign_repository_dirs( if (git_dir == NULL) return GIT_ENOTFOUND; - error = gitfo_prettify_dir_path(path_aux, git_dir); + error = gitfo_prettify_dir_path(path_aux, sizeof(path_aux), git_dir); if (error < GIT_SUCCESS) return error; @@ -81,7 +81,7 @@ static int assign_repository_dirs( if (git_object_directory == NULL) git__joinpath(path_aux, repo->path_repository, GIT_OBJECTS_DIR); else { - error = gitfo_prettify_dir_path(path_aux, git_object_directory); + error = gitfo_prettify_dir_path(path_aux, sizeof(path_aux), git_object_directory); if (error < GIT_SUCCESS) return error; } @@ -95,7 +95,7 @@ static int assign_repository_dirs( if (git_work_tree == NULL) repo->is_bare = 1; else { - error = gitfo_prettify_dir_path(path_aux, git_work_tree); + error = gitfo_prettify_dir_path(path_aux, sizeof(path_aux), git_work_tree); if (error < GIT_SUCCESS) return error; @@ -108,7 +108,7 @@ static int assign_repository_dirs( if (git_index_file == NULL) git__joinpath(path_aux, repo->path_repository, GIT_INDEX_FILE); else { - error = gitfo_prettify_file_path(path_aux, git_index_file); + error = gitfo_prettify_file_path(path_aux, sizeof(path_aux), git_index_file); if (error < GIT_SUCCESS) return error; } @@ -403,7 +403,7 @@ static int repo_init_find_dir(repo_init *results, const char* path) char temp_path[GIT_PATH_MAX]; int error = GIT_SUCCESS; - error = gitfo_prettify_dir_path(temp_path, path); + error = gitfo_prettify_dir_path(temp_path, sizeof(temp_path), path); if (error < GIT_SUCCESS) return error; -- cgit v1.2.1 From 3644e98fd984c039d0c84b2311b410583739b6a5 Mon Sep 17 00:00:00 2001 From: nulltoken Date: Fri, 18 Mar 2011 19:10:36 +0100 Subject: Fix detection of attempt to escape the root directory on Windows --- src/fileops.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) (limited to 'src') diff --git a/src/fileops.c b/src/fileops.c index 41239091b..e440ea6ab 100644 --- a/src/fileops.c +++ b/src/fileops.c @@ -503,13 +503,15 @@ int gitfo_mkdir_recurs(const char *path, int mode) static int retrieve_previous_path_component_start(const char *path) { - int offset, len, start = 0; - + int offset, len, root_offset, start = 0; + + root_offset = retrieve_path_root_offset(path); + if (root_offset > -1) + start += root_offset; + len = strlen(path); offset = len - 1; - //TODO: Deal with Windows rooted path - /* Skip leading slash */ if (path[start] == '/') start++; @@ -518,7 +520,7 @@ static int retrieve_previous_path_component_start(const char *path) if (path[offset] == '/') offset--; - if (offset < 0) + if (offset < root_offset) return GIT_ERROR; while (offset > start && path[offset-1] != '/') { -- cgit v1.2.1 From 2b861d7589467734c2371c4daaab6e540857b988 Mon Sep 17 00:00:00 2001 From: nulltoken Date: Sat, 19 Mar 2011 08:31:40 +0100 Subject: Add detection of incorrect usage to git__joinpath() --- src/util.c | 3 +++ 1 file changed, 3 insertions(+) (limited to 'src') diff --git a/src/util.c b/src/util.c index bfc4f7b27..995daf314 100644 --- a/src/util.c +++ b/src/util.c @@ -223,6 +223,9 @@ void git__joinpath_n(char *buffer_out, int count, ...) int len; path = va_arg(ap, const char *); + + assert((i == 0) || path != buffer_start); + if (i > 0 && *path == '/' && buffer_out > buffer_start && buffer_out[-1] == '/') path++; -- cgit v1.2.1 From 7c80c19e1dffb4421f91913bc79b9cb7596634a4 Mon Sep 17 00:00:00 2001 From: Vicent Marti Date: Wed, 23 Mar 2011 01:58:18 +0200 Subject: Fix compilation in MinGW --- src/fileops.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/fileops.c b/src/fileops.c index e440ea6ab..d754c49ee 100644 --- a/src/fileops.c +++ b/src/fileops.c @@ -32,7 +32,7 @@ static int creat_tempfile(char *path_out, const char *tmp_dir, const char *filen git__joinpath(path_out, tmp_dir, filename); strcat(path_out, "_git2_XXXXXX"); -#ifdef GIT_WIN32 +#if defined(_MSC_VER) /* FIXME: there may be race conditions when multi-threading * with the library */ if (_mktemp_s(path_out, GIT_PATH_MAX) != 0) -- cgit v1.2.1 From f6f72d7ef8091bf1fcf19f284e1db62a43f93381 Mon Sep 17 00:00:00 2001 From: Vicent Marti Date: Wed, 23 Mar 2011 18:44:53 +0200 Subject: Improve the ODB writing backend Temporary files when doing streaming writes are now stored inside the Objects folder, to prevent issues when moving files between disks/partitions. Add support for block writes to the ODB again (for those backends that cannot implement streaming). --- src/filebuf.c | 11 ++-------- src/fileops.c | 64 ++------------------------------------------------------- src/fileops.h | 2 +- src/odb.c | 35 +++++++++++++++++++++++++++++++ src/odb_loose.c | 6 ++++-- 5 files changed, 44 insertions(+), 74 deletions(-) (limited to 'src') diff --git a/src/filebuf.c b/src/filebuf.c index 607ad618d..dff9373f6 100644 --- a/src/filebuf.c +++ b/src/filebuf.c @@ -151,8 +151,7 @@ int git_filebuf_open(git_filebuf *file, const char *path, int flags) int error; size_t path_len; - if (file == NULL) - return GIT_ERROR; + assert(file && path); memset(file, 0x0, sizeof(git_filebuf)); @@ -203,7 +202,7 @@ int git_filebuf_open(git_filebuf *file, const char *path, int flags) char tmp_path[GIT_PATH_MAX]; /* Open the file as temporary for locking */ - file->fd = gitfo_creat_tmp(tmp_path, "_filebuf_"); + file->fd = gitfo_mktemp(tmp_path, path); if (file->fd < 0) { error = GIT_EOSERR; goto cleanup; @@ -218,12 +217,6 @@ int git_filebuf_open(git_filebuf *file, const char *path, int flags) goto cleanup; } } else { - /* If the file is not temporary, make sure we have a path */ - if (path == NULL) { - error = GIT_ERROR; - goto cleanup; - } - path_len = strlen(path); /* Save the original path of the file */ diff --git a/src/fileops.c b/src/fileops.c index d754c49ee..5dd4a3806 100644 --- a/src/fileops.c +++ b/src/fileops.c @@ -25,11 +25,11 @@ int gitfo_mkdir_2file(const char *file_path) return GIT_SUCCESS; } -static int creat_tempfile(char *path_out, const char *tmp_dir, const char *filename) +int gitfo_mktemp(char *path_out, const char *filename) { int fd; - git__joinpath(path_out, tmp_dir, filename); + strcpy(path_out, filename); strcat(path_out, "_git2_XXXXXX"); #if defined(_MSC_VER) @@ -46,66 +46,6 @@ static int creat_tempfile(char *path_out, const char *tmp_dir, const char *filen return fd >= 0 ? fd : GIT_EOSERR; } -static const char *find_tmpdir(void) -{ - static int tmpdir_not_found = 0; - static char temp_dir[GIT_PATH_MAX]; - static const char *env_vars[] = { - "TEMP", "TMP", "TMPDIR" - }; - - unsigned int i, j; - char test_file[GIT_PATH_MAX]; - - if (tmpdir_not_found) - return NULL; - - if (temp_dir[0] != '\0') - return temp_dir; - - for (i = 0; i < ARRAY_SIZE(env_vars); ++i) { - char *env_path; - - env_path = getenv(env_vars[i]); - if (env_path == NULL) - continue; - - strcpy(temp_dir, env_path); - - /* Fix backslashes because Windows environment vars - * are probably fucked up */ - for (j = 0; j < strlen(temp_dir); ++j) - if (temp_dir[j] == '\\') - temp_dir[j] = '/'; - - if (creat_tempfile(test_file, temp_dir, "writetest") >= 0) { - gitfo_unlink(test_file); - return temp_dir; - } - } - - /* last resort: current folder. */ - strcpy(temp_dir, "./"); - if (creat_tempfile(test_file, temp_dir, "writetest") >= 0) { - gitfo_unlink(test_file); - return temp_dir; - } - - tmpdir_not_found = 1; - return NULL; -} - -int gitfo_creat_tmp(char *path_out, const char *filename) -{ - const char *tmp_dir; - - tmp_dir = find_tmpdir(); - if (tmp_dir == NULL) - return GIT_EOSERR; - - return creat_tempfile(path_out, tmp_dir, filename); -} - int gitfo_open(const char *path, int flags) { int fd = open(path, flags | O_BINARY); diff --git a/src/fileops.h b/src/fileops.h index ce236f608..6e0fd9d14 100644 --- a/src/fileops.h +++ b/src/fileops.h @@ -58,7 +58,7 @@ extern int gitfo_exists(const char *path); extern int gitfo_open(const char *path, int flags); extern int gitfo_creat(const char *path, int mode); extern int gitfo_creat_force(const char *path, int mode); -extern int gitfo_creat_tmp(char *path_out, const char *filename); +extern int gitfo_mktemp(char *path_out, const char *filename); extern int gitfo_isdir(const char *path); extern int gitfo_mkdir_recurs(const char *path, int mode); extern int gitfo_mkdir_2file(const char *path); diff --git a/src/odb.c b/src/odb.c index d825fd95c..33d5468d9 100644 --- a/src/odb.c +++ b/src/odb.c @@ -415,6 +415,41 @@ int git_odb_read(git_odb_object **out, git_odb *db, const git_oid *id) return error; } +int git_odb_write(git_oid *oid, git_odb *db, const void *data, size_t len, git_otype type) +{ + unsigned int i; + int error = GIT_ERROR; + + assert(oid && db); + + for (i = 0; i < db->backends.length && error < 0; ++i) { + backend_internal *internal = git_vector_get(&db->backends, i); + git_odb_backend *b = internal->backend; + + /* we don't write in alternates! */ + if (internal->is_alternate) + continue; + + if (b->write != NULL) + error = b->write(oid, b, data, len, type); + } + + /* if no backends were able to write the object directly, we try a streaming + * write to the backends; just write the whole object into the stream in one + * push */ + if (error < GIT_SUCCESS) { + git_odb_stream *stream; + + if ((error = git_odb_open_wstream(&stream, db, len, type)) == GIT_SUCCESS) { + stream->write(stream, data, len); + error = stream->finalize_write(oid, stream); + stream->free(stream); + } + } + + return error; +} + int git_odb_open_wstream(git_odb_stream **stream, git_odb *db, size_t size, git_otype type) { unsigned int i; diff --git a/src/odb_loose.c b/src/odb_loose.c index 4ab1128f3..8ee01cd2c 100644 --- a/src/odb_loose.c +++ b/src/odb_loose.c @@ -579,7 +579,7 @@ int loose_backend__stream(git_odb_stream **stream_out, git_odb_backend *_backend loose_backend *backend; loose_writestream *stream; - char hdr[64]; + char hdr[64], tmp_path[GIT_PATH_MAX]; int hdrlen; int error; @@ -603,7 +603,9 @@ int loose_backend__stream(git_odb_stream **stream_out, git_odb_backend *_backend stream->stream.free = &loose_backend__stream_free; stream->stream.mode = GIT_STREAM_WRONLY; - error = git_filebuf_open(&stream->fbuf, NULL, + git__joinpath(tmp_path, backend->objects_dir, "tmp_object"); + + error = git_filebuf_open(&stream->fbuf, tmp_path, GIT_FILEBUF_HASH_CONTENTS | GIT_FILEBUF_DEFLATE_CONTENTS | GIT_FILEBUF_TEMPORARY); -- cgit v1.2.1 From 3bdc0d4c8c3a51e6b338c8166d98d5b82cb01a43 Mon Sep 17 00:00:00 2001 From: Jakob Pfender Date: Thu, 24 Mar 2011 15:32:24 +0100 Subject: index.c: Read index after initialization The current behaviour of git_index_open{bare,inrepo}() is unexpected. When an index is opened, an in-memory index object is created that is linked to the index discovered by git_repository_open(). However, this index object is empty, as the on-disk index is not read. To fully open the on-disk index file, git_index_read() has to be called. This leads to confusing behaviour. Consider the following code: git_index *idx; git_index_open_inrepo(&idx, repo); git_index_write(idx); You would expect this to have no effect, as the index is never ostensibly manipulated. However, what actually happens is that the index entries are removed from the on-disk index because the empty in-memory index object created by open_inrepo() is written back to the disk. This patch reads the index after opening it. --- src/index.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/index.c b/src/index.c index 34208731c..7e4d6bbc7 100644 --- a/src/index.c +++ b/src/index.c @@ -148,7 +148,7 @@ static int index_initialize(git_index **index_out, git_repository *owner, const index->on_disk = 1; *index_out = index; - return GIT_SUCCESS; + return git_index_read(index); } int git_index_open_bare(git_index **index_out, const char *index_path) -- cgit v1.2.1 From 09e8de0f38ed8f28c02e5a3fa987cbb74e98a671 Mon Sep 17 00:00:00 2001 From: Vicent Marti Date: Fri, 25 Mar 2011 23:53:38 +0200 Subject: New external API method: `git_reference_listcb` List all the references in the repository, calling a custom callback for each one. The listed references may be filtered by type, or using a bitwise OR of several types. Use the magic value `GIT_REF_LISTALL` to obtain all references, including packed ones. The `callback` function will be called for each of the references in the repository, and will receive the name of the reference and the `payload` value passed to this method. --- src/refs.c | 66 ++++++++++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 45 insertions(+), 21 deletions(-) (limited to 'src') diff --git a/src/refs.c b/src/refs.c index 8e2965528..16bd74149 100644 --- a/src/refs.c +++ b/src/refs.c @@ -606,10 +606,12 @@ cleanup: struct dirent_list_data { - git_vector ref_list; git_repository *repo; size_t repo_path_len; unsigned int list_flags; + + int (*callback)(const char *, void *); + void *callback_payload; }; static int _dirent_loose_listall(void *_data, char *full_path) @@ -625,10 +627,12 @@ static int _dirent_loose_listall(void *_data, char *full_path) git_hashtable_lookup(data->repo->references.packfile, file_path) != NULL) return GIT_SUCCESS; - if ((data->list_flags & loose_guess_rtype(full_path)) == 0) - return GIT_SUCCESS; /* we are filtering out this reference */ + if (data->list_flags != GIT_REF_LISTALL) { + if ((data->list_flags & loose_guess_rtype(full_path)) == 0) + return GIT_SUCCESS; /* we are filtering out this reference */ + } - return git_vector_insert(&data->ref_list, git__strdup(file_path)); + return data->callback(file_path, data->callback_payload); } static int _dirent_loose_load(void *data, char *full_path) @@ -1402,47 +1406,67 @@ int git_reference_packall(git_repository *repo) return packed_write(repo); } -int git_reference_listall(git_strarray *array, git_repository *repo, unsigned int list_flags) +int git_reference_listcb(git_repository *repo, unsigned int list_flags, int (*callback)(const char *, void *), void *payload) { int error; struct dirent_list_data data; char refs_path[GIT_PATH_MAX]; - array->strings = NULL; - array->count = 0; - - git_vector_init(&data.ref_list, 8, NULL); - data.repo_path_len = strlen(repo->path_repository); - data.list_flags = list_flags; - data.repo = repo; - /* list all the packed references first */ if (list_flags & GIT_REF_PACKED) { const char *ref_name; void *_unused; - if ((error = packed_load(repo)) < GIT_SUCCESS) { - git_vector_free(&data.ref_list); + if ((error = packed_load(repo)) < GIT_SUCCESS) return error; - } GIT_HASHTABLE_FOREACH(repo->references.packfile, ref_name, _unused, - git_vector_insert(&data.ref_list, git__strdup(ref_name)); + if ((error = callback(ref_name, payload)) < GIT_SUCCESS) + return error; ); } /* now list the loose references, trying not to * duplicate the ref names already in the packed-refs file */ + + data.repo_path_len = strlen(repo->path_repository); + data.list_flags = list_flags; + data.repo = repo; + data.callback = callback; + data.callback_payload = payload; + + git__joinpath(refs_path, repo->path_repository, GIT_REFS_DIR); - error = gitfo_dirent(refs_path, GIT_PATH_MAX, _dirent_loose_listall, &data); + return gitfo_dirent(refs_path, GIT_PATH_MAX, _dirent_loose_listall, &data); +} + +int cb__reflist_add(const char *ref, void *data) +{ + return git_vector_insert((git_vector *)data, git__strdup(ref)); +} + +int git_reference_listall(git_strarray *array, git_repository *repo, unsigned int list_flags) +{ + int error; + git_vector ref_list; + + assert(array && repo); + + array->strings = NULL; + array->count = 0; + + if (git_vector_init(&ref_list, 8, NULL) < GIT_SUCCESS) + return GIT_ENOMEM; + + error = git_reference_listcb(repo, list_flags, &cb__reflist_add, (void *)&ref_list); if (error < GIT_SUCCESS) { - git_vector_free(&data.ref_list); + git_vector_free(&ref_list); return error; } - array->strings = (char **)data.ref_list.contents; - array->count = data.ref_list.length; + array->strings = (char **)ref_list.contents; + array->count = ref_list.length; return GIT_SUCCESS; } -- cgit v1.2.1 From 683581a3e91032d9fdc203933d691b9353f24492 Mon Sep 17 00:00:00 2001 From: schu Date: Mon, 28 Mar 2011 17:59:13 +0200 Subject: index.c: Fix tiny typos --- src/index.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/index.c b/src/index.c index 7e4d6bbc7..6a31dd5cb 100644 --- a/src/index.c +++ b/src/index.c @@ -74,7 +74,7 @@ struct entry_short { uint32_t file_size; git_oid oid; uint16_t flags; - char path[1]; /* arbritrary length */ + char path[1]; /* arbitrary length */ }; struct entry_long { @@ -89,7 +89,7 @@ struct entry_long { git_oid oid; uint16_t flags; uint16_t flags_extended; - char path[1]; /* arbritrary length */ + char path[1]; /* arbitrary length */ }; /* local declarations */ -- cgit v1.2.1 From 483526ebec8b50682b6edd3e1db92df7569ca7b0 Mon Sep 17 00:00:00 2001 From: Vicent Marti Date: Mon, 28 Mar 2011 22:23:44 +0300 Subject: Update the SQLite backend --- src/backends/sqlite.c | 38 +++++++++++++++++--------------------- 1 file changed, 17 insertions(+), 21 deletions(-) (limited to 'src') diff --git a/src/backends/sqlite.c b/src/backends/sqlite.c index 72d7b4d8e..a4c6d4825 100644 --- a/src/backends/sqlite.c +++ b/src/backends/sqlite.c @@ -44,21 +44,20 @@ typedef struct { sqlite3_stmt *st_read_header; } sqlite_backend; -int sqlite_backend__read_header(git_rawobj *obj, git_odb_backend *_backend, const git_oid *oid) +int sqlite_backend__read_header(size_t *len_p, git_otype *type_p, git_odb_backend *_backend, const git_oid *oid) { sqlite_backend *backend; int error; - assert(obj && _backend && oid); + assert(len_p && type_p && _backend && oid); backend = (sqlite_backend *)_backend; error = GIT_ERROR; - obj->data = NULL; if (sqlite3_bind_text(backend->st_read_header, 1, (char *)oid->id, 20, SQLITE_TRANSIENT) == SQLITE_OK) { if (sqlite3_step(backend->st_read_header) == SQLITE_ROW) { - obj->type = sqlite3_column_int(backend->st_read_header, 0); - obj->len = sqlite3_column_int(backend->st_read_header, 1); + *type_p = (git_otype)sqlite3_column_int(backend->st_read_header, 0); + *len_p = (size_t)sqlite3_column_int(backend->st_read_header, 1); assert(sqlite3_step(backend->st_read_header) == SQLITE_DONE); error = GIT_SUCCESS; } else { @@ -71,26 +70,26 @@ int sqlite_backend__read_header(git_rawobj *obj, git_odb_backend *_backend, cons } -int sqlite_backend__read(git_rawobj *obj, git_odb_backend *_backend, const git_oid *oid) +int sqlite_backend__read(void **data_p, size_t *len_p, git_otype *type_p, git_odb_backend *_backend, const git_oid *oid) { sqlite_backend *backend; int error; - assert(obj && _backend && oid); + assert(data_p && len_p && type_p && _backend && oid); backend = (sqlite_backend *)_backend; error = GIT_ERROR; if (sqlite3_bind_text(backend->st_read, 1, (char *)oid->id, 20, SQLITE_TRANSIENT) == SQLITE_OK) { if (sqlite3_step(backend->st_read) == SQLITE_ROW) { - obj->type = sqlite3_column_int(backend->st_read, 0); - obj->len = sqlite3_column_int(backend->st_read, 1); - obj->data = git__malloc(obj->len); + *type_p = (git_otype)sqlite3_column_int(backend->st_read, 0); + *len_p = (size_t)sqlite3_column_int(backend->st_read, 1); + *data_p = git__malloc(*len_p); - if (obj->data == NULL) { + if (*data_p == NULL) { error = GIT_ENOMEM; } else { - memcpy(obj->data, sqlite3_column_blob(backend->st_read, 2), obj->len); + memcpy(*data_p, sqlite3_column_blob(backend->st_read, 2), *len_p); error = GIT_SUCCESS; } @@ -126,27 +125,24 @@ int sqlite_backend__exists(git_odb_backend *_backend, const git_oid *oid) } -int sqlite_backend__write(git_oid *id, git_odb_backend *_backend, git_rawobj *obj) +int sqlite_backend__write(git_oid *id, git_odb_backend *_backend, const void *data, size_t len, git_otype type) { - char hdr[64]; - int hdrlen; - int error; sqlite_backend *backend; - assert(id && _backend && obj); + assert(id && _backend && data); backend = (sqlite_backend *)_backend; - if ((error = git_odb__hash_obj(id, hdr, sizeof(hdr), &hdrlen, obj)) < 0) + if ((error = git_odb_hash(id, data, len, type)) < 0) return error; error = SQLITE_ERROR; if (sqlite3_bind_text(backend->st_write, 1, (char *)id->id, 20, SQLITE_TRANSIENT) == SQLITE_OK && - sqlite3_bind_int(backend->st_write, 2, (int)obj->type) == SQLITE_OK && - sqlite3_bind_int(backend->st_write, 3, obj->len) == SQLITE_OK && - sqlite3_bind_blob(backend->st_write, 4, obj->data, obj->len, SQLITE_TRANSIENT) == SQLITE_OK) { + sqlite3_bind_int(backend->st_write, 2, (int)type) == SQLITE_OK && + sqlite3_bind_int(backend->st_write, 3, len) == SQLITE_OK && + sqlite3_bind_blob(backend->st_write, 4, data, len, SQLITE_TRANSIENT) == SQLITE_OK) { error = sqlite3_step(backend->st_write); } -- cgit v1.2.1 From baad182cbcafb2f7332ad5864695a42a0354f684 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Mart=C3=ADn=20Nieto?= Date: Mon, 28 Mar 2011 11:31:58 +0200 Subject: Add GIT_EEXISTS error code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Carlos Martín Nieto --- src/errors.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/errors.c b/src/errors.c index f6b964837..3c0e8eb1f 100644 --- a/src/errors.c +++ b/src/errors.c @@ -28,7 +28,8 @@ static struct { {GIT_EINVALIDPATH, "The path is invalid" }, {GIT_EREVWALKOVER, "The revision walker is empty; there are no more commits left to iterate"}, {GIT_EINVALIDREFSTATE, "The state of the reference is not valid"}, - {GIT_ENOTIMPLEMENTED, "This feature has not been implemented yet"} + {GIT_ENOTIMPLEMENTED, "This feature has not been implemented yet"}, + {GIT_EEXISTS, "A reference with this name already exists"} }; const char *git_strerror(int num) -- cgit v1.2.1 From fa204962006541bd9c111bd7f3fa37ed2f8a2d7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Mart=C3=ADn=20Nieto?= Date: Mon, 28 Mar 2011 12:00:50 +0200 Subject: Allow forcing the creation or renaming of references MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add internal reference create and rename functions which take a force parameter, telling them to overwrite an existing reference if it exists. These functions try to update the reference if it's of the same type as the one it's going to be replaced by. Otherwise the old reference becomes invalid. Signed-off-by: Carlos Martín Nieto --- src/refs.c | 412 ++++++++++++++++++++++++++++++++++++------------------------- 1 file changed, 245 insertions(+), 167 deletions(-) (limited to 'src') diff --git a/src/refs.c b/src/refs.c index 16bd74149..20c1d001f 100644 --- a/src/refs.c +++ b/src/refs.c @@ -915,8 +915,238 @@ cleanup: return error; } +/***************************************** + * Internal methods - reference creation + *****************************************/ + +int git_reference_create_symbolic_internal(git_reference **ref_out, git_repository *repo, const char *name, const char *target, int force) +{ + char normalized[MAX_GITDIR_TREE_STRUCTURE_PATH_LENGTH]; + int error = GIT_SUCCESS, updated = 0; + git_reference *ref = NULL, *old_ref = NULL; + + if (git_reference_lookup(&ref, repo, name) == GIT_SUCCESS && !force) + return GIT_EEXISTS; + + /* + * If they old ref was of the same type, then we can just update + * it (once we've checked that the target is valid). Otherwise we + * need a new reference because we can't make a symbolic ref out + * of an oid one. + * If if didn't exist, then we need to create a new one anyway. + */ + if (ref && ref->type & GIT_REF_SYMBOLIC){ + updated = 1; + } else { + ref = NULL; + error = reference_create(&ref, repo, name, GIT_REF_SYMBOLIC); + if (error < GIT_SUCCESS) + goto cleanup; + } + + /* The target can aither be the name of an object id reference or the name of another symbolic reference */ + error = normalize_name(normalized, target, 0); + if (error < GIT_SUCCESS) + goto cleanup; + + /* set the target; this will write the reference on disk */ + error = git_reference_set_target(ref, normalized); + if (error < GIT_SUCCESS) + goto cleanup; + + /* + * If we didn't update the ref, then we need to insert or replace + * it in the loose cache. If we replaced a ref, free it. + */ + if (!updated){ + error = git_hashtable_insert2(repo->references.loose_cache, ref->name, ref, (void **) &old_ref); + if (error < GIT_SUCCESS) + goto cleanup; + + if(old_ref) + reference_free(old_ref); + } + + *ref_out = ref; + + return error; + +cleanup: + reference_free(ref); + return error; +} + +int git_reference_create_oid_internal(git_reference **ref_out, git_repository *repo, const char *name, const git_oid *id, int force) +{ + int error = GIT_SUCCESS, updated = 0; + git_reference *ref = NULL, *old_ref = NULL; + + if(git_reference_lookup(&ref, repo, name) == GIT_SUCCESS && !force) + return GIT_EEXISTS; + + /* + * If they old ref was of the same type, then we can just update + * it (once we've checked that the target is valid). Otherwise we + * need a new reference because we can't make a symbolic ref out + * of an oid one. + * If if didn't exist, then we need to create a new one anyway. + */ + if (ref && ref-> type & GIT_REF_OID){ + updated = 1; + } else { + ref = NULL; + error = reference_create(&ref, repo, name, GIT_REF_OID); + if (error < GIT_SUCCESS) + goto cleanup; + } + + /* set the oid; this will write the reference on disk */ + error = git_reference_set_oid(ref, id); + if (error < GIT_SUCCESS) + goto cleanup; + + if(!updated){ + error = git_hashtable_insert2(repo->references.loose_cache, ref->name, ref, (void **) &old_ref); + if (error < GIT_SUCCESS) + goto cleanup; + + if(old_ref) + reference_free(old_ref); + } + + *ref_out = ref; + + return error; + +cleanup: + reference_free(ref); + return error; +} + +/* + * Rename a reference + * + * If the reference is packed, we need to rewrite the + * packfile to remove the reference from it and create + * the reference back as a loose one. + * + * If the reference is loose, we just rename it on + * the filesystem. + * + * We also need to re-insert the reference on its corresponding + * in-memory cache, since the caches are indexed by refname. + */ +int git_reference_rename_internal(git_reference *ref, const char *new_name, int force) +{ + int error; + char *old_name; + char old_path[GIT_PATH_MAX], new_path[GIT_PATH_MAX], normalized_name[MAX_GITDIR_TREE_STRUCTURE_PATH_LENGTH]; + git_reference *looked_up_ref, *old_ref = NULL; + + assert(ref); + + /* Ensure the name is valid */ + error = normalize_name(normalized_name, new_name, ref->type & GIT_REF_OID); + if (error < GIT_SUCCESS) + return error; + + /* Ensure we're not going to overwrite an existing reference */ + error = git_reference_lookup(&looked_up_ref, ref->owner, new_name); + if (error == GIT_SUCCESS && !force) + return GIT_EEXISTS; + + if (error != GIT_ENOTFOUND) + return error; + + + old_name = ref->name; + ref->name = git__strdup(new_name); + + if (ref->name == NULL) { + ref->name = old_name; + return GIT_ENOMEM; + } + + if (ref->type & GIT_REF_PACKED) { + /* write the packfile to disk; note + * that the state of the in-memory cache is not + * consistent, because the reference is indexed + * by its old name but it already has the new one. + * This doesn't affect writing, though, and allows + * us to rollback if writing fails + */ + + ref->type &= ~GIT_REF_PACKED; + + /* Create the loose ref under its new name */ + error = loose_write(ref); + if (error < GIT_SUCCESS) { + ref->type |= GIT_REF_PACKED; + goto cleanup; + } + + /* Remove from the packfile cache in order to avoid packing it back + * Note : we do not rely on git_reference_delete() because this would + * invalidate the reference. + */ + git_hashtable_remove(ref->owner->references.packfile, old_name); + + /* Recreate the packed-refs file without the reference */ + error = packed_write(ref->owner); + if (error < GIT_SUCCESS) + goto rename_loose_to_old_name; + + } else { + git__joinpath(old_path, ref->owner->path_repository, old_name); + git__joinpath(new_path, ref->owner->path_repository, ref->name); + + error = gitfo_mv_force(old_path, new_path); + if (error < GIT_SUCCESS) + goto cleanup; + + /* Once succesfully renamed, remove from the cache the reference known by its old name*/ + git_hashtable_remove(ref->owner->references.loose_cache, old_name); + } + + /* Store the renamed reference into the loose ref cache */ + error = git_hashtable_insert2(ref->owner->references.loose_cache, ref->name, ref, (void **) &old_ref); + + /* If we force-replaced, we need to free the old reference */ + if(old_ref) + reference_free(old_ref); + + free(old_name); + return error; + +cleanup: + /* restore the old name if this failed */ + free(ref->name); + ref->name = old_name; + return error; + +rename_loose_to_old_name: + /* If we hit this point. Something *bad* happened! Think "Ghostbusters + * crossing the streams" definition of bad. + * Either the packed-refs has been correctly generated and something else + * has gone wrong, or the writing of the new packed-refs has failed, and + * we're stuck with the old one. As a loose ref always takes priority over + * a packed ref, we'll eventually try and rename the generated loose ref to + * its former name. It even that fails, well... we might have lost the reference + * for good. :-/ + */ + + git__joinpath(old_path, ref->owner->path_repository, ref->name); + git__joinpath(new_path, ref->owner->path_repository, old_name); + /* No error checking. We'll return the initial error */ + gitfo_mv_force(old_path, new_path); + /* restore the old name */ + free(ref->name); + ref->name = old_name; + + return error; +} /***************************************** * External Library API @@ -975,64 +1205,23 @@ int git_reference_lookup(git_reference **ref_out, git_repository *repo, const ch int git_reference_create_symbolic(git_reference **ref_out, git_repository *repo, const char *name, const char *target) { - char normalized[MAX_GITDIR_TREE_STRUCTURE_PATH_LENGTH]; - int error = GIT_SUCCESS; - git_reference *ref = NULL; - - error = reference_create(&ref, repo, name, GIT_REF_SYMBOLIC); - if (error < GIT_SUCCESS) - goto cleanup; - - /* The target can aither be the name of an object id reference or the name of another symbolic reference */ - error = normalize_name(normalized, target, 0); - if (error < GIT_SUCCESS) - goto cleanup; - - /* set the target; this will write the reference on disk */ - error = git_reference_set_target(ref, normalized); - if (error < GIT_SUCCESS) - goto cleanup; - - error = git_hashtable_insert(repo->references.loose_cache, ref->name, ref); - if (error < GIT_SUCCESS) - goto cleanup; - - *ref_out = ref; - - return error; + return git_reference_create_symbolic_internal(ref_out, repo, name, target, 0); +} -cleanup: - reference_free(ref); - return error; +int git_reference_create_symbolic_force(git_reference **ref_out, git_repository *repo, const char *name, const char *target) +{ + return git_reference_create_symbolic_internal(ref_out, repo, name, target, 1); } int git_reference_create_oid(git_reference **ref_out, git_repository *repo, const char *name, const git_oid *id) { - int error = GIT_SUCCESS; - git_reference *ref = NULL; - - error = reference_create(&ref, repo, name, GIT_REF_OID); - if (error < GIT_SUCCESS) - goto cleanup; - - /* set the oid; this will write the reference on disk */ - error = git_reference_set_oid(ref, id); - if (error < GIT_SUCCESS) - goto cleanup; - - error = git_hashtable_insert(repo->references.loose_cache, ref->name, ref); - if (error < GIT_SUCCESS) - goto cleanup; - - *ref_out = ref; - - return error; - -cleanup: - reference_free(ref); - return error; + return git_reference_create_oid_internal(ref_out, repo, name, id, 0); } +int git_reference_create_oid_force(git_reference **ref_out, git_repository *repo, const char *name, const git_oid *id) +{ + return git_reference_create_oid_internal(ref_out, repo, name, id, 1); +} /** * Getters @@ -1240,125 +1429,14 @@ cleanup: return error; } -/* - * Rename a reference - * - * If the reference is packed, we need to rewrite the - * packfile to remove the reference from it and create - * the reference back as a loose one. - * - * If the reference is loose, we just rename it on - * the filesystem. - * - * We also need to re-insert the reference on its corresponding - * in-memory cache, since the caches are indexed by refname. - */ int git_reference_rename(git_reference *ref, const char *new_name) { - int error; - char *old_name; - char old_path[GIT_PATH_MAX], new_path[GIT_PATH_MAX], normalized_name[MAX_GITDIR_TREE_STRUCTURE_PATH_LENGTH]; - git_reference *looked_up_ref; - - assert(ref); - - /* Ensure the name is valid */ - error = normalize_name(normalized_name, new_name, ref->type & GIT_REF_OID); - if (error < GIT_SUCCESS) - return error; - - /* Ensure we're not going to overwrite an existing reference */ - error = git_reference_lookup(&looked_up_ref, ref->owner, new_name); - if (error == GIT_SUCCESS) - return GIT_EINVALIDREFNAME; - - if (error != GIT_ENOTFOUND) - return error; - - - old_name = ref->name; - ref->name = git__strdup(new_name); - - if (ref->name == NULL) { - ref->name = old_name; - return GIT_ENOMEM; - } - - if (ref->type & GIT_REF_PACKED) { - /* write the packfile to disk; note - * that the state of the in-memory cache is not - * consistent, because the reference is indexed - * by its old name but it already has the new one. - * This doesn't affect writing, though, and allows - * us to rollback if writing fails - */ - - ref->type &= ~GIT_REF_PACKED; - - /* Create the loose ref under its new name */ - error = loose_write(ref); - if (error < GIT_SUCCESS) { - ref->type |= GIT_REF_PACKED; - goto cleanup; - } - - /* Remove from the packfile cache in order to avoid packing it back - * Note : we do not rely on git_reference_delete() because this would - * invalidate the reference. - */ - git_hashtable_remove(ref->owner->references.packfile, old_name); - - /* Recreate the packed-refs file without the reference */ - error = packed_write(ref->owner); - if (error < GIT_SUCCESS) - goto rename_loose_to_old_name; - - } else { - git__joinpath(old_path, ref->owner->path_repository, old_name); - git__joinpath(new_path, ref->owner->path_repository, ref->name); - - error = gitfo_mv_force(old_path, new_path); - if (error < GIT_SUCCESS) - goto cleanup; - - /* Once succesfully renamed, remove from the cache the reference known by its old name*/ - git_hashtable_remove(ref->owner->references.loose_cache, old_name); - } - - /* Store the renamed reference into the loose ref cache */ - error = git_hashtable_insert(ref->owner->references.loose_cache, ref->name, ref); - - free(old_name); - return error; - -cleanup: - /* restore the old name if this failed */ - free(ref->name); - ref->name = old_name; - return error; - -rename_loose_to_old_name: - /* If we hit this point. Something *bad* happened! Think "Ghostbusters - * crossing the streams" definition of bad. - * Either the packed-refs has been correctly generated and something else - * has gone wrong, or the writing of the new packed-refs has failed, and - * we're stuck with the old one. As a loose ref always takes priority over - * a packed ref, we'll eventually try and rename the generated loose ref to - * its former name. It even that fails, well... we might have lost the reference - * for good. :-/ - */ - - git__joinpath(old_path, ref->owner->path_repository, ref->name); - git__joinpath(new_path, ref->owner->path_repository, old_name); - - /* No error checking. We'll return the initial error */ - gitfo_mv_force(old_path, new_path); - - /* restore the old name */ - free(ref->name); - ref->name = old_name; + return git_reference_rename_internal(ref, new_name, 0); +} - return error; +int git_reference_rename_force(git_reference *ref, const char *new_name) +{ + return git_reference_rename_internal(ref, new_name, 1); } int git_reference_resolve(git_reference **resolved_ref, git_reference *ref) -- cgit v1.2.1 From dad4a4d582c55394031f8e78a4994da47fca7540 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Mart=C3=ADn=20Nieto?= Date: Tue, 29 Mar 2011 10:47:48 +0200 Subject: rename: don't return early if the target ref exists MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Carlos Martín Nieto --- src/refs.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/refs.c b/src/refs.c index 20c1d001f..ae745e579 100644 --- a/src/refs.c +++ b/src/refs.c @@ -1050,12 +1050,14 @@ int git_reference_rename_internal(git_reference *ref, const char *new_name, int if (error < GIT_SUCCESS) return error; - /* Ensure we're not going to overwrite an existing reference */ + /* Ensure we're not going to overwrite an existing reference + unless the user has allowed us */ error = git_reference_lookup(&looked_up_ref, ref->owner, new_name); if (error == GIT_SUCCESS && !force) return GIT_EEXISTS; - if (error != GIT_ENOTFOUND) + if (error < GIT_SUCCESS && + error != GIT_ENOTFOUND) return error; -- cgit v1.2.1 From 95cde17ca40de7ad7f245c9302f26c5d59f5d873 Mon Sep 17 00:00:00 2001 From: Vicent Marti Date: Tue, 29 Mar 2011 19:22:21 +0300 Subject: Enforce coding conventions in refs.c Internal methods are static and without the git prefix. 'Force' methods have a `_f` prefix to match the other 'force' methods. --- src/refs.c | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) (limited to 'src') diff --git a/src/refs.c b/src/refs.c index ae745e579..c6508ca8e 100644 --- a/src/refs.c +++ b/src/refs.c @@ -79,6 +79,11 @@ static int packed_remove_loose(git_repository *repo, git_vector *packing_list); static int packed_sort(const void *a, const void *b); static int packed_write(git_repository *repo); +/* internal helpers */ +static int reference_create_symbolic(git_reference **ref_out, git_repository *repo, const char *name, const char *target, int force); +static int reference_create_oid(git_reference **ref_out, git_repository *repo, const char *name, const git_oid *id, int force); +static int reference_rename(git_reference *ref, const char *new_name, int force); + /* name normalization */ static int check_valid_ref_char(char ch); static int normalize_name(char *buffer_out, const char *name, int is_oid_ref); @@ -919,7 +924,7 @@ cleanup: * Internal methods - reference creation *****************************************/ -int git_reference_create_symbolic_internal(git_reference **ref_out, git_repository *repo, const char *name, const char *target, int force) +static int reference_create_symbolic(git_reference **ref_out, git_repository *repo, const char *name, const char *target, int force) { char normalized[MAX_GITDIR_TREE_STRUCTURE_PATH_LENGTH]; int error = GIT_SUCCESS, updated = 0; @@ -976,7 +981,7 @@ cleanup: return error; } -int git_reference_create_oid_internal(git_reference **ref_out, git_repository *repo, const char *name, const git_oid *id, int force) +static int reference_create_oid(git_reference **ref_out, git_repository *repo, const char *name, const git_oid *id, int force) { int error = GIT_SUCCESS, updated = 0; git_reference *ref = NULL, *old_ref = NULL; @@ -1036,7 +1041,7 @@ cleanup: * We also need to re-insert the reference on its corresponding * in-memory cache, since the caches are indexed by refname. */ -int git_reference_rename_internal(git_reference *ref, const char *new_name, int force) +static int reference_rename(git_reference *ref, const char *new_name, int force) { int error; char *old_name; @@ -1207,22 +1212,22 @@ int git_reference_lookup(git_reference **ref_out, git_repository *repo, const ch int git_reference_create_symbolic(git_reference **ref_out, git_repository *repo, const char *name, const char *target) { - return git_reference_create_symbolic_internal(ref_out, repo, name, target, 0); + return reference_create_symbolic(ref_out, repo, name, target, 0); } -int git_reference_create_symbolic_force(git_reference **ref_out, git_repository *repo, const char *name, const char *target) +int git_reference_create_symbolic_f(git_reference **ref_out, git_repository *repo, const char *name, const char *target) { - return git_reference_create_symbolic_internal(ref_out, repo, name, target, 1); + return reference_create_symbolic(ref_out, repo, name, target, 1); } int git_reference_create_oid(git_reference **ref_out, git_repository *repo, const char *name, const git_oid *id) { - return git_reference_create_oid_internal(ref_out, repo, name, id, 0); + return reference_create_oid(ref_out, repo, name, id, 0); } -int git_reference_create_oid_force(git_reference **ref_out, git_repository *repo, const char *name, const git_oid *id) +int git_reference_create_oid_f(git_reference **ref_out, git_repository *repo, const char *name, const git_oid *id) { - return git_reference_create_oid_internal(ref_out, repo, name, id, 1); + return reference_create_oid(ref_out, repo, name, id, 1); } /** @@ -1433,12 +1438,12 @@ cleanup: int git_reference_rename(git_reference *ref, const char *new_name) { - return git_reference_rename_internal(ref, new_name, 0); + return reference_rename(ref, new_name, 0); } -int git_reference_rename_force(git_reference *ref, const char *new_name) +int git_reference_rename_f(git_reference *ref, const char *new_name) { - return git_reference_rename_internal(ref, new_name, 1); + return reference_rename(ref, new_name, 1); } int git_reference_resolve(git_reference **resolved_ref, git_reference *ref) -- cgit v1.2.1 From 9a53df7e0ac364ccba5bd11e9d6c62091f564397 Mon Sep 17 00:00:00 2001 From: Vicent Marti Date: Tue, 29 Mar 2011 19:40:02 +0300 Subject: refs: Don't allow references to inexistent OIDs --- src/refs.c | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'src') diff --git a/src/refs.c b/src/refs.c index c6508ca8e..81706952a 100644 --- a/src/refs.c +++ b/src/refs.c @@ -1319,6 +1319,13 @@ int git_reference_set_oid(git_reference *ref, const git_oid *id) ref_oid = (reference_oid *)ref; + assert(ref->owner); + + /* Don't let the user create references to OIDs that + * don't exist in the ODB */ + if (!git_odb_exists(git_repository_database(ref->owner), id)) + return GIT_ENOTFOUND; + /* duplicate the reference; * this copy will stay on the packfile cache */ if (ref->type & GIT_REF_PACKED) { -- cgit v1.2.1 From 83c95128d924b029b30551f95170df8696583e10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Mart=C3=ADn=20Nieto?= Date: Thu, 24 Mar 2011 14:15:00 +0100 Subject: normalize_name: allow more references under refs/ MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Allow any well-formed reference name to live under refs/ removing the condition that they be under refs/{heads,tags,remotes}/ as was the design of git. An exception is made for HEAD which is allowed to contain an OID reference in detached HEAD state. Signed-off-by: Carlos Martín Nieto --- src/refs.c | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) (limited to 'src') diff --git a/src/refs.c b/src/refs.c index 81706952a..9661988cc 100644 --- a/src/refs.c +++ b/src/refs.c @@ -1680,8 +1680,9 @@ static int normalize_name(char *buffer_out, const char *name, int is_oid_ref) *buffer_out++ = *current++; } - /* Object id refname have to contain at least one slash */ - if (is_oid_ref && !contains_a_slash) + /* Object id refname have to contain at least one slash, except + * for HEAD in a detached state */ + if (is_oid_ref && !contains_a_slash && strcmp(name, GIT_HEAD_FILE)) return GIT_EINVALIDREFNAME; /* A refname can not end with ".lock" */ @@ -1690,9 +1691,13 @@ static int normalize_name(char *buffer_out, const char *name, int is_oid_ref) *buffer_out = '\0'; - /* For object id references, name has to start with refs/(heads|tags|remotes) */ - if (is_oid_ref && !(!git__prefixcmp(buffer_out_start, GIT_REFS_HEADS_DIR) || - !git__prefixcmp(buffer_out_start, GIT_REFS_TAGS_DIR) || !git__prefixcmp(buffer_out_start, GIT_REFS_REMOTES_DIR))) + /* + * For object id references, name has to start with refs/. Again, + * we need to allow HEAD to be in a detached state. + */ + if (is_oid_ref && + !(git__prefixcmp(buffer_out_start, GIT_REFS_DIR) || + strcmp(buffer_out_start, GIT_HEAD_FILE))) return GIT_EINVALIDREFNAME; return error; -- cgit v1.2.1