1 #include "git-compat-util.h"
2 #include "environment.h"
8 #include "chunk-format.h"
9 #include "object-file.h"
10 #include "pack-mtimes.h"
11 #include "pack-objects.h"
12 #include "pack-revindex.h"
14 #include "repository.h"
17 void reset_pack_idx_option(struct pack_idx_option
*opts
)
19 memset(opts
, 0, sizeof(*opts
));
21 opts
->off32_limit
= 0x7fffffff;
22 opts
->delta_base_cache_limit
= DEFAULT_DELTA_BASE_CACHE_LIMIT
;
25 static int sha1_compare(const void *_a
, const void *_b
)
27 struct pack_idx_entry
*a
= *(struct pack_idx_entry
**)_a
;
28 struct pack_idx_entry
*b
= *(struct pack_idx_entry
**)_b
;
29 return oidcmp(&a
->oid
, &b
->oid
);
32 static int cmp_uint32(const void *a_
, const void *b_
)
34 uint32_t a
= *((uint32_t *)a_
);
35 uint32_t b
= *((uint32_t *)b_
);
37 return (a
< b
) ? -1 : (a
!= b
);
40 static int need_large_offset(off_t offset
, const struct pack_idx_option
*opts
)
44 if ((offset
>> 31) || (opts
->off32_limit
< offset
))
46 if (!opts
->anomaly_nr
)
49 return !!bsearch(&ofsval
, opts
->anomaly
, opts
->anomaly_nr
,
50 sizeof(ofsval
), cmp_uint32
);
54 * The *sha1 contains the pack content SHA1 hash.
55 * The objects array passed in will be sorted by SHA1 on exit.
57 const char *write_idx_file(struct repository
*repo
,
58 const char *index_name
, struct pack_idx_entry
**objects
,
59 int nr_objects
, const struct pack_idx_option
*opts
,
60 const unsigned char *sha1
)
63 struct pack_idx_entry
**sorted_by_sha
, **list
, **last
;
64 off_t last_obj_offset
= 0;
66 uint32_t index_version
;
69 sorted_by_sha
= objects
;
71 last
= sorted_by_sha
+ nr_objects
;
72 for (i
= 0; i
< nr_objects
; ++i
) {
73 if (objects
[i
]->offset
> last_obj_offset
)
74 last_obj_offset
= objects
[i
]->offset
;
76 QSORT(sorted_by_sha
, nr_objects
, sha1_compare
);
79 sorted_by_sha
= list
= last
= NULL
;
81 if (opts
->flags
& WRITE_IDX_VERIFY
) {
83 f
= hashfd_check(repo
->hash_algo
, index_name
);
86 struct strbuf tmp_file
= STRBUF_INIT
;
87 fd
= odb_mkstemp(repo
->objects
, &tmp_file
,
88 "pack/tmp_idx_XXXXXX");
89 index_name
= strbuf_detach(&tmp_file
, NULL
);
92 fd
= xopen(index_name
, O_CREAT
|O_EXCL
|O_WRONLY
, 0600);
94 f
= hashfd(repo
->hash_algo
, fd
, index_name
);
97 /* if last object's offset is >= 2^31 we should use index V2 */
98 index_version
= need_large_offset(last_obj_offset
, opts
) ? 2 : opts
->version
;
100 /* index versions 2 and above need a header */
101 if (index_version
>= 2) {
102 struct pack_idx_header hdr
;
103 hdr
.idx_signature
= htonl(PACK_IDX_SIGNATURE
);
104 hdr
.idx_version
= htonl(index_version
);
105 hashwrite(f
, &hdr
, sizeof(hdr
));
109 * Write the first-level table (the list is sorted,
110 * but we use a 256-entry lookup to be able to avoid
111 * having to do eight extra binary search iterations).
113 for (i
= 0; i
< 256; i
++) {
114 struct pack_idx_entry
**next
= list
;
115 while (next
< last
) {
116 struct pack_idx_entry
*obj
= *next
;
117 if (obj
->oid
.hash
[0] != i
)
121 hashwrite_be32(f
, next
- sorted_by_sha
);
126 * Write the actual SHA1 entries..
128 list
= sorted_by_sha
;
129 for (i
= 0; i
< nr_objects
; i
++) {
130 struct pack_idx_entry
*obj
= *list
++;
131 if (index_version
< 2)
132 hashwrite_be32(f
, obj
->offset
);
133 hashwrite(f
, obj
->oid
.hash
, repo
->hash_algo
->rawsz
);
134 if ((opts
->flags
& WRITE_IDX_STRICT
) &&
135 (i
&& oideq(&list
[-2]->oid
, &obj
->oid
)))
136 die("The same object %s appears twice in the pack",
137 oid_to_hex(&obj
->oid
));
140 if (index_version
>= 2) {
141 unsigned int nr_large_offset
= 0;
143 /* write the crc32 table */
144 list
= sorted_by_sha
;
145 for (i
= 0; i
< nr_objects
; i
++) {
146 struct pack_idx_entry
*obj
= *list
++;
147 hashwrite_be32(f
, obj
->crc32
);
150 /* write the 32-bit offset table */
151 list
= sorted_by_sha
;
152 for (i
= 0; i
< nr_objects
; i
++) {
153 struct pack_idx_entry
*obj
= *list
++;
156 offset
= (need_large_offset(obj
->offset
, opts
)
157 ? (0x80000000 | nr_large_offset
++)
159 hashwrite_be32(f
, offset
);
162 /* write the large offset table */
163 list
= sorted_by_sha
;
164 while (nr_large_offset
) {
165 struct pack_idx_entry
*obj
= *list
++;
166 uint64_t offset
= obj
->offset
;
168 if (!need_large_offset(offset
, opts
))
170 hashwrite_be64(f
, offset
);
175 hashwrite(f
, sha1
, repo
->hash_algo
->rawsz
);
176 finalize_hashfile(f
, NULL
, FSYNC_COMPONENT_PACK_METADATA
,
177 CSUM_HASH_IN_STREAM
| CSUM_CLOSE
|
178 ((opts
->flags
& WRITE_IDX_VERIFY
) ? 0 : CSUM_FSYNC
));
182 static int pack_order_cmp(const void *va
, const void *vb
, void *ctx
)
184 struct pack_idx_entry
**objects
= ctx
;
186 off_t oa
= objects
[*(uint32_t*)va
]->offset
;
187 off_t ob
= objects
[*(uint32_t*)vb
]->offset
;
196 static void write_rev_header(const struct git_hash_algo
*hash_algo
,
199 hashwrite_be32(f
, RIDX_SIGNATURE
);
200 hashwrite_be32(f
, RIDX_VERSION
);
201 hashwrite_be32(f
, oid_version(hash_algo
));
204 static void write_rev_index_positions(struct hashfile
*f
,
205 uint32_t *pack_order
,
209 for (i
= 0; i
< nr_objects
; i
++)
210 hashwrite_be32(f
, pack_order
[i
]);
213 static void write_rev_trailer(const struct git_hash_algo
*hash_algo
,
214 struct hashfile
*f
, const unsigned char *hash
)
216 hashwrite(f
, hash
, hash_algo
->rawsz
);
219 char *write_rev_file(struct repository
*repo
,
220 const char *rev_name
,
221 struct pack_idx_entry
**objects
,
223 const unsigned char *hash
,
226 uint32_t *pack_order
;
230 if (!(flags
& WRITE_REV
) && !(flags
& WRITE_REV_VERIFY
))
233 ALLOC_ARRAY(pack_order
, nr_objects
);
234 for (i
= 0; i
< nr_objects
; i
++)
236 QSORT_S(pack_order
, nr_objects
, pack_order_cmp
, objects
);
238 ret
= write_rev_file_order(repo
, rev_name
, pack_order
, nr_objects
,
246 char *write_rev_file_order(struct repository
*repo
,
247 const char *rev_name
,
248 uint32_t *pack_order
,
250 const unsigned char *hash
,
257 if ((flags
& WRITE_REV
) && (flags
& WRITE_REV_VERIFY
))
258 die(_("cannot both write and verify reverse index"));
260 if (flags
& WRITE_REV
) {
262 struct strbuf tmp_file
= STRBUF_INIT
;
263 fd
= odb_mkstemp(repo
->objects
, &tmp_file
,
264 "pack/tmp_rev_XXXXXX");
265 path
= strbuf_detach(&tmp_file
, NULL
);
268 fd
= xopen(rev_name
, O_CREAT
|O_EXCL
|O_WRONLY
, 0600);
269 path
= xstrdup(rev_name
);
271 f
= hashfd(repo
->hash_algo
, fd
, path
);
272 } else if (flags
& WRITE_REV_VERIFY
) {
274 if (stat(rev_name
, &statbuf
)) {
275 if (errno
== ENOENT
) {
276 /* .rev files are optional */
279 die_errno(_("could not stat: %s"), rev_name
);
281 f
= hashfd_check(repo
->hash_algo
, rev_name
);
282 path
= xstrdup(rev_name
);
287 write_rev_header(repo
->hash_algo
, f
);
289 write_rev_index_positions(f
, pack_order
, nr_objects
);
290 write_rev_trailer(repo
->hash_algo
, f
, hash
);
292 if (adjust_shared_perm(repo
, path
) < 0)
293 die(_("failed to make %s readable"), path
);
295 finalize_hashfile(f
, NULL
, FSYNC_COMPONENT_PACK_METADATA
,
296 CSUM_HASH_IN_STREAM
| CSUM_CLOSE
|
297 ((flags
& WRITE_IDX_VERIFY
) ? 0 : CSUM_FSYNC
));
302 static void write_mtimes_header(const struct git_hash_algo
*hash_algo
,
305 hashwrite_be32(f
, MTIMES_SIGNATURE
);
306 hashwrite_be32(f
, MTIMES_VERSION
);
307 hashwrite_be32(f
, oid_version(hash_algo
));
311 * Writes the object mtimes of "objects" for use in a .mtimes file.
312 * Note that objects must be in lexicographic (index) order, which is
313 * the expected ordering of these values in the .mtimes file.
315 static void write_mtimes_objects(struct hashfile
*f
,
316 struct packing_data
*to_pack
,
317 struct pack_idx_entry
**objects
,
321 for (i
= 0; i
< nr_objects
; i
++) {
322 struct object_entry
*e
= (struct object_entry
*)objects
[i
];
323 hashwrite_be32(f
, oe_cruft_mtime(to_pack
, e
));
327 static void write_mtimes_trailer(const struct git_hash_algo
*hash_algo
,
328 struct hashfile
*f
, const unsigned char *hash
)
330 hashwrite(f
, hash
, hash_algo
->rawsz
);
333 static char *write_mtimes_file(struct repository
*repo
,
334 struct packing_data
*to_pack
,
335 struct pack_idx_entry
**objects
,
337 const unsigned char *hash
)
339 struct strbuf tmp_file
= STRBUF_INIT
;
345 BUG("cannot call write_mtimes_file with NULL packing_data");
347 fd
= odb_mkstemp(repo
->objects
, &tmp_file
, "pack/tmp_mtimes_XXXXXX");
348 mtimes_name
= strbuf_detach(&tmp_file
, NULL
);
349 f
= hashfd(repo
->hash_algo
, fd
, mtimes_name
);
351 write_mtimes_header(repo
->hash_algo
, f
);
352 write_mtimes_objects(f
, to_pack
, objects
, nr_objects
);
353 write_mtimes_trailer(repo
->hash_algo
, f
, hash
);
355 if (adjust_shared_perm(repo
, mtimes_name
) < 0)
356 die(_("failed to make %s readable"), mtimes_name
);
358 finalize_hashfile(f
, NULL
, FSYNC_COMPONENT_PACK_METADATA
,
359 CSUM_HASH_IN_STREAM
| CSUM_CLOSE
| CSUM_FSYNC
);
364 off_t
write_pack_header(struct hashfile
*f
, uint32_t nr_entries
)
366 struct pack_header hdr
;
368 hdr
.hdr_signature
= htonl(PACK_SIGNATURE
);
369 hdr
.hdr_version
= htonl(PACK_VERSION
);
370 hdr
.hdr_entries
= htonl(nr_entries
);
371 hashwrite(f
, &hdr
, sizeof(hdr
));
376 * Update pack header with object_count and compute new SHA1 for pack data
377 * associated to pack_fd, and write that SHA1 at the end. That new SHA1
378 * is also returned in new_pack_sha1.
380 * If partial_pack_sha1 is non null, then the SHA1 of the existing pack
381 * (without the header update) is computed and validated against the
382 * one provided in partial_pack_sha1. The validation is performed at
383 * partial_pack_offset bytes in the pack file. The SHA1 of the remaining
384 * data (i.e. from partial_pack_offset to the end) is then computed and
385 * returned in partial_pack_sha1.
387 * Note that new_pack_sha1 is updated last, so both new_pack_sha1 and
388 * partial_pack_sha1 can refer to the same buffer if the caller is not
389 * interested in the resulting SHA1 of pack data above partial_pack_offset.
391 void fixup_pack_header_footer(const struct git_hash_algo
*hash_algo
,
393 unsigned char *new_pack_hash
,
394 const char *pack_name
,
395 uint32_t object_count
,
396 unsigned char *partial_pack_hash
,
397 off_t partial_pack_offset
)
399 int aligned_sz
, buf_sz
= 8 * 1024;
400 struct git_hash_ctx old_hash_ctx
, new_hash_ctx
;
401 struct pack_header hdr
;
405 hash_algo
->init_fn(&old_hash_ctx
);
406 hash_algo
->init_fn(&new_hash_ctx
);
408 if (lseek(pack_fd
, 0, SEEK_SET
) != 0)
409 die_errno("Failed seeking to start of '%s'", pack_name
);
410 read_result
= read_in_full(pack_fd
, &hdr
, sizeof(hdr
));
412 die_errno("Unable to reread header of '%s'", pack_name
);
413 else if (read_result
!= sizeof(hdr
))
414 die_errno("Unexpected short read for header of '%s'",
416 if (lseek(pack_fd
, 0, SEEK_SET
) != 0)
417 die_errno("Failed seeking to start of '%s'", pack_name
);
418 git_hash_update(&old_hash_ctx
, &hdr
, sizeof(hdr
));
419 hdr
.hdr_entries
= htonl(object_count
);
420 git_hash_update(&new_hash_ctx
, &hdr
, sizeof(hdr
));
421 write_or_die(pack_fd
, &hdr
, sizeof(hdr
));
422 partial_pack_offset
-= sizeof(hdr
);
424 buf
= xmalloc(buf_sz
);
425 aligned_sz
= buf_sz
- sizeof(hdr
);
428 m
= (partial_pack_hash
&& partial_pack_offset
< aligned_sz
) ?
429 partial_pack_offset
: aligned_sz
;
430 n
= xread(pack_fd
, buf
, m
);
434 die_errno("Failed to checksum '%s'", pack_name
);
435 git_hash_update(&new_hash_ctx
, buf
, n
);
441 if (!partial_pack_hash
)
444 git_hash_update(&old_hash_ctx
, buf
, n
);
445 partial_pack_offset
-= n
;
446 if (partial_pack_offset
== 0) {
447 unsigned char hash
[GIT_MAX_RAWSZ
];
448 git_hash_final(hash
, &old_hash_ctx
);
449 if (!hasheq(hash
, partial_pack_hash
,
451 die("Unexpected checksum for %s "
452 "(disk corruption?)", pack_name
);
454 * Now let's compute the SHA1 of the remainder of the
455 * pack, which also means making partial_pack_offset
456 * big enough not to matter anymore.
458 hash_algo
->init_fn(&old_hash_ctx
);
459 partial_pack_offset
= ~partial_pack_offset
;
460 partial_pack_offset
-= MSB(partial_pack_offset
, 1);
465 if (partial_pack_hash
)
466 git_hash_final(partial_pack_hash
, &old_hash_ctx
);
467 git_hash_final(new_pack_hash
, &new_hash_ctx
);
468 write_or_die(pack_fd
, new_pack_hash
, hash_algo
->rawsz
);
469 fsync_component_or_die(FSYNC_COMPONENT_PACK
, pack_fd
, pack_name
);
472 char *index_pack_lockfile(struct repository
*r
, int ip_out
, int *is_well_formed
)
474 char packname
[GIT_MAX_HEXSZ
+ 6];
475 const int len
= r
->hash_algo
->hexsz
+ 6;
478 * The first thing we expect from index-pack's output
479 * is "pack\t%40s\n" or "keep\t%40s\n" (46 bytes) where
480 * %40s is the newly created pack SHA1 name. In the "keep"
481 * case, we need it to remove the corresponding .keep file
482 * later on. If we don't get that then tough luck with it.
484 if (read_in_full(ip_out
, packname
, len
) == len
&& packname
[len
-1] == '\n') {
490 if (skip_prefix(packname
, "keep\t", &name
))
491 return xstrfmt("%s/pack/pack-%s.keep",
492 repo_get_object_directory(r
), name
);
501 * The per-object header is a pretty dense thing, which is
502 * - first byte: low four bits are "size", then three bits of "type",
503 * and the high bit is "size continues".
504 * - each byte afterwards: low seven bits are size continuation,
505 * with the high bit being "size continues"
507 int encode_in_pack_object_header(unsigned char *hdr
, int hdr_len
,
508 enum object_type type
, uintmax_t size
)
513 if (type
< OBJ_COMMIT
|| type
> OBJ_REF_DELTA
)
514 die("bad type %d", type
);
516 c
= (type
<< 4) | (size
& 15);
520 die("object size is too enormous to format");
530 struct hashfile
*create_tmp_packfile(struct repository
*repo
,
531 char **pack_tmp_name
)
533 struct strbuf tmpname
= STRBUF_INIT
;
536 fd
= odb_mkstemp(repo
->objects
, &tmpname
, "pack/tmp_pack_XXXXXX");
537 *pack_tmp_name
= strbuf_detach(&tmpname
, NULL
);
538 return hashfd(repo
->hash_algo
, fd
, *pack_tmp_name
);
541 static void rename_tmp_packfile(struct strbuf
*name_prefix
, const char *source
,
544 size_t name_prefix_len
= name_prefix
->len
;
546 strbuf_addstr(name_prefix
, ext
);
547 if (finalize_object_file(source
, name_prefix
->buf
))
548 die("unable to rename temporary file to '%s'",
550 strbuf_setlen(name_prefix
, name_prefix_len
);
553 void rename_tmp_packfile_idx(struct strbuf
*name_buffer
,
556 rename_tmp_packfile(name_buffer
, *idx_tmp_name
, "idx");
559 void stage_tmp_packfiles(struct repository
*repo
,
560 struct strbuf
*name_buffer
,
561 const char *pack_tmp_name
,
562 struct pack_idx_entry
**written_list
,
564 struct packing_data
*to_pack
,
565 struct pack_idx_option
*pack_idx_opts
,
566 unsigned char hash
[],
569 char *rev_tmp_name
= NULL
;
570 char *mtimes_tmp_name
= NULL
;
572 if (adjust_shared_perm(repo
, pack_tmp_name
))
573 die_errno("unable to make temporary pack file readable");
575 *idx_tmp_name
= (char *)write_idx_file(repo
, NULL
, written_list
,
576 nr_written
, pack_idx_opts
, hash
);
577 if (adjust_shared_perm(repo
, *idx_tmp_name
))
578 die_errno("unable to make temporary index file readable");
580 rev_tmp_name
= write_rev_file(repo
, NULL
, written_list
, nr_written
,
581 hash
, pack_idx_opts
->flags
);
583 if (pack_idx_opts
->flags
& WRITE_MTIMES
) {
584 mtimes_tmp_name
= write_mtimes_file(repo
, to_pack
,
585 written_list
, nr_written
,
589 rename_tmp_packfile(name_buffer
, pack_tmp_name
, "pack");
591 rename_tmp_packfile(name_buffer
, rev_tmp_name
, "rev");
593 rename_tmp_packfile(name_buffer
, mtimes_tmp_name
, "mtimes");
596 free(mtimes_tmp_name
);
599 void write_promisor_file(const char *promisor_name
, struct ref
**sought
, int nr_sought
)
602 FILE *output
= xfopen(promisor_name
, "w");
604 for (i
= 0; i
< nr_sought
; i
++)
605 fprintf(output
, "%s %s\n", oid_to_hex(&sought
[i
]->old_oid
),
608 err
= ferror(output
);
609 err
|= fclose(output
);
611 die(_("could not write '%s' promisor file"), promisor_name
);