odb.c

   1 #include "git-compat-util.h"
   2 #include "abspath.h"
   3 #include "commit-graph.h"
   4 #include "config.h"
   5 #include "dir.h"
   6 #include "environment.h"
   7 #include "gettext.h"
   8 #include "hex.h"
   9 #include "khash.h"
  10 #include "lockfile.h"
  11 #include "loose.h"
  12 #include "object-file-convert.h"
  13 #include "object-file.h"
  14 #include "odb.h"
  15 #include "packfile.h"
  16 #include "path.h"
  17 #include "promisor-remote.h"
  18 #include "quote.h"
  19 #include "replace-object.h"
  20 #include "run-command.h"
  21 #include "setup.h"
  22 #include "strbuf.h"
  23 #include "strvec.h"
  24 #include "submodule.h"
  25 #include "trace2.h"
  26 #include "write-or-die.h"
  27
  28 KHASH_INIT(odb_path_map, const char * /* key: odb_path */,
  29         struct odb_source *, 1, fspathhash, fspatheq)
  30
  31 /*
  32  * This is meant to hold a *small* number of objects that you would
  33  * want odb_read_object() to be able to return, but yet you do not want
  34  * to write them into the object store (e.g. a browse-only
  35  * application).
  36  */
  37 struct cached_object_entry {
  38         struct object_id oid;
  39         struct cached_object {
  40                 enum object_type type;
  41                 const void *buf;
  42                 unsigned long size;
  43         } value;
  44 };
  45
  46 static const struct cached_object *find_cached_object(struct object_database *object_store,
  47                                                       const struct object_id *oid)
  48 {
  49         static const struct cached_object empty_tree = {
  50                 .type = OBJ_TREE,
  51                 .buf = "",
  52         };
  53         const struct cached_object_entry *co = object_store->cached_objects;
  54
  55         for (size_t i = 0; i < object_store->cached_object_nr; i++, co++)
  56                 if (oideq(&co->oid, oid))
  57                         return &co->value;
  58
  59         if (oid->algo && oideq(oid, hash_algos[oid->algo].empty_tree))
  60                 return &empty_tree;
  61
  62         return NULL;
  63 }
  64
  65 int odb_mkstemp(struct object_database *odb,
  66                 struct strbuf *temp_filename, const char *pattern)
  67 {
  68         int fd;
  69         /*
  70          * we let the umask do its job, don't try to be more
  71          * restrictive except to remove write permission.
  72          */
  73         int mode = 0444;
  74         repo_git_path_replace(odb->repo, temp_filename, "objects/%s", pattern);
  75         fd = git_mkstemp_mode(temp_filename->buf, mode);
  76         if (0 <= fd)
  77                 return fd;
  78
  79         /* slow path */
  80         /* some mkstemp implementations erase temp_filename on failure */
  81         repo_git_path_replace(odb->repo, temp_filename, "objects/%s", pattern);
  82         safe_create_leading_directories(odb->repo, temp_filename->buf);
  83         return xmkstemp_mode(temp_filename->buf, mode);
  84 }
  85
  86 /*
  87  * Return non-zero iff the path is usable as an alternate object database.
  88  */
  89 static int alt_odb_usable(struct object_database *o,
  90                           struct strbuf *path,
  91                           const char *normalized_objdir, khiter_t *pos)
  92 {
  93         int r;
  94
  95         /* Detect cases where alternate disappeared */
  96         if (!is_directory(path->buf)) {
  97                 error(_("object directory %s does not exist; "
  98                         "check .git/objects/info/alternates"),
  99                       path->buf);
 100                 return 0;
 101         }
 102
 103         /*
 104          * Prevent the common mistake of listing the same
 105          * thing twice, or object directory itself.
 106          */
 107         if (!o->source_by_path) {
 108                 khiter_t p;
 109
 110                 o->source_by_path = kh_init_odb_path_map();
 111                 assert(!o->sources->next);
 112                 p = kh_put_odb_path_map(o->source_by_path, o->sources->path, &r);
 113                 assert(r == 1); /* never used */
 114                 kh_value(o->source_by_path, p) = o->sources;
 115         }
 116         if (fspatheq(path->buf, normalized_objdir))
 117                 return 0;
 118         *pos = kh_put_odb_path_map(o->source_by_path, path->buf, &r);
 119         /* r: 0 = exists, 1 = never used, 2 = deleted */
 120         return r == 0 ? 0 : 1;
 121 }
 122
 123 /*
 124  * Prepare alternate object database registry.
 125  *
 126  * The variable alt_odb_list points at the list of struct
 127  * odb_source.  The elements on this list come from
 128  * non-empty elements from colon separated ALTERNATE_DB_ENVIRONMENT
 129  * environment variable, and $GIT_OBJECT_DIRECTORY/info/alternates,
 130  * whose contents is similar to that environment variable but can be
 131  * LF separated.  Its base points at a statically allocated buffer that
 132  * contains "/the/directory/corresponding/to/.git/objects/...", while
 133  * its name points just after the slash at the end of ".git/objects/"
 134  * in the example above, and has enough space to hold all hex characters
 135  * of the object ID, an extra slash for the first level indirection, and
 136  * the terminating NUL.
 137  */
 138 static void read_info_alternates(struct object_database *odb,
 139                                  const char *relative_base,
 140                                  int depth);
 141
 142 static int link_alt_odb_entry(struct object_database *odb,
 143                               const struct strbuf *entry,
 144                               const char *relative_base,
 145                               int depth,
 146                               const char *normalized_objdir)
 147 {
 148         struct odb_source *alternate;
 149         struct strbuf pathbuf = STRBUF_INIT;
 150         struct strbuf tmp = STRBUF_INIT;
 151         khiter_t pos;
 152         int ret = -1;
 153
 154         if (!is_absolute_path(entry->buf) && relative_base) {
 155                 strbuf_realpath(&pathbuf, relative_base, 1);
 156                 strbuf_addch(&pathbuf, '/');
 157         }
 158         strbuf_addbuf(&pathbuf, entry);
 159
 160         if (!strbuf_realpath(&tmp, pathbuf.buf, 0)) {
 161                 error(_("unable to normalize alternate object path: %s"),
 162                       pathbuf.buf);
 163                 goto error;
 164         }
 165         strbuf_swap(&pathbuf, &tmp);
 166
 167         /*
 168          * The trailing slash after the directory name is given by
 169          * this function at the end. Remove duplicates.
 170          */
 171         while (pathbuf.len && pathbuf.buf[pathbuf.len - 1] == '/')
 172                 strbuf_setlen(&pathbuf, pathbuf.len - 1);
 173
 174         if (!alt_odb_usable(odb, &pathbuf, normalized_objdir, &pos))
 175                 goto error;
 176
 177         CALLOC_ARRAY(alternate, 1);
 178         alternate->odb = odb;
 179         /* pathbuf.buf is already in r->objects->source_by_path */
 180         alternate->path = strbuf_detach(&pathbuf, NULL);
 181
 182         /* add the alternate entry */
 183         *odb->sources_tail = alternate;
 184         odb->sources_tail = &(alternate->next);
 185         alternate->next = NULL;
 186         assert(odb->source_by_path);
 187         kh_value(odb->source_by_path, pos) = alternate;
 188
 189         /* recursively add alternates */
 190         read_info_alternates(odb, alternate->path, depth + 1);
 191         ret = 0;
 192  error:
 193         strbuf_release(&tmp);
 194         strbuf_release(&pathbuf);
 195         return ret;
 196 }
 197
 198 static const char *parse_alt_odb_entry(const char *string,
 199                                        int sep,
 200                                        struct strbuf *out)
 201 {
 202         const char *end;
 203
 204         strbuf_reset(out);
 205
 206         if (*string == '#') {
 207                 /* comment; consume up to next separator */
 208                 end = strchrnul(string, sep);
 209         } else if (*string == '"' && !unquote_c_style(out, string, &end)) {
 210                 /*
 211                  * quoted path; unquote_c_style has copied the
 212                  * data for us and set "end". Broken quoting (e.g.,
 213                  * an entry that doesn't end with a quote) falls
 214                  * back to the unquoted case below.
 215                  */
 216         } else {
 217                 /* normal, unquoted path */
 218                 end = strchrnul(string, sep);
 219                 strbuf_add(out, string, end - string);
 220         }
 221
 222         if (*end)
 223                 end++;
 224         return end;
 225 }
 226
 227 static void link_alt_odb_entries(struct object_database *odb, const char *alt,
 228                                  int sep, const char *relative_base, int depth)
 229 {
 230         struct strbuf objdirbuf = STRBUF_INIT;
 231         struct strbuf entry = STRBUF_INIT;
 232
 233         if (!alt || !*alt)
 234                 return;
 235
 236         if (depth > 5) {
 237                 error(_("%s: ignoring alternate object stores, nesting too deep"),
 238                                 relative_base);
 239                 return;
 240         }
 241
 242         strbuf_realpath(&objdirbuf, odb->sources->path, 1);
 243
 244         while (*alt) {
 245                 alt = parse_alt_odb_entry(alt, sep, &entry);
 246                 if (!entry.len)
 247                         continue;
 248                 link_alt_odb_entry(odb, &entry,
 249                                    relative_base, depth, objdirbuf.buf);
 250         }
 251         strbuf_release(&entry);
 252         strbuf_release(&objdirbuf);
 253 }
 254
 255 static void read_info_alternates(struct object_database *odb,
 256                                  const char *relative_base,
 257                                  int depth)
 258 {
 259         char *path;
 260         struct strbuf buf = STRBUF_INIT;
 261
 262         path = xstrfmt("%s/info/alternates", relative_base);
 263         if (strbuf_read_file(&buf, path, 1024) < 0) {
 264                 warn_on_fopen_errors(path);
 265                 free(path);
 266                 return;
 267         }
 268
 269         link_alt_odb_entries(odb, buf.buf, '\n', relative_base, depth);
 270         strbuf_release(&buf);
 271         free(path);
 272 }
 273
 274 void odb_add_to_alternates_file(struct object_database *odb,
 275                                 const char *reference)
 276 {
 277         struct lock_file lock = LOCK_INIT;
 278         char *alts = repo_git_path(odb->repo, "objects/info/alternates");
 279         FILE *in, *out;
 280         int found = 0;
 281
 282         hold_lock_file_for_update(&lock, alts, LOCK_DIE_ON_ERROR);
 283         out = fdopen_lock_file(&lock, "w");
 284         if (!out)
 285                 die_errno(_("unable to fdopen alternates lockfile"));
 286
 287         in = fopen(alts, "r");
 288         if (in) {
 289                 struct strbuf line = STRBUF_INIT;
 290
 291                 while (strbuf_getline(&line, in) != EOF) {
 292                         if (!strcmp(reference, line.buf)) {
 293                                 found = 1;
 294                                 break;
 295                         }
 296                         fprintf_or_die(out, "%s\n", line.buf);
 297                 }
 298
 299                 strbuf_release(&line);
 300                 fclose(in);
 301         }
 302         else if (errno != ENOENT)
 303                 die_errno(_("unable to read alternates file"));
 304
 305         if (found) {
 306                 rollback_lock_file(&lock);
 307         } else {
 308                 fprintf_or_die(out, "%s\n", reference);
 309                 if (commit_lock_file(&lock))
 310                         die_errno(_("unable to move new alternates file into place"));
 311                 if (odb->loaded_alternates)
 312                         link_alt_odb_entries(odb, reference,
 313                                              '\n', NULL, 0);
 314         }
 315         free(alts);
 316 }
 317
 318 void odb_add_to_alternates_memory(struct object_database *odb,
 319                                   const char *reference)
 320 {
 321         /*
 322          * Make sure alternates are initialized, or else our entry may be
 323          * overwritten when they are.
 324          */
 325         odb_prepare_alternates(odb);
 326
 327         link_alt_odb_entries(odb, reference,
 328                              '\n', NULL, 0);
 329 }
 330
 331 struct odb_source *odb_set_temporary_primary_source(struct object_database *odb,
 332                                                     const char *dir, int will_destroy)
 333 {
 334         struct odb_source *source;
 335
 336         /*
 337          * Make sure alternates are initialized, or else our entry may be
 338          * overwritten when they are.
 339          */
 340         odb_prepare_alternates(odb);
 341
 342         /*
 343          * Make a new primary odb and link the old primary ODB in as an
 344          * alternate
 345          */
 346         source = xcalloc(1, sizeof(*source));
 347         source->odb = odb;
 348         source->path = xstrdup(dir);
 349
 350         /*
 351          * Disable ref updates while a temporary odb is active, since
 352          * the objects in the database may roll back.
 353          */
 354         source->disable_ref_updates = 1;
 355         source->will_destroy = will_destroy;
 356         source->next = odb->sources;
 357         odb->sources = source;
 358         return source->next;
 359 }
 360
 361 static void free_object_directory(struct odb_source *source)
 362 {
 363         free(source->path);
 364         odb_clear_loose_cache(source);
 365         loose_object_map_clear(&source->loose_map);
 366         free(source);
 367 }
 368
 369 void odb_restore_primary_source(struct object_database *odb,
 370                                 struct odb_source *restore_source,
 371                                 const char *old_path)
 372 {
 373         struct odb_source *cur_source = odb->sources;
 374
 375         if (strcmp(old_path, cur_source->path))
 376                 BUG("expected %s as primary object store; found %s",
 377                     old_path, cur_source->path);
 378
 379         if (cur_source->next != restore_source)
 380                 BUG("we expect the old primary object store to be the first alternate");
 381
 382         odb->sources = restore_source;
 383         free_object_directory(cur_source);
 384 }
 385
 386 char *compute_alternate_path(const char *path, struct strbuf *err)
 387 {
 388         char *ref_git = NULL;
 389         const char *repo;
 390         int seen_error = 0;
 391
 392         ref_git = real_pathdup(path, 0);
 393         if (!ref_git) {
 394                 seen_error = 1;
 395                 strbuf_addf(err, _("path '%s' does not exist"), path);
 396                 goto out;
 397         }
 398
 399         repo = read_gitfile(ref_git);
 400         if (!repo)
 401                 repo = read_gitfile(mkpath("%s/.git", ref_git));
 402         if (repo) {
 403                 free(ref_git);
 404                 ref_git = xstrdup(repo);
 405         }
 406
 407         if (!repo && is_directory(mkpath("%s/.git/objects", ref_git))) {
 408                 char *ref_git_git = mkpathdup("%s/.git", ref_git);
 409                 free(ref_git);
 410                 ref_git = ref_git_git;
 411         } else if (!is_directory(mkpath("%s/objects", ref_git))) {
 412                 struct strbuf sb = STRBUF_INIT;
 413                 seen_error = 1;
 414                 if (get_common_dir(&sb, ref_git)) {
 415                         strbuf_addf(err,
 416                                     _("reference repository '%s' as a linked "
 417                                       "checkout is not supported yet."),
 418                                     path);
 419                         goto out;
 420                 }
 421
 422                 strbuf_addf(err, _("reference repository '%s' is not a "
 423                                         "local repository."), path);
 424                 goto out;
 425         }
 426
 427         if (!access(mkpath("%s/shallow", ref_git), F_OK)) {
 428                 strbuf_addf(err, _("reference repository '%s' is shallow"),
 429                             path);
 430                 seen_error = 1;
 431                 goto out;
 432         }
 433
 434         if (!access(mkpath("%s/info/grafts", ref_git), F_OK)) {
 435                 strbuf_addf(err,
 436                             _("reference repository '%s' is grafted"),
 437                             path);
 438                 seen_error = 1;
 439                 goto out;
 440         }
 441
 442 out:
 443         if (seen_error) {
 444                 FREE_AND_NULL(ref_git);
 445         }
 446
 447         return ref_git;
 448 }
 449
 450 struct odb_source *odb_find_source(struct object_database *odb, const char *obj_dir)
 451 {
 452         struct odb_source *source;
 453         char *obj_dir_real = real_pathdup(obj_dir, 1);
 454         struct strbuf odb_path_real = STRBUF_INIT;
 455
 456         odb_prepare_alternates(odb);
 457         for (source = odb->sources; source; source = source->next) {
 458                 strbuf_realpath(&odb_path_real, source->path, 1);
 459                 if (!strcmp(obj_dir_real, odb_path_real.buf))
 460                         break;
 461         }
 462
 463         free(obj_dir_real);
 464         strbuf_release(&odb_path_real);
 465
 466         if (!source)
 467                 die(_("could not find object directory matching %s"), obj_dir);
 468         return source;
 469 }
 470
 471 void odb_add_submodule_source_by_path(struct object_database *odb,
 472                                       const char *path)
 473 {
 474         string_list_insert(&odb->submodule_source_paths, path);
 475 }
 476
 477 static void fill_alternate_refs_command(struct repository *repo,
 478                                         struct child_process *cmd,
 479                                         const char *repo_path)
 480 {
 481         const char *value;
 482
 483         if (!repo_config_get_value(repo, "core.alternateRefsCommand", &value)) {
 484                 cmd->use_shell = 1;
 485
 486                 strvec_push(&cmd->args, value);
 487                 strvec_push(&cmd->args, repo_path);
 488         } else {
 489                 cmd->git_cmd = 1;
 490
 491                 strvec_pushf(&cmd->args, "--git-dir=%s", repo_path);
 492                 strvec_push(&cmd->args, "for-each-ref");
 493                 strvec_push(&cmd->args, "--format=%(objectname)");
 494
 495                 if (!repo_config_get_value(repo, "core.alternateRefsPrefixes", &value)) {
 496                         strvec_push(&cmd->args, "--");
 497                         strvec_split(&cmd->args, value);
 498                 }
 499         }
 500
 501         strvec_pushv(&cmd->env, (const char **)local_repo_env);
 502         cmd->out = -1;
 503 }
 504
 505 static void read_alternate_refs(struct repository *repo,
 506                                 const char *path,
 507                                 odb_for_each_alternate_ref_fn *cb,
 508                                 void *payload)
 509 {
 510         struct child_process cmd = CHILD_PROCESS_INIT;
 511         struct strbuf line = STRBUF_INIT;
 512         FILE *fh;
 513
 514         fill_alternate_refs_command(repo, &cmd, path);
 515
 516         if (start_command(&cmd))
 517                 return;
 518
 519         fh = xfdopen(cmd.out, "r");
 520         while (strbuf_getline_lf(&line, fh) != EOF) {
 521                 struct object_id oid;
 522                 const char *p;
 523
 524                 if (parse_oid_hex_algop(line.buf, &oid, &p, repo->hash_algo) || *p) {
 525                         warning(_("invalid line while parsing alternate refs: %s"),
 526                                 line.buf);
 527                         break;
 528                 }
 529
 530                 cb(&oid, payload);
 531         }
 532
 533         fclose(fh);
 534         finish_command(&cmd);
 535         strbuf_release(&line);
 536 }
 537
 538 struct alternate_refs_data {
 539         odb_for_each_alternate_ref_fn *fn;
 540         void *payload;
 541 };
 542
 543 static int refs_from_alternate_cb(struct odb_source *alternate,
 544                                   void *payload)
 545 {
 546         struct strbuf path = STRBUF_INIT;
 547         size_t base_len;
 548         struct alternate_refs_data *cb = payload;
 549
 550         if (!strbuf_realpath(&path, alternate->path, 0))
 551                 goto out;
 552         if (!strbuf_strip_suffix(&path, "/objects"))
 553                 goto out;
 554         base_len = path.len;
 555
 556         /* Is this a git repository with refs? */
 557         strbuf_addstr(&path, "/refs");
 558         if (!is_directory(path.buf))
 559                 goto out;
 560         strbuf_setlen(&path, base_len);
 561
 562         read_alternate_refs(alternate->odb->repo, path.buf, cb->fn, cb->payload);
 563
 564 out:
 565         strbuf_release(&path);
 566         return 0;
 567 }
 568
 569 void odb_for_each_alternate_ref(struct object_database *odb,
 570                                 odb_for_each_alternate_ref_fn cb, void *payload)
 571 {
 572         struct alternate_refs_data data;
 573         data.fn = cb;
 574         data.payload = payload;
 575         odb_for_each_alternate(odb, refs_from_alternate_cb, &data);
 576 }
 577
 578 int odb_for_each_alternate(struct object_database *odb,
 579                          odb_for_each_alternate_fn cb, void *payload)
 580 {
 581         struct odb_source *alternate;
 582         int r = 0;
 583
 584         odb_prepare_alternates(odb);
 585         for (alternate = odb->sources->next; alternate; alternate = alternate->next) {
 586                 r = cb(alternate, payload);
 587                 if (r)
 588                         break;
 589         }
 590         return r;
 591 }
 592
 593 void odb_prepare_alternates(struct object_database *odb)
 594 {
 595         if (odb->loaded_alternates)
 596                 return;
 597
 598         link_alt_odb_entries(odb, odb->alternate_db, PATH_SEP, NULL, 0);
 599
 600         read_info_alternates(odb, odb->sources->path, 0);
 601         odb->loaded_alternates = 1;
 602 }
 603
 604 int odb_has_alternates(struct object_database *odb)
 605 {
 606         odb_prepare_alternates(odb);
 607         return !!odb->sources->next;
 608 }
 609
 610 int obj_read_use_lock = 0;
 611 pthread_mutex_t obj_read_mutex;
 612
 613 void enable_obj_read_lock(void)
 614 {
 615         if (obj_read_use_lock)
 616                 return;
 617
 618         obj_read_use_lock = 1;
 619         init_recursive_mutex(&obj_read_mutex);
 620 }
 621
 622 void disable_obj_read_lock(void)
 623 {
 624         if (!obj_read_use_lock)
 625                 return;
 626
 627         obj_read_use_lock = 0;
 628         pthread_mutex_destroy(&obj_read_mutex);
 629 }
 630
 631 int fetch_if_missing = 1;
 632
 633 static int register_all_submodule_sources(struct object_database *odb)
 634 {
 635         int ret = odb->submodule_source_paths.nr;
 636
 637         for (size_t i = 0; i < odb->submodule_source_paths.nr; i++)
 638                 odb_add_to_alternates_memory(odb,
 639                                              odb->submodule_source_paths.items[i].string);
 640         if (ret) {
 641                 string_list_clear(&odb->submodule_source_paths, 0);
 642                 trace2_data_intmax("submodule", odb->repo,
 643                                    "register_all_submodule_sources/registered", ret);
 644                 if (git_env_bool("GIT_TEST_FATAL_REGISTER_SUBMODULE_ODB", 0))
 645                         BUG("register_all_submodule_sources() called");
 646         }
 647         return ret;
 648 }
 649
 650 static int do_oid_object_info_extended(struct object_database *odb,
 651                                        const struct object_id *oid,
 652                                        struct object_info *oi, unsigned flags)
 653 {
 654         static struct object_info blank_oi = OBJECT_INFO_INIT;
 655         const struct cached_object *co;
 656         struct pack_entry e;
 657         int rtype;
 658         const struct object_id *real = oid;
 659         int already_retried = 0;
 660
 661
 662         if (flags & OBJECT_INFO_LOOKUP_REPLACE)
 663                 real = lookup_replace_object(odb->repo, oid);
 664
 665         if (is_null_oid(real))
 666                 return -1;
 667
 668         if (!oi)
 669                 oi = &blank_oi;
 670
 671         co = find_cached_object(odb, real);
 672         if (co) {
 673                 if (oi->typep)
 674                         *(oi->typep) = co->type;
 675                 if (oi->sizep)
 676                         *(oi->sizep) = co->size;
 677                 if (oi->disk_sizep)
 678                         *(oi->disk_sizep) = 0;
 679                 if (oi->delta_base_oid)
 680                         oidclr(oi->delta_base_oid, odb->repo->hash_algo);
 681                 if (oi->contentp)
 682                         *oi->contentp = xmemdupz(co->buf, co->size);
 683                 oi->whence = OI_CACHED;
 684                 return 0;
 685         }
 686
 687         while (1) {
 688                 if (find_pack_entry(odb->repo, real, &e))
 689                         break;
 690
 691                 /* Most likely it's a loose object. */
 692                 if (!loose_object_info(odb->repo, real, oi, flags))
 693                         return 0;
 694
 695                 /* Not a loose object; someone else may have just packed it. */
 696                 if (!(flags & OBJECT_INFO_QUICK)) {
 697                         reprepare_packed_git(odb->repo);
 698                         if (find_pack_entry(odb->repo, real, &e))
 699                                 break;
 700                 }
 701
 702                 /*
 703                  * This might be an attempt at accessing a submodule object as
 704                  * if it were in main object store (having called
 705                  * `odb_add_submodule_source_by_path()` on that submodule's
 706                  * ODB). If any such ODBs exist, register them and try again.
 707                  */
 708                 if (register_all_submodule_sources(odb))
 709                         /* We added some alternates; retry */
 710                         continue;
 711
 712                 /* Check if it is a missing object */
 713                 if (fetch_if_missing && repo_has_promisor_remote(odb->repo) &&
 714                     !already_retried &&
 715                     !(flags & OBJECT_INFO_SKIP_FETCH_OBJECT)) {
 716                         promisor_remote_get_direct(odb->repo, real, 1);
 717                         already_retried = 1;
 718                         continue;
 719                 }
 720
 721                 if (flags & OBJECT_INFO_DIE_IF_CORRUPT) {
 722                         const struct packed_git *p;
 723                         if ((flags & OBJECT_INFO_LOOKUP_REPLACE) && !oideq(real, oid))
 724                                 die(_("replacement %s not found for %s"),
 725                                     oid_to_hex(real), oid_to_hex(oid));
 726                         if ((p = has_packed_and_bad(odb->repo, real)))
 727                                 die(_("packed object %s (stored in %s) is corrupt"),
 728                                     oid_to_hex(real), p->pack_name);
 729                 }
 730                 return -1;
 731         }
 732
 733         if (oi == &blank_oi)
 734                 /*
 735                  * We know that the caller doesn't actually need the
 736                  * information below, so return early.
 737                  */
 738                 return 0;
 739         rtype = packed_object_info(odb->repo, e.p, e.offset, oi);
 740         if (rtype < 0) {
 741                 mark_bad_packed_object(e.p, real);
 742                 return do_oid_object_info_extended(odb, real, oi, 0);
 743         } else if (oi->whence == OI_PACKED) {
 744                 oi->u.packed.offset = e.offset;
 745                 oi->u.packed.pack = e.p;
 746                 oi->u.packed.is_delta = (rtype == OBJ_REF_DELTA ||
 747                                          rtype == OBJ_OFS_DELTA);
 748         }
 749
 750         return 0;
 751 }
 752
 753 static int oid_object_info_convert(struct repository *r,
 754                                    const struct object_id *input_oid,
 755                                    struct object_info *input_oi, unsigned flags)
 756 {
 757         const struct git_hash_algo *input_algo = &hash_algos[input_oid->algo];
 758         int do_die = flags & OBJECT_INFO_DIE_IF_CORRUPT;
 759         enum object_type type;
 760         struct object_id oid, delta_base_oid;
 761         struct object_info new_oi, *oi;
 762         unsigned long size;
 763         void *content;
 764         int ret;
 765
 766         if (repo_oid_to_algop(r, input_oid, r->hash_algo, &oid)) {
 767                 if (do_die)
 768                         die(_("missing mapping of %s to %s"),
 769                             oid_to_hex(input_oid), r->hash_algo->name);
 770                 return -1;
 771         }
 772
 773         /* Is new_oi needed? */
 774         oi = input_oi;
 775         if (input_oi && (input_oi->delta_base_oid || input_oi->sizep ||
 776                          input_oi->contentp)) {
 777                 new_oi = *input_oi;
 778                 /* Does delta_base_oid need to be converted? */
 779                 if (input_oi->delta_base_oid)
 780                         new_oi.delta_base_oid = &delta_base_oid;
 781                 /* Will the attributes differ when converted? */
 782                 if (input_oi->sizep || input_oi->contentp) {
 783                         new_oi.contentp = &content;
 784                         new_oi.sizep = &size;
 785                         new_oi.typep = &type;
 786                 }
 787                 oi = &new_oi;
 788         }
 789
 790         ret = odb_read_object_info_extended(r->objects, &oid, oi, flags);
 791         if (ret)
 792                 return -1;
 793         if (oi == input_oi)
 794                 return ret;
 795
 796         if (new_oi.contentp) {
 797                 struct strbuf outbuf = STRBUF_INIT;
 798
 799                 if (type != OBJ_BLOB) {
 800                         ret = convert_object_file(r, &outbuf,
 801                                                   r->hash_algo, input_algo,
 802                                                   content, size, type, !do_die);
 803                         free(content);
 804                         if (ret == -1)
 805                                 return -1;
 806                         size = outbuf.len;
 807                         content = strbuf_detach(&outbuf, NULL);
 808                 }
 809                 if (input_oi->sizep)
 810                         *input_oi->sizep = size;
 811                 if (input_oi->contentp)
 812                         *input_oi->contentp = content;
 813                 else
 814                         free(content);
 815                 if (input_oi->typep)
 816                         *input_oi->typep = type;
 817         }
 818         if (new_oi.delta_base_oid == &delta_base_oid) {
 819                 if (repo_oid_to_algop(r, &delta_base_oid, input_algo,
 820                                  input_oi->delta_base_oid)) {
 821                         if (do_die)
 822                                 die(_("missing mapping of %s to %s"),
 823                                     oid_to_hex(&delta_base_oid),
 824                                     input_algo->name);
 825                         return -1;
 826                 }
 827         }
 828         input_oi->whence = new_oi.whence;
 829         input_oi->u = new_oi.u;
 830         return ret;
 831 }
 832
 833 int odb_read_object_info_extended(struct object_database *odb,
 834                                   const struct object_id *oid,
 835                                   struct object_info *oi,
 836                                   unsigned flags)
 837 {
 838         int ret;
 839
 840         if (oid->algo && (hash_algo_by_ptr(odb->repo->hash_algo) != oid->algo))
 841                 return oid_object_info_convert(odb->repo, oid, oi, flags);
 842
 843         obj_read_lock();
 844         ret = do_oid_object_info_extended(odb, oid, oi, flags);
 845         obj_read_unlock();
 846         return ret;
 847 }
 848
 849
 850 /* returns enum object_type or negative */
 851 int odb_read_object_info(struct object_database *odb,
 852                          const struct object_id *oid,
 853                          unsigned long *sizep)
 854 {
 855         enum object_type type;
 856         struct object_info oi = OBJECT_INFO_INIT;
 857
 858         oi.typep = &type;
 859         oi.sizep = sizep;
 860         if (odb_read_object_info_extended(odb, oid, &oi,
 861                                           OBJECT_INFO_LOOKUP_REPLACE) < 0)
 862                 return -1;
 863         return type;
 864 }
 865
 866 int odb_pretend_object(struct object_database *odb,
 867                        void *buf, unsigned long len, enum object_type type,
 868                        struct object_id *oid)
 869 {
 870         struct cached_object_entry *co;
 871         char *co_buf;
 872
 873         hash_object_file(odb->repo->hash_algo, buf, len, type, oid);
 874         if (odb_has_object(odb, oid, 0) ||
 875             find_cached_object(odb, oid))
 876                 return 0;
 877
 878         ALLOC_GROW(odb->cached_objects,
 879                    odb->cached_object_nr + 1, odb->cached_object_alloc);
 880         co = &odb->cached_objects[odb->cached_object_nr++];
 881         co->value.size = len;
 882         co->value.type = type;
 883         co_buf = xmalloc(len);
 884         memcpy(co_buf, buf, len);
 885         co->value.buf = co_buf;
 886         oidcpy(&co->oid, oid);
 887         return 0;
 888 }
 889
 890 void *odb_read_object(struct object_database *odb,
 891                       const struct object_id *oid,
 892                       enum object_type *type,
 893                       unsigned long *size)
 894 {
 895         struct object_info oi = OBJECT_INFO_INIT;
 896         unsigned flags = OBJECT_INFO_DIE_IF_CORRUPT | OBJECT_INFO_LOOKUP_REPLACE;
 897         void *data;
 898
 899         oi.typep = type;
 900         oi.sizep = size;
 901         oi.contentp = &data;
 902         if (odb_read_object_info_extended(odb, oid, &oi, flags))
 903                 return NULL;
 904
 905         return data;
 906 }
 907
 908 void *odb_read_object_peeled(struct object_database *odb,
 909                              const struct object_id *oid,
 910                              enum object_type required_type,
 911                              unsigned long *size,
 912                              struct object_id *actual_oid_return)
 913 {
 914         enum object_type type;
 915         void *buffer;
 916         unsigned long isize;
 917         struct object_id actual_oid;
 918
 919         oidcpy(&actual_oid, oid);
 920         while (1) {
 921                 int ref_length = -1;
 922                 const char *ref_type = NULL;
 923
 924                 buffer = odb_read_object(odb, &actual_oid, &type, &isize);
 925                 if (!buffer)
 926                         return NULL;
 927                 if (type == required_type) {
 928                         *size = isize;
 929                         if (actual_oid_return)
 930                                 oidcpy(actual_oid_return, &actual_oid);
 931                         return buffer;
 932                 }
 933                 /* Handle references */
 934                 else if (type == OBJ_COMMIT)
 935                         ref_type = "tree ";
 936                 else if (type == OBJ_TAG)
 937                         ref_type = "object ";
 938                 else {
 939                         free(buffer);
 940                         return NULL;
 941                 }
 942                 ref_length = strlen(ref_type);
 943
 944                 if (ref_length + odb->repo->hash_algo->hexsz > isize ||
 945                     memcmp(buffer, ref_type, ref_length) ||
 946                     get_oid_hex_algop((char *) buffer + ref_length, &actual_oid,
 947                                       odb->repo->hash_algo)) {
 948                         free(buffer);
 949                         return NULL;
 950                 }
 951                 free(buffer);
 952                 /* Now we have the ID of the referred-to object in
 953                  * actual_oid.  Check again. */
 954         }
 955 }
 956
 957 int odb_has_object(struct object_database *odb, const struct object_id *oid,
 958                unsigned flags)
 959 {
 960         unsigned object_info_flags = 0;
 961
 962         if (!startup_info->have_repository)
 963                 return 0;
 964         if (!(flags & HAS_OBJECT_RECHECK_PACKED))
 965                 object_info_flags |= OBJECT_INFO_QUICK;
 966         if (!(flags & HAS_OBJECT_FETCH_PROMISOR))
 967                 object_info_flags |= OBJECT_INFO_SKIP_FETCH_OBJECT;
 968
 969         return odb_read_object_info_extended(odb, oid, NULL, object_info_flags) >= 0;
 970 }
 971
 972 void odb_assert_oid_type(struct object_database *odb,
 973                          const struct object_id *oid, enum object_type expect)
 974 {
 975         enum object_type type = odb_read_object_info(odb, oid, NULL);
 976         if (type < 0)
 977                 die(_("%s is not a valid object"), oid_to_hex(oid));
 978         if (type != expect)
 979                 die(_("%s is not a valid '%s' object"), oid_to_hex(oid),
 980                     type_name(expect));
 981 }
 982
 983 struct object_database *odb_new(struct repository *repo)
 984 {
 985         struct object_database *o = xmalloc(sizeof(*o));
 986
 987         memset(o, 0, sizeof(*o));
 988         o->repo = repo;
 989         INIT_LIST_HEAD(&o->packed_git_mru);
 990         hashmap_init(&o->pack_map, pack_map_entry_cmp, NULL, 0);
 991         pthread_mutex_init(&o->replace_mutex, NULL);
 992         string_list_init_dup(&o->submodule_source_paths);
 993         return o;
 994 }
 995
 996 static void free_object_directories(struct object_database *o)
 997 {
 998         while (o->sources) {
 999                 struct odb_source *next;
1000
1001                 next = o->sources->next;
1002                 free_object_directory(o->sources);
1003                 o->sources = next;
1004         }
1005         kh_destroy_odb_path_map(o->source_by_path);
1006         o->source_by_path = NULL;
1007 }
1008
1009 void odb_clear(struct object_database *o)
1010 {
1011         FREE_AND_NULL(o->alternate_db);
1012
1013         oidmap_clear(&o->replace_map, 1);
1014         pthread_mutex_destroy(&o->replace_mutex);
1015
1016         free_commit_graph(o->commit_graph);
1017         o->commit_graph = NULL;
1018         o->commit_graph_attempted = 0;
1019
1020         free_object_directories(o);
1021         o->sources_tail = NULL;
1022         o->loaded_alternates = 0;
1023
1024         for (size_t i = 0; i < o->cached_object_nr; i++)
1025                 free((char *) o->cached_objects[i].value.buf);
1026         FREE_AND_NULL(o->cached_objects);
1027
1028         INIT_LIST_HEAD(&o->packed_git_mru);
1029         close_object_store(o);
1030
1031         /*
1032          * `close_object_store()` only closes the packfiles, but doesn't free
1033          * them. We thus have to do this manually.
1034          */
1035         for (struct packed_git *p = o->packed_git, *next; p; p = next) {
1036                 next = p->next;
1037                 free(p);
1038         }
1039         o->packed_git = NULL;
1040
1041         hashmap_clear(&o->pack_map);
1042         string_list_clear(&o->submodule_source_paths, 0);
1043 }