bundle-uri.c

   1 #define USE_THE_REPOSITORY_VARIABLE
   2 #define DISABLE_SIGN_COMPARE_WARNINGS
   3
   4 #include "git-compat-util.h"
   5 #include "bundle-uri.h"
   6 #include "bundle.h"
   7 #include "copy.h"
   8 #include "gettext.h"
   9 #include "refs.h"
  10 #include "run-command.h"
  11 #include "hashmap.h"
  12 #include "pkt-line.h"
  13 #include "config.h"
  14 #include "fetch-pack.h"
  15 #include "remote.h"
  16 #include "trace2.h"
  17 #include "odb.h"
  18
  19 static struct {
  20         enum bundle_list_heuristic heuristic;
  21         const char *name;
  22 } heuristics[BUNDLE_HEURISTIC__COUNT] = {
  23         { BUNDLE_HEURISTIC_NONE, ""},
  24         { BUNDLE_HEURISTIC_CREATIONTOKEN, "creationToken" },
  25 };
  26
  27 static int compare_bundles(const void *hashmap_cmp_fn_data UNUSED,
  28                            const struct hashmap_entry *he1,
  29                            const struct hashmap_entry *he2,
  30                            const void *id)
  31 {
  32         const struct remote_bundle_info *e1 =
  33                 container_of(he1, const struct remote_bundle_info, ent);
  34         const struct remote_bundle_info *e2 =
  35                 container_of(he2, const struct remote_bundle_info, ent);
  36
  37         return strcmp(e1->id, id ? (const char *)id : e2->id);
  38 }
  39
  40 void init_bundle_list(struct bundle_list *list)
  41 {
  42         memset(list, 0, sizeof(*list));
  43
  44         /* Implied defaults. */
  45         list->mode = BUNDLE_MODE_ALL;
  46         list->version = 1;
  47
  48         hashmap_init(&list->bundles, compare_bundles, NULL, 0);
  49 }
  50
  51 static int clear_remote_bundle_info(struct remote_bundle_info *bundle,
  52                                     void *data UNUSED)
  53 {
  54         FREE_AND_NULL(bundle->id);
  55         FREE_AND_NULL(bundle->uri);
  56         FREE_AND_NULL(bundle->file);
  57         bundle->unbundled = 0;
  58         return 0;
  59 }
  60
  61 void clear_bundle_list(struct bundle_list *list)
  62 {
  63         if (!list)
  64                 return;
  65
  66         for_all_bundles_in_list(list, clear_remote_bundle_info, NULL);
  67         hashmap_clear_and_free(&list->bundles, struct remote_bundle_info, ent);
  68         free(list->baseURI);
  69 }
  70
  71 int for_all_bundles_in_list(struct bundle_list *list,
  72                             bundle_iterator iter,
  73                             void *data)
  74 {
  75         struct remote_bundle_info *info;
  76         struct hashmap_iter i;
  77
  78         hashmap_for_each_entry(&list->bundles, &i, info, ent) {
  79                 int result = iter(info, data);
  80
  81                 if (result)
  82                         return result;
  83         }
  84
  85         return 0;
  86 }
  87
  88 static int summarize_bundle(struct remote_bundle_info *info, void *data)
  89 {
  90         FILE *fp = data;
  91         fprintf(fp, "[bundle \"%s\"]\n", info->id);
  92         fprintf(fp, "\turi = %s\n", info->uri);
  93
  94         if (info->creationToken)
  95                 fprintf(fp, "\tcreationToken = %"PRIu64"\n", info->creationToken);
  96         return 0;
  97 }
  98
  99 void print_bundle_list(FILE *fp, struct bundle_list *list)
 100 {
 101         const char *mode;
 102
 103         switch (list->mode) {
 104         case BUNDLE_MODE_ALL:
 105                 mode = "all";
 106                 break;
 107
 108         case BUNDLE_MODE_ANY:
 109                 mode = "any";
 110                 break;
 111
 112         case BUNDLE_MODE_NONE:
 113         default:
 114                 mode = "<unknown>";
 115         }
 116
 117         fprintf(fp, "[bundle]\n");
 118         fprintf(fp, "\tversion = %d\n", list->version);
 119         fprintf(fp, "\tmode = %s\n", mode);
 120
 121         if (list->heuristic) {
 122                 int i;
 123                 for (i = 0; i < BUNDLE_HEURISTIC__COUNT; i++) {
 124                         if (heuristics[i].heuristic == list->heuristic) {
 125                                 fprintf(fp, "\theuristic = %s\n",
 126                                        heuristics[list->heuristic].name);
 127                                 break;
 128                         }
 129                 }
 130         }
 131
 132         for_all_bundles_in_list(list, summarize_bundle, fp);
 133 }
 134
 135 /**
 136  * Given a key-value pair, update the state of the given bundle list.
 137  * Returns 0 if the key-value pair is understood. Returns -1 if the key
 138  * is not understood or the value is malformed.
 139  */
 140 static int bundle_list_update(const char *key, const char *value,
 141                               struct bundle_list *list)
 142 {
 143         struct strbuf id = STRBUF_INIT;
 144         struct remote_bundle_info lookup = REMOTE_BUNDLE_INFO_INIT;
 145         struct remote_bundle_info *bundle;
 146         const char *subsection, *subkey;
 147         size_t subsection_len;
 148
 149         if (parse_config_key(key, "bundle", &subsection, &subsection_len, &subkey))
 150                 return -1;
 151
 152         if (!subsection_len) {
 153                 if (!strcmp(subkey, "version")) {
 154                         int version;
 155                         if (!git_parse_int(value, &version))
 156                                 return -1;
 157                         if (version != 1)
 158                                 return -1;
 159
 160                         list->version = version;
 161                         return 0;
 162                 }
 163
 164                 if (!strcmp(subkey, "mode")) {
 165                         if (!strcmp(value, "all"))
 166                                 list->mode = BUNDLE_MODE_ALL;
 167                         else if (!strcmp(value, "any"))
 168                                 list->mode = BUNDLE_MODE_ANY;
 169                         else
 170                                 return -1;
 171                         return 0;
 172                 }
 173
 174                 if (!strcmp(subkey, "heuristic")) {
 175                         int i;
 176                         for (i = 0; i < BUNDLE_HEURISTIC__COUNT; i++) {
 177                                 if (heuristics[i].heuristic &&
 178                                     heuristics[i].name &&
 179                                     !strcmp(value, heuristics[i].name)) {
 180                                         list->heuristic = heuristics[i].heuristic;
 181                                         return 0;
 182                                 }
 183                         }
 184
 185                         /* Ignore unknown heuristics. */
 186                         return 0;
 187                 }
 188
 189                 /* Ignore other unknown global keys. */
 190                 return 0;
 191         }
 192
 193         strbuf_add(&id, subsection, subsection_len);
 194
 195         /*
 196          * Check for an existing bundle with this <id>, or create one
 197          * if necessary.
 198          */
 199         lookup.id = id.buf;
 200         hashmap_entry_init(&lookup.ent, strhash(lookup.id));
 201         if (!(bundle = hashmap_get_entry(&list->bundles, &lookup, ent, NULL))) {
 202                 CALLOC_ARRAY(bundle, 1);
 203                 bundle->id = strbuf_detach(&id, NULL);
 204                 hashmap_entry_init(&bundle->ent, strhash(bundle->id));
 205                 hashmap_add(&list->bundles, &bundle->ent);
 206         }
 207         strbuf_release(&id);
 208
 209         if (!strcmp(subkey, "uri")) {
 210                 if (bundle->uri)
 211                         return -1;
 212                 bundle->uri = relative_url(list->baseURI, value, NULL);
 213                 return 0;
 214         }
 215
 216         if (!strcmp(subkey, "creationtoken")) {
 217                 if (sscanf(value, "%"PRIu64, &bundle->creationToken) != 1)
 218                         warning(_("could not parse bundle list key %s with value '%s'"),
 219                                 "creationToken", value);
 220                 return 0;
 221         }
 222
 223         /*
 224          * At this point, we ignore any information that we don't
 225          * understand, assuming it to be hints for a heuristic the client
 226          * does not currently understand.
 227          */
 228         return 0;
 229 }
 230
 231 static int config_to_bundle_list(const char *key, const char *value,
 232                                  const struct config_context *ctx UNUSED,
 233                                  void *data)
 234 {
 235         struct bundle_list *list = data;
 236         return bundle_list_update(key, value, list);
 237 }
 238
 239 int bundle_uri_parse_config_format(const char *uri,
 240                                    const char *filename,
 241                                    struct bundle_list *list)
 242 {
 243         int result;
 244         struct config_options opts = {
 245                 .error_action = CONFIG_ERROR_ERROR,
 246         };
 247
 248         if (!list->baseURI) {
 249                 struct strbuf baseURI = STRBUF_INIT;
 250                 strbuf_addstr(&baseURI, uri);
 251
 252                 /*
 253                  * If the URI does not end with a trailing slash, then
 254                  * remove the filename portion of the path. This is
 255                  * important for relative URIs.
 256                  */
 257                 strbuf_strip_file_from_path(&baseURI);
 258                 list->baseURI = strbuf_detach(&baseURI, NULL);
 259         }
 260         result = git_config_from_file_with_options(config_to_bundle_list,
 261                                                    filename, list,
 262                                                    CONFIG_SCOPE_UNKNOWN,
 263                                                    &opts);
 264
 265         if (!result && list->mode == BUNDLE_MODE_NONE) {
 266                 warning(_("bundle list at '%s' has no mode"), uri);
 267                 result = 1;
 268         }
 269
 270         return result;
 271 }
 272
 273 static char *find_temp_filename(void)
 274 {
 275         int fd;
 276         struct strbuf name = STRBUF_INIT;
 277         /*
 278          * Find a temporary filename that is available. This is briefly
 279          * racy, but unlikely to collide.
 280          */
 281         fd = odb_mkstemp(the_repository->objects, &name,
 282                          "bundles/tmp_uri_XXXXXX");
 283         if (fd < 0) {
 284                 warning(_("failed to create temporary file"));
 285                 return NULL;
 286         }
 287
 288         close(fd);
 289         unlink(name.buf);
 290         return strbuf_detach(&name, NULL);
 291 }
 292
 293 static int download_https_uri_to_file(const char *file, const char *uri)
 294 {
 295         int result = 0;
 296         struct child_process cp = CHILD_PROCESS_INIT;
 297         FILE *child_in = NULL, *child_out = NULL;
 298         struct strbuf line = STRBUF_INIT;
 299         int found_get = 0;
 300
 301         /*
 302          * The protocol we speak with git-remote-https(1) uses a space to
 303          * separate between URI and file, so the URI itself must not contain a
 304          * space. If it did, an adversary could change the location where the
 305          * downloaded file is being written to.
 306          *
 307          * Similarly, we use newlines to separate commands from one another.
 308          * Consequently, neither the URI nor the file must contain a newline or
 309          * otherwise an adversary could inject arbitrary commands.
 310          *
 311          * TODO: Restricting newlines in the target paths may break valid
 312          *       usecases, even if those are a bit more on the esoteric side.
 313          *       If this ever becomes a problem we should probably think about
 314          *       alternatives. One alternative could be to use NUL-delimited
 315          *       requests in git-remote-http(1). Another alternative could be
 316          *       to use URL quoting.
 317          */
 318         if (strpbrk(uri, " \n"))
 319                 return error("bundle-uri: URI is malformed: '%s'", file);
 320         if (strchr(file, '\n'))
 321                 return error("bundle-uri: filename is malformed: '%s'", file);
 322
 323         strvec_pushl(&cp.args, "git-remote-https", uri, NULL);
 324         cp.err = -1;
 325         cp.in = -1;
 326         cp.out = -1;
 327
 328         if (start_command(&cp))
 329                 return 1;
 330
 331         child_in = fdopen(cp.in, "w");
 332         if (!child_in) {
 333                 result = 1;
 334                 goto cleanup;
 335         }
 336
 337         child_out = fdopen(cp.out, "r");
 338         if (!child_out) {
 339                 result = 1;
 340                 goto cleanup;
 341         }
 342
 343         fprintf(child_in, "capabilities\n");
 344         fflush(child_in);
 345
 346         while (!strbuf_getline(&line, child_out)) {
 347                 if (!line.len)
 348                         break;
 349                 if (!strcmp(line.buf, "get"))
 350                         found_get = 1;
 351         }
 352         strbuf_release(&line);
 353
 354         if (!found_get) {
 355                 result = error(_("insufficient capabilities"));
 356                 goto cleanup;
 357         }
 358
 359         fprintf(child_in, "get %s %s\n\n", uri, file);
 360
 361 cleanup:
 362         if (child_in)
 363                 fclose(child_in);
 364         if (finish_command(&cp))
 365                 return 1;
 366         if (child_out)
 367                 fclose(child_out);
 368         return result;
 369 }
 370
 371 static int copy_uri_to_file(const char *filename, const char *uri)
 372 {
 373         const char *out;
 374
 375         if (starts_with(uri, "https:") ||
 376             starts_with(uri, "http:"))
 377                 return download_https_uri_to_file(filename, uri);
 378
 379         if (skip_prefix(uri, "file://", &out))
 380                 uri = out;
 381
 382         /* Copy as a file */
 383         return copy_file(filename, uri, 0);
 384 }
 385
 386 static int unbundle_from_file(struct repository *r, const char *file)
 387 {
 388         int result = 0;
 389         int bundle_fd;
 390         struct bundle_header header = BUNDLE_HEADER_INIT;
 391         struct string_list_item *refname;
 392         struct strbuf bundle_ref = STRBUF_INIT;
 393         size_t bundle_prefix_len;
 394         struct unbundle_opts opts = {
 395                 .flags = VERIFY_BUNDLE_QUIET |
 396                          (fetch_pack_fsck_objects() ? VERIFY_BUNDLE_FSCK : 0),
 397         };
 398
 399         bundle_fd = read_bundle_header(file, &header);
 400         if (bundle_fd < 0) {
 401                 result = 1;
 402                 goto cleanup;
 403         }
 404
 405         /*
 406          * Skip the reachability walk here, since we will be adding
 407          * a reachable ref pointing to the new tips, which will reach
 408          * the prerequisite commits.
 409          */
 410         result = unbundle(r, &header, bundle_fd, NULL, &opts);
 411         if (result) {
 412                 result = 1;
 413                 goto cleanup;
 414         }
 415
 416         /*
 417          * Convert all refs/heads/ from the bundle into refs/bundles/
 418          * in the local repository.
 419          */
 420         strbuf_addstr(&bundle_ref, "refs/bundles/");
 421         bundle_prefix_len = bundle_ref.len;
 422
 423         for_each_string_list_item(refname, &header.references) {
 424                 struct object_id *oid = refname->util;
 425                 struct object_id old_oid;
 426                 const char *branch_name;
 427                 int has_old;
 428
 429                 if (!skip_prefix(refname->string, "refs/", &branch_name))
 430                         continue;
 431
 432                 strbuf_setlen(&bundle_ref, bundle_prefix_len);
 433                 strbuf_addstr(&bundle_ref, branch_name);
 434
 435                 has_old = !refs_read_ref(get_main_ref_store(the_repository),
 436                                          bundle_ref.buf, &old_oid);
 437                 refs_update_ref(get_main_ref_store(the_repository),
 438                                 "fetched bundle", bundle_ref.buf, oid,
 439                                 has_old ? &old_oid : NULL,
 440                                 0, UPDATE_REFS_MSG_ON_ERR);
 441         }
 442
 443 cleanup:
 444         strbuf_release(&bundle_ref);
 445         bundle_header_release(&header);
 446         return result;
 447 }
 448
 449 struct bundle_list_context {
 450         struct repository *r;
 451         struct bundle_list *list;
 452         enum bundle_list_mode mode;
 453         int count;
 454         int depth;
 455 };
 456
 457 /*
 458  * This early definition is necessary because we use indirect recursion:
 459  *
 460  * While iterating through a bundle list that was downloaded as part
 461  * of fetch_bundle_uri_internal(), iterator methods eventually call it
 462  * again, but with depth + 1.
 463  */
 464 static int fetch_bundle_uri_internal(struct repository *r,
 465                                      struct remote_bundle_info *bundle,
 466                                      int depth,
 467                                      struct bundle_list *list);
 468
 469 static int download_bundle_to_file(struct remote_bundle_info *bundle, void *data)
 470 {
 471         int res;
 472         struct bundle_list_context *ctx = data;
 473
 474         if (ctx->mode == BUNDLE_MODE_ANY && ctx->count)
 475                 return 0;
 476
 477         res = fetch_bundle_uri_internal(ctx->r, bundle, ctx->depth + 1, ctx->list);
 478
 479         /*
 480          * Only increment count if the download succeeded. If our mode is
 481          * BUNDLE_MODE_ANY, then we will want to try other URIs in the
 482          * list in case they work instead.
 483          */
 484         if (!res)
 485                 ctx->count++;
 486
 487         /*
 488          * To be opportunistic as possible, we continue iterating and
 489          * download as many bundles as we can, so we can apply the ones
 490          * that work, even in BUNDLE_MODE_ALL mode.
 491          */
 492         return 0;
 493 }
 494
 495 struct bundles_for_sorting {
 496         struct remote_bundle_info **items;
 497         size_t alloc;
 498         size_t nr;
 499 };
 500
 501 static int append_bundle(struct remote_bundle_info *bundle, void *data)
 502 {
 503         struct bundles_for_sorting *list = data;
 504         list->items[list->nr++] = bundle;
 505         return 0;
 506 }
 507
 508 /**
 509  * For use in QSORT() to get a list sorted by creationToken
 510  * in decreasing order.
 511  */
 512 static int compare_creation_token_decreasing(const void *va, const void *vb)
 513 {
 514         const struct remote_bundle_info * const *a = va;
 515         const struct remote_bundle_info * const *b = vb;
 516
 517         if ((*a)->creationToken > (*b)->creationToken)
 518                 return -1;
 519         if ((*a)->creationToken < (*b)->creationToken)
 520                 return 1;
 521         return 0;
 522 }
 523
 524 static int fetch_bundles_by_token(struct repository *r,
 525                                   struct bundle_list *list)
 526 {
 527         int cur;
 528         int move_direction = 0;
 529         const char *creationTokenStr;
 530         uint64_t maxCreationToken = 0, newMaxCreationToken = 0;
 531         struct bundle_list_context ctx = {
 532                 .r = r,
 533                 .list = list,
 534                 .mode = list->mode,
 535         };
 536         struct bundles_for_sorting bundles = {
 537                 .alloc = hashmap_get_size(&list->bundles),
 538         };
 539
 540         ALLOC_ARRAY(bundles.items, bundles.alloc);
 541
 542         for_all_bundles_in_list(list, append_bundle, &bundles);
 543
 544         if (!bundles.nr) {
 545                 free(bundles.items);
 546                 return 0;
 547         }
 548
 549         QSORT(bundles.items, bundles.nr, compare_creation_token_decreasing);
 550
 551         /*
 552          * If fetch.bundleCreationToken exists, parses to a uint64t, and
 553          * is not strictly smaller than the maximum creation token in the
 554          * bundle list, then do not download any bundles.
 555          */
 556         if (!repo_config_get_value(r,
 557                                    "fetch.bundlecreationtoken",
 558                                    &creationTokenStr)) {
 559                 if (sscanf(creationTokenStr, "%"PRIu64, &maxCreationToken) != 1)
 560                         maxCreationToken = 0;
 561                 if (bundles.items[0]->creationToken <= maxCreationToken) {
 562                         free(bundles.items);
 563                         return 0;
 564                 }
 565         }
 566
 567         /*
 568          * Attempt to download and unbundle the minimum number of bundles by
 569          * creationToken in decreasing order. If we fail to unbundle (after
 570          * a successful download) then move to the next non-downloaded bundle
 571          * and attempt downloading. Once we succeed in applying a bundle,
 572          * move to the previous unapplied bundle and attempt to unbundle it
 573          * again.
 574          *
 575          * In the case of a fresh clone, we will likely download all of the
 576          * bundles before successfully unbundling the oldest one, then the
 577          * rest of the bundles unbundle successfully in increasing order
 578          * of creationToken.
 579          *
 580          * If there are existing objects, then this process may terminate
 581          * early when all required commits from "new" bundles exist in the
 582          * repo's object store.
 583          */
 584         cur = 0;
 585         while (cur >= 0 && cur < bundles.nr) {
 586                 struct remote_bundle_info *bundle = bundles.items[cur];
 587
 588                 /*
 589                  * If we need to dig into bundles below the previous
 590                  * creation token value, then likely we are in an erroneous
 591                  * state due to missing or invalid bundles. Halt the process
 592                  * instead of continuing to download extra data.
 593                  */
 594                 if (bundle->creationToken <= maxCreationToken)
 595                         break;
 596
 597                 if (!bundle->file) {
 598                         /*
 599                          * Not downloaded yet. Try downloading.
 600                          *
 601                          * Note that bundle->file is non-NULL if a download
 602                          * was attempted, even if it failed to download.
 603                          */
 604                         if (fetch_bundle_uri_internal(ctx.r, bundle, ctx.depth + 1, ctx.list)) {
 605                                 /* Mark as unbundled so we do not retry. */
 606                                 bundle->unbundled = 1;
 607
 608                                 /* Try looking deeper in the list. */
 609                                 move_direction = 1;
 610                                 goto move;
 611                         }
 612
 613                         /* We expect bundles when using creationTokens. */
 614                         if (!is_bundle(bundle->file, 1)) {
 615                                 warning(_("file downloaded from '%s' is not a bundle"),
 616                                         bundle->uri);
 617                                 break;
 618                         }
 619                 }
 620
 621                 if (bundle->file && !bundle->unbundled) {
 622                         /*
 623                          * This was downloaded, but not successfully
 624                          * unbundled. Try unbundling again.
 625                          */
 626                         if (unbundle_from_file(ctx.r, bundle->file)) {
 627                                 /* Try looking deeper in the list. */
 628                                 move_direction = 1;
 629                         } else {
 630                                 /*
 631                                  * Succeeded in unbundle. Retry bundles
 632                                  * that previously failed to unbundle.
 633                                  */
 634                                 move_direction = -1;
 635                                 bundle->unbundled = 1;
 636
 637                                 if (bundle->creationToken > newMaxCreationToken)
 638                                         newMaxCreationToken = bundle->creationToken;
 639                         }
 640                 }
 641
 642                 /*
 643                  * Else case: downloaded and unbundled successfully.
 644                  * Skip this by moving in the same direction as the
 645                  * previous step.
 646                  */
 647
 648 move:
 649                 /* Move in the specified direction and repeat. */
 650                 cur += move_direction;
 651         }
 652
 653         /*
 654          * We succeed if the loop terminates because 'cur' drops below
 655          * zero. The other case is that we terminate because 'cur'
 656          * reaches the end of the list, so we have a failure no matter
 657          * which bundles we apply from the list.
 658          */
 659         if (cur < 0) {
 660                 struct strbuf value = STRBUF_INIT;
 661                 strbuf_addf(&value, "%"PRIu64"", newMaxCreationToken);
 662                 if (repo_config_set_multivar_gently(ctx.r,
 663                                                     "fetch.bundleCreationToken",
 664                                                     value.buf, NULL, 0))
 665                         warning(_("failed to store maximum creation token"));
 666
 667                 strbuf_release(&value);
 668         }
 669
 670         free(bundles.items);
 671         return cur >= 0;
 672 }
 673
 674 static int download_bundle_list(struct repository *r,
 675                                 struct bundle_list *local_list,
 676                                 struct bundle_list *global_list,
 677                                 int depth)
 678 {
 679         struct bundle_list_context ctx = {
 680                 .r = r,
 681                 .list = global_list,
 682                 .depth = depth + 1,
 683                 .mode = local_list->mode,
 684         };
 685
 686         return for_all_bundles_in_list(local_list, download_bundle_to_file, &ctx);
 687 }
 688
 689 static int fetch_bundle_list_in_config_format(struct repository *r,
 690                                               struct bundle_list *global_list,
 691                                               struct remote_bundle_info *bundle,
 692                                               int depth)
 693 {
 694         int result;
 695         struct bundle_list list_from_bundle;
 696
 697         init_bundle_list(&list_from_bundle);
 698
 699         if ((result = bundle_uri_parse_config_format(bundle->uri,
 700                                                      bundle->file,
 701                                                      &list_from_bundle)))
 702                 goto cleanup;
 703
 704         if (list_from_bundle.mode == BUNDLE_MODE_NONE) {
 705                 warning(_("unrecognized bundle mode from URI '%s'"),
 706                         bundle->uri);
 707                 result = -1;
 708                 goto cleanup;
 709         }
 710
 711         /*
 712          * If this list uses the creationToken heuristic, then the URIs
 713          * it advertises are expected to be bundles, not nested lists.
 714          * We can drop 'global_list' and 'depth'.
 715          */
 716         if (list_from_bundle.heuristic == BUNDLE_HEURISTIC_CREATIONTOKEN) {
 717                 result = fetch_bundles_by_token(r, &list_from_bundle);
 718                 global_list->heuristic = BUNDLE_HEURISTIC_CREATIONTOKEN;
 719         } else if ((result = download_bundle_list(r, &list_from_bundle,
 720                                            global_list, depth)))
 721                 goto cleanup;
 722
 723 cleanup:
 724         clear_bundle_list(&list_from_bundle);
 725         return result;
 726 }
 727
 728 /**
 729  * This limits the recursion on fetch_bundle_uri_internal() when following
 730  * bundle lists.
 731  */
 732 static int max_bundle_uri_depth = 4;
 733
 734 /**
 735  * Recursively download all bundles advertised at the given URI
 736  * to files. If the file is a bundle, then add it to the given
 737  * 'list'. Otherwise, expect a bundle list and recurse on the
 738  * URIs in that list according to the list mode (ANY or ALL).
 739  */
 740 static int fetch_bundle_uri_internal(struct repository *r,
 741                                      struct remote_bundle_info *bundle,
 742                                      int depth,
 743                                      struct bundle_list *list)
 744 {
 745         int result = 0;
 746         struct remote_bundle_info *bcopy;
 747
 748         if (depth >= max_bundle_uri_depth) {
 749                 warning(_("exceeded bundle URI recursion limit (%d)"),
 750                         max_bundle_uri_depth);
 751                 return -1;
 752         }
 753
 754         if (!bundle->file &&
 755             !(bundle->file = find_temp_filename())) {
 756                 result = -1;
 757                 goto cleanup;
 758         }
 759
 760         if ((result = copy_uri_to_file(bundle->file, bundle->uri))) {
 761                 warning(_("failed to download bundle from URI '%s'"), bundle->uri);
 762                 goto cleanup;
 763         }
 764
 765         if ((result = !is_bundle(bundle->file, 1))) {
 766                 result = fetch_bundle_list_in_config_format(
 767                                 r, list, bundle, depth);
 768                 if (result)
 769                         warning(_("file at URI '%s' is not a bundle or bundle list"),
 770                                 bundle->uri);
 771                 goto cleanup;
 772         }
 773
 774         /* Copy the bundle and insert it into the global list. */
 775         CALLOC_ARRAY(bcopy, 1);
 776         bcopy->id = xstrdup(bundle->id);
 777         bcopy->file = xstrdup(bundle->file);
 778         hashmap_entry_init(&bcopy->ent, strhash(bcopy->id));
 779         hashmap_add(&list->bundles, &bcopy->ent);
 780
 781 cleanup:
 782         if (result && bundle->file)
 783                 unlink(bundle->file);
 784         return result;
 785 }
 786
 787 /**
 788  * This loop iterator breaks the loop with nonzero return code on the
 789  * first successful unbundling of a bundle.
 790  */
 791 static int attempt_unbundle(struct remote_bundle_info *info, void *data)
 792 {
 793         struct repository *r = data;
 794
 795         if (!info->file || info->unbundled)
 796                 return 0;
 797
 798         if (!unbundle_from_file(r, info->file)) {
 799                 info->unbundled = 1;
 800                 return 1;
 801         }
 802
 803         return 0;
 804 }
 805
 806 static int unbundle_all_bundles(struct repository *r,
 807                                 struct bundle_list *list)
 808 {
 809         /*
 810          * Iterate through all bundles looking for ones that can
 811          * successfully unbundle. If any succeed, then perhaps another
 812          * will succeed in the next attempt.
 813          *
 814          * Keep in mind that a non-zero result for the loop here means
 815          * the loop terminated early on a successful unbundling, which
 816          * signals that we can try again.
 817          */
 818         while (for_all_bundles_in_list(list, attempt_unbundle, r)) ;
 819
 820         return 0;
 821 }
 822
 823 static int unlink_bundle(struct remote_bundle_info *info, void *data UNUSED)
 824 {
 825         if (info->file)
 826                 unlink_or_warn(info->file);
 827         return 0;
 828 }
 829
 830 int fetch_bundle_uri(struct repository *r, const char *uri,
 831                      int *has_heuristic)
 832 {
 833         int result;
 834         struct bundle_list list;
 835         struct remote_bundle_info bundle = {
 836                 .uri = xstrdup(uri),
 837                 .id = xstrdup(""),
 838         };
 839
 840         trace2_region_enter("fetch", "fetch-bundle-uri", the_repository);
 841
 842         init_bundle_list(&list);
 843
 844         /*
 845          * Do not fetch an empty bundle URI. An empty bundle URI
 846          * could signal that a configured bundle URI has been disabled.
 847          */
 848         if (!*uri) {
 849                 result = 0;
 850                 goto cleanup;
 851         }
 852
 853         /* If a bundle is added to this global list, then it is required. */
 854         list.mode = BUNDLE_MODE_ALL;
 855
 856         if ((result = fetch_bundle_uri_internal(r, &bundle, 0, &list)))
 857                 goto cleanup;
 858
 859         result = unbundle_all_bundles(r, &list);
 860
 861 cleanup:
 862         if (has_heuristic)
 863                 *has_heuristic = (list.heuristic != BUNDLE_HEURISTIC_NONE);
 864         for_all_bundles_in_list(&list, unlink_bundle, NULL);
 865         clear_bundle_list(&list);
 866         clear_remote_bundle_info(&bundle, NULL);
 867         trace2_region_leave("fetch", "fetch-bundle-uri", the_repository);
 868         return result;
 869 }
 870
 871 int fetch_bundle_list(struct repository *r, struct bundle_list *list)
 872 {
 873         int result;
 874         struct bundle_list global_list;
 875
 876         /*
 877          * If the creationToken heuristic is used, then the URIs
 878          * advertised by 'list' are not nested lists and instead
 879          * direct bundles. We do not need to use global_list.
 880          */
 881         if (list->heuristic == BUNDLE_HEURISTIC_CREATIONTOKEN)
 882                 return fetch_bundles_by_token(r, list);
 883
 884         init_bundle_list(&global_list);
 885
 886         /* If a bundle is added to this global list, then it is required. */
 887         global_list.mode = BUNDLE_MODE_ALL;
 888
 889         if ((result = download_bundle_list(r, list, &global_list, 0)))
 890                 goto cleanup;
 891
 892         if (list->heuristic == BUNDLE_HEURISTIC_CREATIONTOKEN)
 893                 result = fetch_bundles_by_token(r, list);
 894         else
 895                 result = unbundle_all_bundles(r, &global_list);
 896
 897 cleanup:
 898         for_all_bundles_in_list(&global_list, unlink_bundle, NULL);
 899         clear_bundle_list(&global_list);
 900         return result;
 901 }
 902
 903 /**
 904  * API for serve.c.
 905  */
 906
 907 int bundle_uri_advertise(struct repository *r, struct strbuf *value UNUSED)
 908 {
 909         static int advertise_bundle_uri = -1;
 910
 911         if (advertise_bundle_uri != -1)
 912                 goto cached;
 913
 914         advertise_bundle_uri = 0;
 915         repo_config_get_maybe_bool(r, "uploadpack.advertisebundleuris", &advertise_bundle_uri);
 916
 917 cached:
 918         return advertise_bundle_uri;
 919 }
 920
 921 static int config_to_packet_line(const char *key, const char *value,
 922                                  const struct config_context *ctx UNUSED,
 923                                  void *data)
 924 {
 925         struct packet_reader *writer = data;
 926
 927         if (starts_with(key, "bundle."))
 928                 packet_write_fmt(writer->fd, "%s=%s", key, value);
 929
 930         return 0;
 931 }
 932
 933 int bundle_uri_command(struct repository *r,
 934                        struct packet_reader *request)
 935 {
 936         struct packet_writer writer;
 937         packet_writer_init(&writer, 1);
 938
 939         while (packet_reader_read(request) == PACKET_READ_NORMAL)
 940                 die(_("bundle-uri: unexpected argument: '%s'"), request->line);
 941         if (request->status != PACKET_READ_FLUSH)
 942                 die(_("bundle-uri: expected flush after arguments"));
 943
 944         /*
 945          * Read all "bundle.*" config lines to the client as key=value
 946          * packet lines.
 947          */
 948         repo_config(r, config_to_packet_line, &writer);
 949
 950         packet_writer_flush(&writer);
 951
 952         return 0;
 953 }
 954
 955 /**
 956  * General API for {transport,connect}.c etc.
 957  */
 958 int bundle_uri_parse_line(struct bundle_list *list, const char *line)
 959 {
 960         int result;
 961         const char *equals;
 962         struct strbuf key = STRBUF_INIT;
 963
 964         if (!strlen(line))
 965                 return error(_("bundle-uri: got an empty line"));
 966
 967         equals = strchr(line, '=');
 968
 969         if (!equals)
 970                 return error(_("bundle-uri: line is not of the form 'key=value'"));
 971         if (line == equals || !*(equals + 1))
 972                 return error(_("bundle-uri: line has empty key or value"));
 973
 974         strbuf_add(&key, line, equals - line);
 975         result = bundle_list_update(key.buf, equals + 1, list);
 976         strbuf_release(&key);
 977
 978         return result;
 979 }