promisor-remote.c

   1 #define USE_THE_REPOSITORY_VARIABLE
   2
   3 #include "git-compat-util.h"
   4 #include "gettext.h"
   5 #include "hex.h"
   6 #include "odb.h"
   7 #include "promisor-remote.h"
   8 #include "config.h"
   9 #include "trace2.h"
  10 #include "transport.h"
  11 #include "strvec.h"
  12 #include "packfile.h"
  13 #include "environment.h"
  14 #include "url.h"
  15 #include "version.h"
  16
  17 struct promisor_remote_config {
  18         struct promisor_remote *promisors;
  19         struct promisor_remote **promisors_tail;
  20 };
  21
  22 static int fetch_objects(struct repository *repo,
  23                          const char *remote_name,
  24                          const struct object_id *oids,
  25                          int oid_nr)
  26 {
  27         struct child_process child = CHILD_PROCESS_INIT;
  28         int i;
  29         FILE *child_in;
  30         int quiet;
  31
  32         if (git_env_bool(NO_LAZY_FETCH_ENVIRONMENT, 0)) {
  33                 static int warning_shown;
  34                 if (!warning_shown) {
  35                         warning_shown = 1;
  36                         warning(_("lazy fetching disabled; some objects may not be available"));
  37                 }
  38                 return -1;
  39         }
  40
  41         child.git_cmd = 1;
  42         child.in = -1;
  43         if (repo != the_repository)
  44                 prepare_other_repo_env(&child.env, repo->gitdir);
  45         strvec_pushl(&child.args, "-c", "fetch.negotiationAlgorithm=noop",
  46                      "fetch", remote_name, "--no-tags",
  47                      "--no-write-fetch-head", "--recurse-submodules=no",
  48                      "--filter=blob:none", "--stdin", NULL);
  49         if (!git_config_get_bool("promisor.quiet", &quiet) && quiet)
  50                 strvec_push(&child.args, "--quiet");
  51         if (start_command(&child))
  52                 die(_("promisor-remote: unable to fork off fetch subprocess"));
  53         child_in = xfdopen(child.in, "w");
  54
  55         trace2_data_intmax("promisor", repo, "fetch_count", oid_nr);
  56
  57         for (i = 0; i < oid_nr; i++) {
  58                 if (fputs(oid_to_hex(&oids[i]), child_in) < 0)
  59                         die_errno(_("promisor-remote: could not write to fetch subprocess"));
  60                 if (fputc('\n', child_in) < 0)
  61                         die_errno(_("promisor-remote: could not write to fetch subprocess"));
  62         }
  63
  64         if (fclose(child_in) < 0)
  65                 die_errno(_("promisor-remote: could not close stdin to fetch subprocess"));
  66         return finish_command(&child) ? -1 : 0;
  67 }
  68
  69 static struct promisor_remote *promisor_remote_new(struct promisor_remote_config *config,
  70                                                    const char *remote_name)
  71 {
  72         struct promisor_remote *r;
  73
  74         if (*remote_name == '/') {
  75                 warning(_("promisor remote name cannot begin with '/': %s"),
  76                         remote_name);
  77                 return NULL;
  78         }
  79
  80         FLEX_ALLOC_STR(r, name, remote_name);
  81
  82         *config->promisors_tail = r;
  83         config->promisors_tail = &r->next;
  84
  85         return r;
  86 }
  87
  88 static struct promisor_remote *promisor_remote_lookup(struct promisor_remote_config *config,
  89                                                       const char *remote_name,
  90                                                       struct promisor_remote **previous)
  91 {
  92         struct promisor_remote *r, *p;
  93
  94         for (p = NULL, r = config->promisors; r; p = r, r = r->next)
  95                 if (!strcmp(r->name, remote_name)) {
  96                         if (previous)
  97                                 *previous = p;
  98                         return r;
  99                 }
 100
 101         return NULL;
 102 }
 103
 104 static void promisor_remote_move_to_tail(struct promisor_remote_config *config,
 105                                          struct promisor_remote *r,
 106                                          struct promisor_remote *previous)
 107 {
 108         if (!r->next)
 109                 return;
 110
 111         if (previous)
 112                 previous->next = r->next;
 113         else
 114                 config->promisors = r->next ? r->next : r;
 115         r->next = NULL;
 116         *config->promisors_tail = r;
 117         config->promisors_tail = &r->next;
 118 }
 119
 120 static int promisor_remote_config(const char *var, const char *value,
 121                                   const struct config_context *ctx UNUSED,
 122                                   void *data)
 123 {
 124         struct promisor_remote_config *config = data;
 125         const char *name;
 126         size_t namelen;
 127         const char *subkey;
 128
 129         if (parse_config_key(var, "remote", &name, &namelen, &subkey) < 0)
 130                 return 0;
 131
 132         if (!strcmp(subkey, "promisor")) {
 133                 char *remote_name;
 134
 135                 if (!git_config_bool(var, value))
 136                         return 0;
 137
 138                 remote_name = xmemdupz(name, namelen);
 139
 140                 if (!promisor_remote_lookup(config, remote_name, NULL))
 141                         promisor_remote_new(config, remote_name);
 142
 143                 free(remote_name);
 144                 return 0;
 145         }
 146         if (!strcmp(subkey, "partialclonefilter")) {
 147                 struct promisor_remote *r;
 148                 char *remote_name = xmemdupz(name, namelen);
 149
 150                 r = promisor_remote_lookup(config, remote_name, NULL);
 151                 if (!r)
 152                         r = promisor_remote_new(config, remote_name);
 153
 154                 free(remote_name);
 155
 156                 if (!r)
 157                         return 0;
 158
 159                 FREE_AND_NULL(r->partial_clone_filter);
 160                 return git_config_string(&r->partial_clone_filter, var, value);
 161         }
 162
 163         return 0;
 164 }
 165
 166 static void promisor_remote_init(struct repository *r)
 167 {
 168         struct promisor_remote_config *config;
 169
 170         if (r->promisor_remote_config)
 171                 return;
 172         config = r->promisor_remote_config =
 173                 xcalloc(1, sizeof(*r->promisor_remote_config));
 174         config->promisors_tail = &config->promisors;
 175
 176         repo_config(r, promisor_remote_config, config);
 177
 178         if (r->repository_format_partial_clone) {
 179                 struct promisor_remote *o, *previous;
 180
 181                 o = promisor_remote_lookup(config,
 182                                            r->repository_format_partial_clone,
 183                                            &previous);
 184                 if (o)
 185                         promisor_remote_move_to_tail(config, o, previous);
 186                 else
 187                         promisor_remote_new(config, r->repository_format_partial_clone);
 188         }
 189 }
 190
 191 void promisor_remote_clear(struct promisor_remote_config *config)
 192 {
 193         while (config->promisors) {
 194                 struct promisor_remote *r = config->promisors;
 195                 free(r->partial_clone_filter);
 196                 config->promisors = config->promisors->next;
 197                 free(r);
 198         }
 199
 200         config->promisors_tail = &config->promisors;
 201 }
 202
 203 void repo_promisor_remote_reinit(struct repository *r)
 204 {
 205         promisor_remote_clear(r->promisor_remote_config);
 206         FREE_AND_NULL(r->promisor_remote_config);
 207         promisor_remote_init(r);
 208 }
 209
 210 struct promisor_remote *repo_promisor_remote_find(struct repository *r,
 211                                                   const char *remote_name)
 212 {
 213         promisor_remote_init(r);
 214
 215         if (!remote_name)
 216                 return r->promisor_remote_config->promisors;
 217
 218         return promisor_remote_lookup(r->promisor_remote_config, remote_name, NULL);
 219 }
 220
 221 int repo_has_promisor_remote(struct repository *r)
 222 {
 223         return !!repo_promisor_remote_find(r, NULL);
 224 }
 225
 226 int repo_has_accepted_promisor_remote(struct repository *r)
 227 {
 228         struct promisor_remote *p;
 229
 230         promisor_remote_init(r);
 231
 232         for (p = r->promisor_remote_config->promisors; p; p = p->next)
 233                 if (p->accepted)
 234                         return 1;
 235         return 0;
 236 }
 237
 238 static int remove_fetched_oids(struct repository *repo,
 239                                struct object_id **oids,
 240                                int oid_nr, int to_free)
 241 {
 242         int i, remaining_nr = 0;
 243         int *remaining = xcalloc(oid_nr, sizeof(*remaining));
 244         struct object_id *old_oids = *oids;
 245         struct object_id *new_oids;
 246
 247         for (i = 0; i < oid_nr; i++)
 248                 if (odb_read_object_info_extended(repo->objects, &old_oids[i], NULL,
 249                                                   OBJECT_INFO_SKIP_FETCH_OBJECT)) {
 250                         remaining[i] = 1;
 251                         remaining_nr++;
 252                 }
 253
 254         if (remaining_nr) {
 255                 int j = 0;
 256                 CALLOC_ARRAY(new_oids, remaining_nr);
 257                 for (i = 0; i < oid_nr; i++)
 258                         if (remaining[i])
 259                                 oidcpy(&new_oids[j++], &old_oids[i]);
 260                 *oids = new_oids;
 261                 if (to_free)
 262                         free(old_oids);
 263         }
 264
 265         free(remaining);
 266
 267         return remaining_nr;
 268 }
 269
 270 void promisor_remote_get_direct(struct repository *repo,
 271                                 const struct object_id *oids,
 272                                 int oid_nr)
 273 {
 274         struct promisor_remote *r;
 275         struct object_id *remaining_oids = (struct object_id *)oids;
 276         int remaining_nr = oid_nr;
 277         int to_free = 0;
 278         int i;
 279
 280         if (oid_nr == 0)
 281                 return;
 282
 283         promisor_remote_init(repo);
 284
 285         for (r = repo->promisor_remote_config->promisors; r; r = r->next) {
 286                 if (fetch_objects(repo, r->name, remaining_oids, remaining_nr) < 0) {
 287                         if (remaining_nr == 1)
 288                                 continue;
 289                         remaining_nr = remove_fetched_oids(repo, &remaining_oids,
 290                                                          remaining_nr, to_free);
 291                         if (remaining_nr) {
 292                                 to_free = 1;
 293                                 continue;
 294                         }
 295                 }
 296                 goto all_fetched;
 297         }
 298
 299         for (i = 0; i < remaining_nr; i++) {
 300                 if (is_promisor_object(repo, &remaining_oids[i]))
 301                         die(_("could not fetch %s from promisor remote"),
 302                             oid_to_hex(&remaining_oids[i]));
 303         }
 304
 305 all_fetched:
 306         if (to_free)
 307                 free(remaining_oids);
 308 }
 309
 310 static int allow_unsanitized(char ch)
 311 {
 312         if (ch == ',' || ch == ';' || ch == '%')
 313                 return 0;
 314         return ch > 32 && ch < 127;
 315 }
 316
 317 static void promisor_info_vecs(struct repository *repo,
 318                                struct strvec *names,
 319                                struct strvec *urls)
 320 {
 321         struct promisor_remote *r;
 322
 323         promisor_remote_init(repo);
 324
 325         for (r = repo->promisor_remote_config->promisors; r; r = r->next) {
 326                 const char *url;
 327                 char *url_key = xstrfmt("remote.%s.url", r->name);
 328
 329                 /* Only add remotes with a non empty URL */
 330                 if (!git_config_get_string_tmp(url_key, &url) && *url) {
 331                         strvec_push(names, r->name);
 332                         strvec_push(urls, url);
 333                 }
 334
 335                 free(url_key);
 336         }
 337 }
 338
 339 char *promisor_remote_info(struct repository *repo)
 340 {
 341         struct strbuf sb = STRBUF_INIT;
 342         int advertise_promisors = 0;
 343         struct strvec names = STRVEC_INIT;
 344         struct strvec urls = STRVEC_INIT;
 345
 346         git_config_get_bool("promisor.advertise", &advertise_promisors);
 347
 348         if (!advertise_promisors)
 349                 return NULL;
 350
 351         promisor_info_vecs(repo, &names, &urls);
 352
 353         if (!names.nr)
 354                 return NULL;
 355
 356         for (size_t i = 0; i < names.nr; i++) {
 357                 if (i)
 358                         strbuf_addch(&sb, ';');
 359                 strbuf_addstr(&sb, "name=");
 360                 strbuf_addstr_urlencode(&sb, names.v[i], allow_unsanitized);
 361                 strbuf_addstr(&sb, ",url=");
 362                 strbuf_addstr_urlencode(&sb, urls.v[i], allow_unsanitized);
 363         }
 364
 365         strvec_clear(&names);
 366         strvec_clear(&urls);
 367
 368         return strbuf_detach(&sb, NULL);
 369 }
 370
 371 /*
 372  * Find first index of 'nicks' where there is 'nick'. 'nick' is
 373  * compared case sensitively to the strings in 'nicks'. If not found
 374  * 'nicks->nr' is returned.
 375  */
 376 static size_t remote_nick_find(struct strvec *nicks, const char *nick)
 377 {
 378         for (size_t i = 0; i < nicks->nr; i++)
 379                 if (!strcmp(nicks->v[i], nick))
 380                         return i;
 381         return nicks->nr;
 382 }
 383
 384 enum accept_promisor {
 385         ACCEPT_NONE = 0,
 386         ACCEPT_KNOWN_URL,
 387         ACCEPT_KNOWN_NAME,
 388         ACCEPT_ALL
 389 };
 390
 391 static int should_accept_remote(enum accept_promisor accept,
 392                                 const char *remote_name, const char *remote_url,
 393                                 struct strvec *names, struct strvec *urls)
 394 {
 395         size_t i;
 396
 397         if (accept == ACCEPT_ALL)
 398                 return 1;
 399
 400         i = remote_nick_find(names, remote_name);
 401
 402         if (i >= names->nr)
 403                 /* We don't know about that remote */
 404                 return 0;
 405
 406         if (accept == ACCEPT_KNOWN_NAME)
 407                 return 1;
 408
 409         if (accept != ACCEPT_KNOWN_URL)
 410                 BUG("Unhandled 'enum accept_promisor' value '%d'", accept);
 411
 412         if (!remote_url || !*remote_url) {
 413                 warning(_("no or empty URL advertised for remote '%s'"), remote_name);
 414                 return 0;
 415         }
 416
 417         if (!strcmp(urls->v[i], remote_url))
 418                 return 1;
 419
 420         warning(_("known remote named '%s' but with URL '%s' instead of '%s'"),
 421                 remote_name, urls->v[i], remote_url);
 422
 423         return 0;
 424 }
 425
 426 static void filter_promisor_remote(struct repository *repo,
 427                                    struct strvec *accepted,
 428                                    const char *info)
 429 {
 430         struct strbuf **remotes;
 431         const char *accept_str;
 432         enum accept_promisor accept = ACCEPT_NONE;
 433         struct strvec names = STRVEC_INIT;
 434         struct strvec urls = STRVEC_INIT;
 435
 436         if (!git_config_get_string_tmp("promisor.acceptfromserver", &accept_str)) {
 437                 if (!*accept_str || !strcasecmp("None", accept_str))
 438                         accept = ACCEPT_NONE;
 439                 else if (!strcasecmp("KnownUrl", accept_str))
 440                         accept = ACCEPT_KNOWN_URL;
 441                 else if (!strcasecmp("KnownName", accept_str))
 442                         accept = ACCEPT_KNOWN_NAME;
 443                 else if (!strcasecmp("All", accept_str))
 444                         accept = ACCEPT_ALL;
 445                 else
 446                         warning(_("unknown '%s' value for '%s' config option"),
 447                                 accept_str, "promisor.acceptfromserver");
 448         }
 449
 450         if (accept == ACCEPT_NONE)
 451                 return;
 452
 453         if (accept != ACCEPT_ALL)
 454                 promisor_info_vecs(repo, &names, &urls);
 455
 456         /* Parse remote info received */
 457
 458         remotes = strbuf_split_str(info, ';', 0);
 459
 460         for (size_t i = 0; remotes[i]; i++) {
 461                 struct strbuf **elems;
 462                 const char *remote_name = NULL;
 463                 const char *remote_url = NULL;
 464                 char *decoded_name = NULL;
 465                 char *decoded_url = NULL;
 466
 467                 strbuf_strip_suffix(remotes[i], ";");
 468                 elems = strbuf_split(remotes[i], ',');
 469
 470                 for (size_t j = 0; elems[j]; j++) {
 471                         int res;
 472                         strbuf_strip_suffix(elems[j], ",");
 473                         res = skip_prefix(elems[j]->buf, "name=", &remote_name) ||
 474                                 skip_prefix(elems[j]->buf, "url=", &remote_url);
 475                         if (!res)
 476                                 warning(_("unknown element '%s' from remote info"),
 477                                         elems[j]->buf);
 478                 }
 479
 480                 if (remote_name)
 481                         decoded_name = url_percent_decode(remote_name);
 482                 if (remote_url)
 483                         decoded_url = url_percent_decode(remote_url);
 484
 485                 if (decoded_name && should_accept_remote(accept, decoded_name, decoded_url, &names, &urls))
 486                         strvec_push(accepted, decoded_name);
 487
 488                 strbuf_list_free(elems);
 489                 free(decoded_name);
 490                 free(decoded_url);
 491         }
 492
 493         strvec_clear(&names);
 494         strvec_clear(&urls);
 495         strbuf_list_free(remotes);
 496 }
 497
 498 char *promisor_remote_reply(const char *info)
 499 {
 500         struct strvec accepted = STRVEC_INIT;
 501         struct strbuf reply = STRBUF_INIT;
 502
 503         filter_promisor_remote(the_repository, &accepted, info);
 504
 505         if (!accepted.nr)
 506                 return NULL;
 507
 508         for (size_t i = 0; i < accepted.nr; i++) {
 509                 if (i)
 510                         strbuf_addch(&reply, ';');
 511                 strbuf_addstr_urlencode(&reply, accepted.v[i], allow_unsanitized);
 512         }
 513
 514         strvec_clear(&accepted);
 515
 516         return strbuf_detach(&reply, NULL);
 517 }
 518
 519 void mark_promisor_remotes_as_accepted(struct repository *r, const char *remotes)
 520 {
 521         struct strbuf **accepted_remotes = strbuf_split_str(remotes, ';', 0);
 522
 523         for (size_t i = 0; accepted_remotes[i]; i++) {
 524                 struct promisor_remote *p;
 525                 char *decoded_remote;
 526
 527                 strbuf_strip_suffix(accepted_remotes[i], ";");
 528                 decoded_remote = url_percent_decode(accepted_remotes[i]->buf);
 529
 530                 p = repo_promisor_remote_find(r, decoded_remote);
 531                 if (p)
 532                         p->accepted = 1;
 533                 else
 534                         warning(_("accepted promisor remote '%s' not found"),
 535                                 decoded_remote);
 536
 537                 free(decoded_remote);
 538         }
 539
 540         strbuf_list_free(accepted_remotes);
 541 }