Skip to content

Commit 747cf4a

Browse files
bepbep
committed
modules: Add support for direct version module imports in hugo.toml
Fixes #13964
1 parent d8774d7 commit 747cf4a

File tree

12 files changed

+563
-85
lines changed

12 files changed

+563
-85
lines changed

‎main_test.go‎

Lines changed: 60 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ func TestUnfinished(t *testing.T) {
5151
p := commonTestScriptsParam
5252
p.Dir = "testscripts/unfinished"
5353
// p.UpdateScripts = true
54+
// p.TestWork = true
5455

5556
testscript.Run(t, p)
5657
}
@@ -110,6 +111,35 @@ var commonTestScriptsParam = testscript.Params{
110111
}
111112
time.Sleep(time.Duration(i) * time.Second)
112113
},
114+
// tree lists a directory recursively to stdout as a simple tree.
115+
"tree": func(ts *testscript.TestScript, neg bool, args []string) {
116+
dirname := ts.MkAbs(args[0])
117+
118+
err := filepath.WalkDir(dirname, func(path string, d fs.DirEntry, err error) error {
119+
if err != nil {
120+
return err
121+
}
122+
rel, err := filepath.Rel(dirname, path)
123+
if err != nil {
124+
return err
125+
}
126+
if rel == "." {
127+
fmt.Fprintln(ts.Stdout(), ".")
128+
return nil
129+
}
130+
depth := strings.Count(rel, string(os.PathSeparator))
131+
prefix := strings.Repeat(" ", depth) + "└─"
132+
if d.IsDir() {
133+
fmt.Fprintf(ts.Stdout(), "%s%s/\n", prefix, d.Name())
134+
} else {
135+
fmt.Fprintf(ts.Stdout(), "%s%s\n", prefix, d.Name())
136+
}
137+
return nil
138+
})
139+
if err != nil {
140+
ts.Fatalf("%v", err)
141+
}
142+
},
113143
// ls lists a directory to stdout.
114144
"ls": func(ts *testscript.TestScript, neg bool, args []string) {
115145
dirname := ts.MkAbs(args[0])
@@ -128,7 +158,36 @@ var commonTestScriptsParam = testscript.Params{
128158
return
129159
}
130160
for _, fi := range fis {
131-
fmt.Fprintf(ts.Stdout(), "%s %04o %s %s\n", fi.Mode(), fi.Mode().Perm(), fi.ModTime().Format(time.RFC3339Nano), fi.Name())
161+
fmt.Fprintf(ts.Stdout(), "%s %04o %s %s\n", fi.Mode(), fi.Mode().Perm(), fi.ModTime().Format(time.RFC3339), fi.Name())
162+
}
163+
},
164+
// lsr lists a directory recursively to stdout.
165+
"lsr": func(ts *testscript.TestScript, neg bool, args []string) {
166+
dirname := ts.MkAbs(args[0])
167+
168+
err := filepath.WalkDir(dirname, func(path string, d fs.DirEntry, err error) error {
169+
if err != nil {
170+
return err
171+
}
172+
if d.IsDir() {
173+
return nil
174+
}
175+
176+
fi, err := d.Info()
177+
if err != nil {
178+
return err
179+
}
180+
181+
rel, err := filepath.Rel(dirname, path)
182+
if err != nil {
183+
return err
184+
}
185+
186+
fmt.Fprintf(ts.Stdout(), "%s %04o %s\n", fi.Mode(), fi.Mode().Perm(), filepath.ToSlash(rel))
187+
return nil
188+
})
189+
if err != nil {
190+
ts.Fatalf("%v", err)
132191
}
133192
},
134193
// append appends to a file with a leading newline.

‎modules/client.go‎

Lines changed: 123 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ import (
2525
"os/exec"
2626
"path/filepath"
2727
"regexp"
28+
"sort"
2829
"strings"
2930
"time"
3031

@@ -63,6 +64,10 @@ const vendord = "_vendor"
6364
const (
6465
goModFilename = "go.mod"
6566
goSumFilename = "go.sum"
67+
68+
// Checksum file for direct dependencies only,
69+
// that is, module imports with version set.
70+
hugoDirectSumFilename = "hugo.direct.sum"
6671
)
6772

6873
// NewClient creates a new Client that can be used to manage the Hugo Components
@@ -208,7 +213,7 @@ func (c *Client) Vendor() error {
208213
continue
209214
}
210215

211-
if !c.shouldVendor(t.Path()) {
216+
if c.shouldNotVendor(t.PathVersionQuery(false)) || c.shouldNotVendor(t.PathVersionQuery(true)) {
212217
continue
213218
}
214219

@@ -221,16 +226,16 @@ func (c *Client) Vendor() error {
221226
// See https://github.com/gohugoio/hugo/issues/8239
222227
// This is an error situation. We need something to vendor.
223228
if t.Mounts() == nil {
224-
return fmt.Errorf("cannot vendor module %q, need at least one mount", t.Path())
229+
return fmt.Errorf("cannot vendor module %q, need at least one mount", t.PathVersionQuery(false))
225230
}
226231

227-
fmt.Fprintln(&modulesContent, "# "+t.Path()+" "+t.Version())
232+
fmt.Fprintln(&modulesContent, "# "+t.PathVersionQuery(true)+" "+t.Version())
228233

229234
dir := t.Dir()
230235

231236
for _, mount := range t.Mounts() {
232237
sourceFilename := filepath.Join(dir, mount.Source)
233-
targetFilename := filepath.Join(vendorDir, t.Path(), mount.Source)
238+
targetFilename := filepath.Join(vendorDir, t.PathVersionQuery(true), mount.Source)
234239
fi, err := c.fs.Stat(sourceFilename)
235240
if err != nil {
236241
return fmt.Errorf("failed to vendor module: %w", err)
@@ -257,7 +262,7 @@ func (c *Client) Vendor() error {
257262
resourcesDir := filepath.Join(dir, files.FolderResources)
258263
_, err := c.fs.Stat(resourcesDir)
259264
if err == nil {
260-
if err := hugio.CopyDir(c.fs, resourcesDir, filepath.Join(vendorDir, t.Path(), files.FolderResources), nil); err != nil {
265+
if err := hugio.CopyDir(c.fs, resourcesDir, filepath.Join(vendorDir, t.PathVersionQuery(true), files.FolderResources), nil); err != nil {
261266
return fmt.Errorf("failed to copy resources to vendor dir: %w", err)
262267
}
263268
}
@@ -266,7 +271,7 @@ func (c *Client) Vendor() error {
266271
configDir := filepath.Join(dir, "config")
267272
_, err = c.fs.Stat(configDir)
268273
if err == nil {
269-
if err := hugio.CopyDir(c.fs, configDir, filepath.Join(vendorDir, t.Path(), "config"), nil); err != nil {
274+
if err := hugio.CopyDir(c.fs, configDir, filepath.Join(vendorDir, t.PathVersionQuery(true), "config"), nil); err != nil {
270275
return fmt.Errorf("failed to copy config dir to vendor dir: %w", err)
271276
}
272277
}
@@ -277,7 +282,7 @@ func (c *Client) Vendor() error {
277282
configFiles = append(configFiles, configFiles2...)
278283
configFiles = append(configFiles, filepath.Join(dir, "theme.toml"))
279284
for _, configFile := range configFiles {
280-
if err := hugio.CopyFile(c.fs, configFile, filepath.Join(vendorDir, t.Path(), filepath.Base(configFile))); err != nil {
285+
if err := hugio.CopyFile(c.fs, configFile, filepath.Join(vendorDir, t.PathVersionQuery(true), filepath.Base(configFile))); err != nil {
281286
if !herrors.IsNotExist(err) {
282287
return err
283288
}
@@ -434,13 +439,34 @@ func isProbablyModule(path string) bool {
434439
return module.CheckPath(path) == nil
435440
}
436441

442+
func (c *Client) downloadModuleVersion(path, version string) (*goModule, error) {
443+
args := []string{"mod", "download", "-json", fmt.Sprintf("%s@%s", path, version)}
444+
b := &bytes.Buffer{}
445+
446+
err := c.runGo(context.Background(), b, args...)
447+
if err != nil {
448+
return nil, fmt.Errorf("failed to download module %s@%s: %w", path, version, err)
449+
}
450+
451+
m := &goModule{}
452+
if err := json.NewDecoder(b).Decode(m); err != nil {
453+
return nil, fmt.Errorf("failed to decode module download result: %w", err)
454+
}
455+
456+
if m.Error != nil {
457+
return nil, errors.New(m.Error.Err)
458+
}
459+
460+
return m, nil
461+
}
462+
437463
func (c *Client) listGoMods() (goModules, error) {
438464
if c.GoModulesFilename == "" || !c.moduleConfig.hasModuleImport() {
439465
return nil, nil
440466
}
441467

442468
downloadModules := func(modules ...string) error {
443-
args := []string{"mod", "download", "-modcacherw"}
469+
args := []string{"mod", "download"}
444470
args = append(args, modules...)
445471
out := io.Discard
446472
err := c.runGo(context.Background(), out, args...)
@@ -521,6 +547,86 @@ func (c *Client) listGoMods() (goModules, error) {
521547
return modules, err
522548
}
523549

550+
func (c *Client) writeHugoDirectSum(mods Modules) error {
551+
if c.GoModulesFilename == "" {
552+
return nil
553+
}
554+
var sums []modSum
555+
for _, m := range mods {
556+
if m.Owner() == nil {
557+
// This is the project.
558+
continue
559+
}
560+
if m.IsGoMod() && m.VersionQuery() != "" {
561+
sums = append(sums, modSum{pathVersionKey: pathVersionKey{path: m.Path(), version: m.Version()}, sum: m.Sum()})
562+
}
563+
}
564+
565+
// Read the existing sums.
566+
existingSums, err := c.readModSumFile(hugoDirectSumFilename)
567+
if err != nil {
568+
return err
569+
}
570+
if len(sums) == 0 && len(existingSums) == 0 {
571+
// Nothing to do.
572+
return nil
573+
}
574+
575+
dirty := len(sums) != len(existingSums)
576+
for _, s1 := range sums {
577+
if s2, ok := existingSums[s1.pathVersionKey]; ok {
578+
if s1.sum != s2 {
579+
return fmt.Errorf("verifying %s@%s: checksum mismatch: %s != %s", s1.path, s1.version, s1.sum, s2)
580+
}
581+
} else if !dirty {
582+
dirty = true
583+
}
584+
}
585+
if !dirty {
586+
// Nothing changed.
587+
return nil
588+
}
589+
590+
// Write the sums file.
591+
// First sort the sums for reproducible output.
592+
sort.Slice(sums, func(i, j int) bool {
593+
pvi, pvj := sums[i].pathVersionKey, sums[j].pathVersionKey
594+
return pvi.path < pvj.path || (pvi.path == pvj.path && pvi.version < pvj.version)
595+
})
596+
597+
f, err := c.fs.OpenFile(filepath.Join(c.ccfg.WorkingDir, hugoDirectSumFilename), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o666)
598+
if err != nil {
599+
return err
600+
}
601+
defer f.Close()
602+
603+
for _, s := range sums {
604+
fmt.Fprintf(f, "%s %s %s\n", s.pathVersionKey.path, s.pathVersionKey.version, s.sum)
605+
}
606+
607+
return nil
608+
}
609+
610+
func (c *Client) readModSumFile(filename string) (map[pathVersionKey]string, error) {
611+
b, err := afero.ReadFile(c.fs, filepath.Join(c.ccfg.WorkingDir, filename))
612+
if err != nil {
613+
if herrors.IsNotExist(err) {
614+
return make(map[pathVersionKey]string), nil
615+
}
616+
return nil, err
617+
}
618+
lines := bytes.Split(b, []byte{'\n'})
619+
sums := make(map[pathVersionKey]string)
620+
for _, line := range lines {
621+
parts := bytes.Fields(line)
622+
if len(parts) == 3 {
623+
sums[pathVersionKey{path: string(parts[0]), version: string(parts[1])}] = string(parts[2])
624+
}
625+
}
626+
627+
return sums, nil
628+
}
629+
524630
func (c *Client) rewriteGoMod(name string, isGoMod map[string]bool) error {
525631
data, err := c.rewriteGoModRewrite(name, isGoMod)
526632
if err != nil {
@@ -707,8 +813,8 @@ func (c *Client) tidy(mods Modules, goModOnly bool) error {
707813
return nil
708814
}
709815

710-
func (c *Client) shouldVendor(path string) bool {
711-
return c.noVendor == nil || !c.noVendor.Match(path)
816+
func (c *Client) shouldNotVendor(path string) bool {
817+
return c.noVendor != nil && c.noVendor.Match(path)
712818
}
713819

714820
func (c *Client) createThemeDirname(modulePath string, isProjectMod bool) (string, error) {
@@ -773,6 +879,7 @@ func (cfg ClientConfig) toEnv() []string {
773879
keyVals := []string{
774880
"PWD", cfg.WorkingDir,
775881
"GO111MODULE", "on",
882+
"GOFLAGS", "-modcacherw",
776883
"GOPATH", cfg.CacheDir,
777884
"GOWORK", mcfg.Workspace, // Requires Go 1.18, see https://tip.golang.org/doc/go1.18
778885
// GOCACHE was introduced in Go 1.15. This matches the location derived from GOPATH above.
@@ -807,6 +914,7 @@ type goModule struct {
807914
Replace *goModule // replaced by this module
808915
Time *time.Time // time version was created
809916
Update *goModule // available update, if any (with -u)
917+
Sum string // checksum
810918
Main bool // is this the main module?
811919
Indirect bool // is this module only an indirect dependency of main module?
812920
Dir string // directory holding files for this module, if any
@@ -820,6 +928,11 @@ type goModuleError struct {
820928

821929
type goModules []*goModule
822930

931+
type modSum struct {
932+
pathVersionKey
933+
sum string
934+
}
935+
823936
func (modules goModules) GetByPath(p string) *goModule {
824937
if modules == nil {
825938
return nil

‎modules/client_test.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -227,7 +227,7 @@ func TestClientConfigToEnv(t *testing.T) {
227227

228228
env := ccfg.toEnv()
229229

230-
c.Assert(env, qt.DeepEquals, []string{"PWD=/mywork", "GO111MODULE=on", "GOPATH=/mycache", "GOWORK=", filepath.FromSlash("GOCACHE=/mycache/pkg/mod")})
230+
c.Assert(env, qt.DeepEquals, []string{"PWD=/mywork", "GO111MODULE=on", "GOFLAGS=-modcacherw", "GOPATH=/mycache", "GOWORK=", filepath.FromSlash("GOCACHE=/mycache/pkg/mod")})
231231

232232
ccfg = ClientConfig{
233233
WorkingDir: "/mywork",
@@ -246,6 +246,7 @@ func TestClientConfigToEnv(t *testing.T) {
246246
c.Assert(env, qt.DeepEquals, []string{
247247
"PWD=/mywork",
248248
"GO111MODULE=on",
249+
"GOFLAGS=-modcacherw",
249250
"GOPATH=/mycache",
250251
"GOWORK=myworkspace",
251252
filepath.FromSlash("GOCACHE=/mycache/pkg/mod"),

0 commit comments

Comments
 (0)